Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Install PyYAML package using system apt or dnf #1773

Merged
merged 2 commits into from
Oct 10, 2024
Merged

Install PyYAML package using system apt or dnf #1773

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b6bc99a
Install PyYAML package using system apt or dnf
chmouel Oct 10, 2024
d836da9
docs(github): improve comments for installation ID retrieval
chmouel Oct 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
267 changes: 137 additions & 130 deletions hack/gh-workflow-ci.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,133 +3,140 @@
set -eufo pipefail

create_pac_github_app_secret() {
local app_private_key="${1}"
local application_id="${2}"
local webhook_secret="${3}"
kubectl delete secret -n pipelines-as-code pipelines-as-code-secret || true
kubectl -n pipelines-as-code create secret generic pipelines-as-code-secret \
--from-literal github-private-key="${app_private_key}" \
--from-literal github-application-id="${application_id}" \
--from-literal webhook.secret="${webhook_secret}"
kubectl patch configmap -n pipelines-as-code -p "{\"data\":{\"bitbucket-cloud-check-source-ip\": \"false\"}}" \
--type merge pipelines-as-code

# restart controller
kubectl -n pipelines-as-code delete pod -l app.kubernetes.io/name=controller

echo -n "Waiting for controller to restart"
i=0
while true; do
[[ ${i} == 120 ]] && exit 1
ep=$(kubectl get ep -n pipelines-as-code pipelines-as-code-controller -o jsonpath='{.subsets[*].addresses[*].ip}')
[[ -n ${ep} ]] && break
sleep 2
echo -n "."
i=$((i + 1))
done
echo
local app_private_key="${1}"
local application_id="${2}"
local webhook_secret="${3}"
kubectl delete secret -n pipelines-as-code pipelines-as-code-secret || true
kubectl -n pipelines-as-code create secret generic pipelines-as-code-secret \
--from-literal github-private-key="${app_private_key}" \
--from-literal github-application-id="${application_id}" \
--from-literal webhook.secret="${webhook_secret}"
kubectl patch configmap -n pipelines-as-code -p "{\"data\":{\"bitbucket-cloud-check-source-ip\": \"false\"}}" \
--type merge pipelines-as-code

# restart controller
kubectl -n pipelines-as-code delete pod -l app.kubernetes.io/name=controller

echo -n "Waiting for controller to restart"
i=0
while true; do
[[ ${i} == 120 ]] && exit 1
ep=$(kubectl get ep -n pipelines-as-code pipelines-as-code-controller -o jsonpath='{.subsets[*].addresses[*].ip}')
[[ -n ${ep} ]] && break
sleep 2
echo -n "."
i=$((i + 1))
done
echo
}

create_second_github_app_controller_on_ghe() {
local test_github_second_smee_url="${1}"
local test_github_second_private_key="${2}"
local test_github_second_webhook_secret="${3}"

# this is to handle compatibilty until the PR #1518 is merged
[[ -e ./hack/second-controller.py ]] || exit 0
python3 -m pip install PyYAML
./hack/second-controller.py \
--controller-image="ko" \
--smee-url="${test_github_second_smee_url}" \
--ingress-domain="paac-127-0-0-1.nip.io" \
--namespace="pipelines-as-code" \
ghe | tee /tmp/generated.yaml

ko apply -f /tmp/generated.yaml
kubectl delete secret -n pipelines-as-code ghe-secret || true
kubectl -n pipelines-as-code create secret generic ghe-secret \
--from-literal github-private-key="${test_github_second_private_key}" \
--from-literal github-application-id="2" \
--from-literal webhook.secret="${test_github_second_webhook_secret}"
sed "s/name: pipelines-as-code/name: ghe-configmap/" <config/302-pac-configmap.yaml | kubectl apply -n pipelines-as-code -f-
kubectl patch configmap -n pipelines-as-code ghe-configmap -p '{"data":{"application-name": "Pipelines as Code GHE"}}'
kubectl -n pipelines-as-code delete pod -l app.kubernetes.io/name=ghe-controller
local test_github_second_smee_url="${1}"
local test_github_second_private_key="${2}"
local test_github_second_webhook_secret="${3}"

if hash -v apt >/dev/null 2>&1; then
apt update &&
apt install -y python3-yaml
elif hash -v dnf >/dev/null 2>&1; then
dnf install -y python3-pyyaml
else
# TODO(chmouel): setup a virtualenvironment instead
python3 -m pip install --break-system-packages PyYAML
fi

./hack/second-controller.py \
--controller-image="ko" \
--smee-url="${test_github_second_smee_url}" \
--ingress-domain="paac-127-0-0-1.nip.io" \
--namespace="pipelines-as-code" \
ghe | tee /tmp/generated.yaml

ko apply -f /tmp/generated.yaml
kubectl delete secret -n pipelines-as-code ghe-secret || true
kubectl -n pipelines-as-code create secret generic ghe-secret \
--from-literal github-private-key="${test_github_second_private_key}" \
--from-literal github-application-id="2" \
--from-literal webhook.secret="${test_github_second_webhook_secret}"
sed "s/name: pipelines-as-code/name: ghe-configmap/" <config/302-pac-configmap.yaml | kubectl apply -n pipelines-as-code -f-
kubectl patch configmap -n pipelines-as-code ghe-configmap -p '{"data":{"application-name": "Pipelines as Code GHE"}}'
kubectl -n pipelines-as-code delete pod -l app.kubernetes.io/name=ghe-controller
}

run_e2e_tests() {
bitbucket_cloud_token="${1}"
webhook_secret="${2}"
test_gitea_smeeurl="${3}"
installation_id="${4}"
gh_apps_token="${5}"
test_github_second_token="${6}"
gitlab_token="${7}"

# Nothing specific to webhook here it just that repo is private in that org and that's what we want to test
export TEST_GITHUB_PRIVATE_TASK_URL="https://github.com/openshift-pipelines/pipelines-as-code-e2e-tests-private/blob/main/remote_task.yaml"
export TEST_GITHUB_PRIVATE_TASK_NAME="task-remote"

export GO_TEST_FLAGS="-v -race -failfast"

export TEST_BITBUCKET_CLOUD_API_URL=https://api.bitbucket.org/2.0
export TEST_BITBUCKET_CLOUD_E2E_REPOSITORY=cboudjna/pac-e2e-tests
export TEST_BITBUCKET_CLOUD_TOKEN=${bitbucket_cloud_token}
export TEST_BITBUCKET_CLOUD_USER=cboudjna

export TEST_EL_URL="http://${CONTROLLER_DOMAIN_URL}"
export TEST_EL_WEBHOOK_SECRET="${webhook_secret}"

export TEST_GITEA_API_URL="http://localhost:3000"
## This is the URL used to forward requests from the webhook to the paac controller
## badly named!
export TEST_GITEA_SMEEURL="${test_gitea_smeeurl}"
export TEST_GITEA_USERNAME=pac
export TEST_GITEA_PASSWORD=pac
export TEST_GITEA_REPO_OWNER=pac/pac

export TEST_GITHUB_API_URL=api.github.com
export TEST_GITHUB_REPO_INSTALLATION_ID="${installation_id}"
export TEST_GITHUB_REPO_OWNER_GITHUBAPP=openshift-pipelines/pipelines-as-code-e2e-tests
export TEST_GITHUB_REPO_OWNER_WEBHOOK=openshift-pipelines/pipelines-as-code-e2e-tests-webhook
export TEST_GITHUB_TOKEN="${gh_apps_token}"

export TEST_GITHUB_SECOND_API_URL=ghe.pipelinesascode.com
export TEST_GITHUB_SECOND_EL_URL=http://ghe.paac-127-0-0-1.nip.io
export TEST_GITHUB_SECOND_REPO_OWNER_GITHUBAPP=pipelines-as-code/e2e
# TODO: webhook repo for second github
# export TEST_GITHUB_SECOND_REPO_OWNER_WEBHOOK=openshift-pipelines/pipelines-as-code-e2e-tests-webhook
export TEST_GITHUB_SECOND_REPO_INSTALLATION_ID=1
export TEST_GITHUB_SECOND_TOKEN="${test_github_second_token}"

export TEST_GITLAB_API_URL="https://gitlab.com"
export TEST_GITLAB_PROJECT_ID="34405323"
export TEST_GITLAB_TOKEN=${gitlab_token}
# https://gitlab.com/gitlab-com/alliances/ibm-red-hat/sandbox/openshift-pipelines/pac-e2e-tests
make test-e2e
bitbucket_cloud_token="${1}"
webhook_secret="${2}"
test_gitea_smeeurl="${3}"
installation_id="${4}"
gh_apps_token="${5}"
test_github_second_token="${6}"
gitlab_token="${7}"

# Nothing specific to webhook here it just that repo is private in that org and that's what we want to test
export TEST_GITHUB_PRIVATE_TASK_URL="https://github.com/openshift-pipelines/pipelines-as-code-e2e-tests-private/blob/main/remote_task.yaml"
export TEST_GITHUB_PRIVATE_TASK_NAME="task-remote"

export GO_TEST_FLAGS="-v -race -failfast"

export TEST_BITBUCKET_CLOUD_API_URL=https://api.bitbucket.org/2.0
export TEST_BITBUCKET_CLOUD_E2E_REPOSITORY=cboudjna/pac-e2e-tests
export TEST_BITBUCKET_CLOUD_TOKEN=${bitbucket_cloud_token}
export TEST_BITBUCKET_CLOUD_USER=cboudjna

export TEST_EL_URL="http://${CONTROLLER_DOMAIN_URL}"
export TEST_EL_WEBHOOK_SECRET="${webhook_secret}"

export TEST_GITEA_API_URL="http://localhost:3000"
## This is the URL used to forward requests from the webhook to the paac controller
## badly named!
export TEST_GITEA_SMEEURL="${test_gitea_smeeurl}"
export TEST_GITEA_USERNAME=pac
export TEST_GITEA_PASSWORD=pac
export TEST_GITEA_REPO_OWNER=pac/pac

export TEST_GITHUB_API_URL=api.github.com
export TEST_GITHUB_REPO_INSTALLATION_ID="${installation_id}"
export TEST_GITHUB_REPO_OWNER_GITHUBAPP=openshift-pipelines/pipelines-as-code-e2e-tests
export TEST_GITHUB_REPO_OWNER_WEBHOOK=openshift-pipelines/pipelines-as-code-e2e-tests-webhook
export TEST_GITHUB_TOKEN="${gh_apps_token}"

export TEST_GITHUB_SECOND_API_URL=ghe.pipelinesascode.com
export TEST_GITHUB_SECOND_EL_URL=http://ghe.paac-127-0-0-1.nip.io
export TEST_GITHUB_SECOND_REPO_OWNER_GITHUBAPP=pipelines-as-code/e2e
# TODO: webhook repo for second github
# export TEST_GITHUB_SECOND_REPO_OWNER_WEBHOOK=openshift-pipelines/pipelines-as-code-e2e-tests-webhook
export TEST_GITHUB_SECOND_REPO_INSTALLATION_ID=1
export TEST_GITHUB_SECOND_TOKEN="${test_github_second_token}"

export TEST_GITLAB_API_URL="https://gitlab.com"
export TEST_GITLAB_PROJECT_ID="34405323"
export TEST_GITLAB_TOKEN=${gitlab_token}
# https://gitlab.com/gitlab-com/alliances/ibm-red-hat/sandbox/openshift-pipelines/pac-e2e-tests
make test-e2e
}

collect_logs() {
mkdir -p /tmp/logs
kind export logs /tmp/logs
[[ -d /tmp/gosmee-replay ]] && cp -a /tmp/gosmee-replay /tmp/logs/

kubectl get pipelineruns -A -o yaml >/tmp/logs/pac-pipelineruns.yaml
kubectl get repositories.pipelinesascode.tekton.dev -A -o yaml >/tmp/logs/pac-repositories.yaml
kubectl get configmap -n pipelines-as-code -o yaml >/tmp/logs/pac-configmap
kubectl get events -A >/tmp/logs/events

allNamespaces=$(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}')
for ns in ${allNamespaces}; do
mkdir -p /tmp/logs/ns/${ns}
for type in pods pipelineruns repositories configmap; do
kubectl get ${type} -n ${ns} -o yaml >/tmp/logs/ns/${ns}/${type}.yaml
done
kubectl -n ${ns} get events >/tmp/logs/ns/${ns}/events
done
mkdir -p /tmp/logs
kind export logs /tmp/logs
[[ -d /tmp/gosmee-replay ]] && cp -a /tmp/gosmee-replay /tmp/logs/

kubectl get pipelineruns -A -o yaml >/tmp/logs/pac-pipelineruns.yaml
kubectl get repositories.pipelinesascode.tekton.dev -A -o yaml >/tmp/logs/pac-repositories.yaml
kubectl get configmap -n pipelines-as-code -o yaml >/tmp/logs/pac-configmap
kubectl get events -A >/tmp/logs/events

allNamespaces=$(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}')
for ns in ${allNamespaces}; do
mkdir -p /tmp/logs/ns/${ns}
for type in pods pipelineruns repositories configmap; do
kubectl get ${type} -n ${ns} -o yaml >/tmp/logs/ns/${ns}/${type}.yaml
done
kubectl -n ${ns} get events >/tmp/logs/ns/${ns}/events
done
}

help() {
cat <<EOF
cat <<EOF
Usage: $0 <command> [args]

Shell script to run e2e tests from GitHub Actions CI
Expand All @@ -150,24 +157,24 @@ EOF

case ${1-""} in
create_pac_github_app_secret)
create_pac_github_app_secret "${2}" "${3}" "${4}"
;;
create_pac_github_app_secret "${2}" "${3}" "${4}"
;;
create_second_github_app_controller_on_ghe)
create_second_github_app_controller_on_ghe "${2}" "${3}" "${4}"
;;
create_second_github_app_controller_on_ghe "${2}" "${3}" "${4}"
;;
run_e2e_tests)
run_e2e_tests "${2}" "${3}" "${4}" "${5}" "${6}" "${7}" "${8}"
;;
run_e2e_tests "${2}" "${3}" "${4}" "${5}" "${6}" "${7}" "${8}"
;;
collect_logs)
collect_logs
;;
collect_logs
;;
help)
help
exit 0
;;
help
exit 0
;;
*)
echo "Unknown command ${1-}"
help
exit 1
;;
echo "Unknown command ${1-}"
help
exit 1
;;
esac
22 changes: 16 additions & 6 deletions pkg/provider/github/app/token.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,16 @@ func NewInstallation(req *http.Request, run *params.Run, repo *v1alpha1.Reposito
}
}

// GetAndUpdateInstallationID retrieves and updates the installation ID for the GitHub App.
// It generates a JWT token, lists all installations, and matches repositories to their installation IDs.
// If a matching repository is found, it returns the enterprise host, token, and installation ID.
func (ip *Install) GetAndUpdateInstallationID(ctx context.Context) (string, string, int64, error) {
var (
enterpriseHost, token string
installationID int64
)

// Generate a JWT token for authentication
jwtToken, err := ip.GenerateJWT(ctx)
if err != nil {
return "", "", 0, err
Expand All @@ -50,14 +55,16 @@ func (ip *Install) GetAndUpdateInstallationID(ctx context.Context) (string, stri
apiURL := *ip.ghClient.APIURL
enterpriseHost = ip.request.Header.Get("X-GitHub-Enterprise-Host")
if enterpriseHost != "" {
// NOTE: Hopefully this works even when the ghe URL is on another host than the api URL
// NOTE: Hopefully this works even when the GHE URL is on another host than the API URL
apiURL = "https://" + enterpriseHost + "/api/v3"
}

logger := logging.FromContext(ctx)
opt := &gt.ListOptions{PerPage: ip.ghClient.PaginedNumber}
client, _, _ := github.MakeClient(ctx, apiURL, jwtToken)
installationData := []*gt.Installation{}

// List all installations
for {
installationSet, resp, err := client.Apps.ListInstallations(ctx, opt)
if err != nil {
Expand All @@ -70,9 +77,7 @@ func (ip *Install) GetAndUpdateInstallationID(ctx context.Context) (string, stri
opt.Page = resp.NextPage
}

/* each installationID can have list of repository
ref: https://docs.github.com/en/developers/apps/building-github-apps/authenticating-with-github-apps#authenticating-as-an-installation ,
https://docs.github.com/en/rest/apps/installations?apiVersion=2022-11-28#list-repositories-accessible-to-the-app-installation */
// Iterate through each installation to find a matching repository
for i := range installationData {
if installationData[i].ID == nil {
return "", "", 0, fmt.Errorf("installation ID is nil")
Expand Down Expand Up @@ -102,27 +107,32 @@ func (ip *Install) GetAndUpdateInstallationID(ctx context.Context) (string, stri
return enterpriseHost, token, installationID, nil
}

// matchRepos matching github repositories to its installation IDs.
// matchRepos matches GitHub repositories to their installation IDs.
// It lists all repositories accessible to the app installation and checks if
// any match the repository URL in the spec.
func (ip *Install) matchRepos(ctx context.Context) (bool, error) {
installationRepoList, err := github.ListRepos(ctx, ip.ghClient)
if err != nil {
return false, err
}
ip.repoList = append(ip.repoList, installationRepoList...)
for i := range installationRepoList {
// If URL matches with repo spec url then we can break for loop
// If URL matches with repo spec URL then we can break the loop
if installationRepoList[i] == ip.repo.Spec.URL {
return true, nil
}
}
return false, nil
}

// JWTClaim represents the JWT claims for the GitHub App.
type JWTClaim struct {
Issuer int64 `json:"iss"`
jwt.RegisteredClaims
}

// GenerateJWT generates a JWT token for the GitHub App.
// It retrieves the application ID and private key, sets the claims, and signs the token.
func (ip *Install) GenerateJWT(ctx context.Context) (string, error) {
// TODO: move this out of here
gh := github.New()
Expand Down