diff --git a/.gitlab/gitlab-ci-gitlabdotcom.yml b/.gitlab/gitlab-ci-gitlabdotcom.yml new file mode 100644 index 0000000000..eba4958546 --- /dev/null +++ b/.gitlab/gitlab-ci-gitlabdotcom.yml @@ -0,0 +1,6 @@ +sync-github-prs: + tags: linux + only: + - schedules + script: + - scripts/ci/scripts/github-prs-to-gitlab.sh ornladios/adios2 code.ornl.gov ecpcitest/adios2 diff --git a/.gitlab/gitlab-ci-olcf.yml b/.gitlab/gitlab-ci-olcf.yml index aaf43fe5c1..ac593185a1 100644 --- a/.gitlab/gitlab-ci-olcf.yml +++ b/.gitlab/gitlab-ci-olcf.yml @@ -1,15 +1,3 @@ -sync-github-prs: - tags: [nobatch] - only: - - schedules - variables: - GIT_STRATEGY: clone - script: - - module load python/3.7.0 - - pip3 install --user PyGithub - - module load git - - bash scripts/ci/gitlab-ci/pr-sync.sh - .all-steps: except: - schedules diff --git a/scripts/ci/gitlab-ci/pr-sync.sh b/scripts/ci/gitlab-ci/pr-sync.sh deleted file mode 100755 index 9e093daf98..0000000000 --- a/scripts/ci/gitlab-ci/pr-sync.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash - -set -e - -if [ -z "${SSH_KEY_BASE64}" ] -then - echo "Error: SSH_KEY_BASE64 is empty" - exit 1 -fi - -git --version - -export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - -# Start the ssh agent -eval $(ssh-agent -s) - -# Add the necessary key -echo "${SSH_KEY_BASE64}" | base64 -d | tr -d '\r' | ssh-add - - -# Setup push access -git config remote.origin.pushurl $(echo ${CI_REPOSITORY_URL} | sed 's|.*@\([^\/]*\)\/|git@\1:|') - -# Setup the github upstream remote -if git remote | grep -q upstream -then - git remote rm upstream -fi -git remote add upstream https://github.com/ornladios/ADIOS2.git -git fetch -p upstream - -# Retrieve open PRs -OPEN_PR_BRANCHES="$(python3 -c 'from github import Github; print(" ".join(["pr%d_%s" % (pr.number, pr.head.ref) for pr in Github().get_repo("ornladios/ADIOS2").get_pulls(state="open")]))')" -echo "Open PRs:" -for PR in ${OPEN_PR_BRANCHES} -do - echo " ${PR}" -done - -# Retrieve sync'd PRs -SYNCD_PR_BRANCHES="$(echo $(git ls-remote origin github/pr* | sed -n 's|.*/github/\(pr[0-9].*\)|\1|p'))" -echo "Syncd PRs:" -for PR in ${SYNCD_PR_BRANCHES} -do - echo " ${PR}" -done - -# Determine any closed PRs that are currently sync'd -SYNCD_CLOSED_PR_BRANCHES="" -if [ -n "${SYNCD_PR_BRANCHES}" ] -then - for SPR in ${SYNCD_PR_BRANCHES} - do - if [ -n "${OPEN_PR_BRANCHES}" ] - then - IS_OPEN=0 - for OPR in ${OPEN_PR_BRANCHES} - do - if [ "${SPR}" = "${OPR}" ] - then - IS_OPEN=1 - break - fi - done - if [ ${IS_OPEN} -eq 0 ] - then - SYNCD_CLOSED_PR_BRANCHES="${SYNCD_CLOSED_PR_BRANCHES} ${SPR}" - fi - fi - done -fi -echo "Syncd Closed PRs:" -for PR in ${SYNCD_CLOSED_PR_BRANCHES} -do - echo " ${PR}" -done - -# Delete any sync'd PRs -if [ -n "${SYNCD_CLOSED_PR_BRANCHES}" ] -then - CLOSED_REFSPECS="" - for PR in ${SYNCD_CLOSED_PR_BRANCHES} - do - echo "Adding respec for closed ${PR}" - CLOSED_REFSPECS="${CLOSED_REFSPECS} :github/${PR}" - done - - echo "Removing closed PRs" - git push -f origin ${CLOSED_REFSPECS} -fi - -# Sync open PRs to OLCF -if [ -n "${OPEN_PR_BRANCHES}" ] -then - FETCH_REFSPECS="" - PUSH_REFSPECS="" - for PR in ${OPEN_PR_BRANCHES} - do - PR_NUM=$(expr "${PR}" : 'pr\([0-9]\+\)') - echo "Adding refspecs for ${PR}" - FETCH_REFSPECS="${FETCH_REFSPECS} +refs/pull/${PR_NUM}/head:refs/remotes/upstream/github/${PR}" - PUSH_REFSPECS="${PUSH_REFSPECS} github/${PR}:github/${PR}" - done - - echo "Fetching upstream refs for open PRs" - git fetch upstream ${FETCH_REFSPECS} - - for PR in ${OPEN_PR_BRANCHES} - do - echo "Building local branch for ${PR}" - git branch github/${PR} upstream/github/${PR} - done - - echo "Pushing branches for open PRs" - git push -f origin ${PUSH_REFSPECS} -fi diff --git a/scripts/ci/scripts/github-list-prs.py b/scripts/ci/scripts/github-list-prs.py new file mode 100755 index 0000000000..a365824118 --- /dev/null +++ b/scripts/ci/scripts/github-list-prs.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +import sys +import requests + +import argparse +from github import Github + +parser = argparse.ArgumentParser(description="List open PRs") +group = parser.add_mutually_exclusive_group() +group.add_argument("-o", "--open", help="List open PRs", + action="store_true", default=True) +group.add_argument("-c", "--closed", help="List closed PRs", + action="store_true", default=False) +group.add_argument("-a", "--all", help="List all PRs", + action="store_true", default=False) +parser.add_argument("repo", help="GitHub repo (org/repo or user/repo)") +args = parser.parse_args() + +if args.open: + state = "open" +elif args.closed: + state = "closed" +else: #args.all + state = "all" + +try: + response = requests.get('https://api.github.com/repos/%s/pulls?state=%s' % (args.repo, state)) + response.raise_for_status() + prs = response.json() +except: + sys.exit(1) + +for pr in prs: + print("%d %s" % (pr['number'], pr['head']['ref'])) diff --git a/scripts/ci/scripts/github-prs-to-gitlab.sh b/scripts/ci/scripts/github-prs-to-gitlab.sh new file mode 100755 index 0000000000..fcabbfde86 --- /dev/null +++ b/scripts/ci/scripts/github-prs-to-gitlab.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash + +set -e + +CLEANUP_SSH_AGENT=0 +CLEANUP_WORKDIR=0 + +function cleanup() +{ + echo "Cleanup:" + if [ ${CLEANUP_WORKDIR} -eq 1 ] + then + echo " Removing ${TMP_WORKDIR}" + rm -rf "${TMP_WORKDIR}" + fi + if [ ${CLEANUP_SSH_AGENT} -eq 1 ] + then + echo " Shutting down ssh-agent(${SSH_AGENT_PID})" + eval $(ssh-agent -k) + fi +} + + +if [ $# -ne 3 ] +then + echo "Usage: $0 " + exit 1 +fi + +GITHUB_REPO=$1 +GITLAB_HOST=$2 +GITLAB_REPO=$3 +GITLAB_URL="git@${GITLAB_HOST}:${GITLAB_REPO}.git" +BASEDIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +if [ -z "${GITLAB_SSH_KEY_BASE64}" ] +then + echo "Error: GITLAB_SSH_KEY_BASE64 is empty" + exit 1 +fi + +trap cleanup EXIT + +# Start the ssh agent +eval $(ssh-agent -s) +CLEANUP_SSH_AGENT=1 + +# Add the necessary key +echo "${GITLAB_SSH_KEY_BASE64}" | base64 -d | tr -d '\r' | ssh-add - + +git --version + +export GIT_SSH_COMMAND="ssh -F /dev/null -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + +# Setup the local repo with two remotes +TMP_WORKDIR=$(mktemp -d) +CLEANUP_WORKDIR=1 +echo "Initializing local repo in ${TMP_WORKDIR}" +cd ${TMP_WORKDIR} +git init +git remote add github https://github.com/${GITHUB_REPO}.git +git remote add gitlab ${GITLAB_URL} + +# Retrieve open PRs +OPEN_PR_BRANCHES="$(python3 ${BASEDIR}/github-list-prs.py -o ${GITHUB_REPO} | awk '{print "pr"$1"_"$2}' | sort -r)" +echo "Open PRs:" +for PR in ${OPEN_PR_BRANCHES} +do + echo " ${PR}" +done +echo + +# Retrieve sync'd PRs +SYNCD_PR_BRANCHES="$(echo $(git ls-remote gitlab github/pr* | awk '{print $2}' | sed 's|^refs/heads/github/||' | sort -r))" +echo "Syncd PRs:" +for PR in ${SYNCD_PR_BRANCHES} +do + echo " ${PR}" +done +echo + +# Determine any closed PRs that are currently sync'd +SYNCD_CLOSED_PR_BRANCHES="" +if [ -n "${SYNCD_PR_BRANCHES}" ] +then + for SPR in ${SYNCD_PR_BRANCHES} + do + if [ -n "${OPEN_PR_BRANCHES}" ] + then + IS_OPEN=0 + for OPR in ${OPEN_PR_BRANCHES} + do + if [ "${SPR}" = "${OPR}" ] + then + IS_OPEN=1 + break + fi + done + if [ ${IS_OPEN} -eq 0 ] + then + SYNCD_CLOSED_PR_BRANCHES="${SYNCD_CLOSED_PR_BRANCHES} ${SPR}" + fi + fi + done +fi +echo "Syncd Closed PRs:" +for PR in ${SYNCD_CLOSED_PR_BRANCHES} +do + echo " ${PR}" +done +echo + +# Delete any sync'd PRs +if [ -n "${SYNCD_CLOSED_PR_BRANCHES}" ] +then + CLOSED_REFSPECS="" + for PR in ${SYNCD_CLOSED_PR_BRANCHES} + do + echo "Adding respec for closed ${PR}" + CLOSED_REFSPECS="${CLOSED_REFSPECS} :github/${PR}" + done +fi + +# Sync open PRs to OLCF +if [ -n "${OPEN_PR_BRANCHES}" ] +then + FETCH_REFSPECS="" + OPEN_REFSPECS="" + for PR in ${OPEN_PR_BRANCHES} + do + PR_NUM=$(expr "${PR}" : 'pr\([0-9]\+\)') + FETCH_REFSPECS="${FETCH_REFSPECS} +refs/pull/${PR_NUM}/head:refs/remotes/github/${PR}" + OPEN_REFSPECS="${OPEN_REFSPECS} github/${PR}:github/${PR}" + done + + echo "Fetching GitHub refs for open PRs" + git fetch -q github ${FETCH_REFSPECS} + echo + + echo "Building local branches for open PRs" + for PR in ${OPEN_PR_BRANCHES} + do + git branch -q github/${PR} github/${PR} + done + echo +fi + +if [ -n "${CLOSED_REFSPECS}" -o -n "${OPEN_REFSPECS}" ] +then + echo "Syncing PRs to GitLab" + git push --porcelain -f gitlab ${CLOSED_REFSPECS} ${OPEN_REFSPECS} + echo +fi