diff --git a/.github/workflows/publish-deb.yml b/.github/workflows/publish-deb.yml index d6c4191db..ecbf810e3 100644 --- a/.github/workflows/publish-deb.yml +++ b/.github/workflows/publish-deb.yml @@ -1,7 +1,7 @@ name: Build and Publish Package on: - create + create: jobs: publish_apisix: @@ -10,6 +10,8 @@ jobs: timeout-minutes: 60 env: VAR_DEB_WORKBENCH_DIR: /tmp/output + VAR_COS_BUCKET_REPO: ${{ secrets.VAR_COS_BUCKET_REPO }} + VAR_COS_BUCKET_CI: ${{ secrets.VAR_COS_BUCKET_CI }} VAR_OS: debian VAR_CODENAME: bullseye VAR_OS_RELEASE: bullseye-slim @@ -73,3 +75,40 @@ jobs: name: "apisix-base_${{ steps.tag_env.outputs.version}}-0~${{ env.VAR_OS }}${{ env.VAR_OS_RELEASE }}_amd64.deb" path: "${{ env.VAR_DEB_WORKBENCH_DIR }}/apisix-base_${{ steps.tag_env.outputs.version}}-0~${{ env.VAR_OS }}${{ env.VAR_OS_RELEASE }}_amd64.deb" + - name: DEB repo deps init + env: + TENCENT_COS_SECRETID: ${{ secrets.TENCENT_COS_SECRETID }} + TENCENT_COS_SECRETKEY: ${{ secrets.TENCENT_COS_SECRETKEY }} + run: | + sudo pip install coscmd + sudo -E ./utils/publish-deb.sh init_cos_utils + + - name: DEB repo init for debian + run: | + sudo -E ./utils/publish-deb.sh repo_clone + + - name: DEB repo package update + env: + DEB_GPG_MAIL: ${{ secrets.DEB_GPG_MAIL }} + GPG_NAME: ${{ secrets.GPG_NAME }} + GPG_MAIL: ${{ secrets.GPG_MAIL }} + run: | + echo "${{ secrets.DEB_GPG_PRIV_KEY }}" >> /tmp/deb-gpg-publish.private + echo "${{ secrets.DEB_GPG_PASSPHRASE }}" >> /tmp/deb-gpg-publish.passphrase + sudo -E ./utils/publish-deb.sh init_freight_utils + sudo -E ./utils/publish-deb.sh init_gpg + + - name: DEB repo backup for debian + run: | + sudo -E ./utils/publish-deb.sh dists_backup + + - name: DEB repo refresh for debian + run: | + sudo -E ./utils/publish-deb.sh repo_rebuild + sudo -E ./utils/publish-deb.sh repo_ci_upload + + - name: DEB repo publish for debian + run: | + sudo -E ./utils/publish-deb.sh repo_upload + sudo -E ./utils/publish-rpm.sh repo_publish + diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a88de8e24..08739741f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -130,6 +130,7 @@ jobs: TENCENT_COS_SECRETID: ${{ secrets.TENCENT_COS_SECRETID }} TENCENT_COS_SECRETKEY: ${{ secrets.TENCENT_COS_SECRETKEY }} run: | + sudo pip install coscmd sudo -E ./utils/publish-rpm.sh init_cos_utils - name: RPM repo init for centos and redhat diff --git a/utils/publish-deb.sh b/utils/publish-deb.sh index 53b34d2fb..4d4dcbfc0 100755 --- a/utils/publish-deb.sh +++ b/utils/publish-deb.sh @@ -10,12 +10,16 @@ env # Runtime default config # ======================================= VAR_FREIGHT_UTILS_VERSION=${VAR_FREIGHT_UTILS_VERSION:-v0.3.13} -VAR_TENCENT_COS_UTILS_VERSION=${VAR_TENCENT_COS_UTILS_VERSION:-v0.11.0-beta} VAR_DEB_WORKBENCH_DIR=${VAR_DEB_WORKBENCH_DIR:-/tmp/output} VAR_GPG_PRIV_KET=${VAR_GPG_PRIV_KET:-/tmp/deb-gpg-publish.private} VAR_GPG_PASSPHRASE=${VAR_GPG_PASSPHRASE:-/tmp/deb-gpg-publish.passphrase} -COS_CMD=coscli +COS_REGION=${COS_REGION:-"ap-guangzhou"} +COS_GLOBAL_REGION=${COS_GLOBAL_REGION:-"accelerate"} +COS_PART_SIZE=${COS_PART_SIZE:-"10"} +VAR_COS_REGION_DNS="cos.${COS_REGION}.myqcloud.com" +VAR_COS_GLOBAL_REGION_DNS="cos.${COS_GLOBAL_REGION}.myqcloud.com" + TAG_DATE=$(date +%Y%m%d) ARCH=${ARCH:-`(uname -m | tr '[:upper:]' '[:lower:]')`} arch_path="" @@ -41,30 +45,11 @@ _EOC_ # ======================================= # COS extension # ======================================= -func_cos_utils_install() { - if [[ $ARCH == "arm64" ]] || [[ $ARCH == "aarch64" ]]; then - wget https://github.com/tencentyun/coscli/archive/refs/tags/${VAR_TENCENT_COS_UTILS_VERSION}.tar.gz - tar -zxvf ${VAR_TENCENT_COS_UTILS_VERSION}.tar.gz - cd coscli-* && go build - mv coscli ../ - else - sudo curl -o /usr/bin/coscli -L "https://github.com/tencentyun/coscli/releases/download/${VAR_TENCENT_COS_UTILS_VERSION}/coscli-linux" - sudo chmod 755 /usr/bin/coscli - fi -} - func_cos_utils_credential_init() { - # ${1} - COS endpoint - # ${2} - COS SECRET_ID - # ${3} - COS SECRET_KEY - cat > "${HOME}/.cos.yaml" <<_EOC_ -cos: - base: - secretid: ${2} - secretkey: ${3} - sessiontoken: "" - protocol: https -_EOC_ + # ${1} - COS SECRET_ID + # ${2} - COS SECRET_KEY + # ${3} - COS bucket name + coscmd config -a "${1}" -s "${2}" -b "${3}" -r ${COS_REGION} -p ${COS_PART_SIZE} } @@ -91,21 +76,20 @@ func_dists_backup() { # ${1} - bucket name # ${2} - COS path # ${3} - backup tag - $COS_CMD -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${arch_path}${2}/dists" "cos://${1}/packages/${arch_path}backup/${2}_dists_${3}" || true + coscmd copy -r "${1}.${VAR_COS_REGION_DNS}/packages/${arch_path}${2}/dists" "/packages/${arch_path}backup/${2}_dists_${3}" } func_pool_clone() { # ${1} - bucket name # ${2} - COS path # ${3} - local pool path - mkdir -p ${3} # --part-size indicates the file chunk size. # when the file is larger than --part-size, coscli will chunk the file by --part-size. # when uploading/downloading the file in chunks, it will enable breakpoint transfer by default, # which will generate cosresumabletask file and interfere with the file integrity. # ref: https://cloud.tencent.com/document/product/436/63669 - $COS_CMD -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${arch_path}${2}/pool" "${3}" + coscmd -b "${1}" -r "${COS_GLOBAL_REGION}" download -r "/packages/${arch_path}${2}/pool" "${3}" } func_dists_rebuild() { @@ -139,8 +123,8 @@ func_dists_rebuild() { } func_dists_upload_ci_repo() { - $COS_CMD -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${2}/packages/${arch_path}${3}" || true - $COS_CMD -e "${VAR_COS_ENDPOINT}" cp -r --part-size 512 "${1}" "cos://${2}/packages/${arch_path}${3}/dists/" + coscmd -b "${2}" delete -r -f "/packages/${arch_path}${3}" || true + coscmd -b "${2}" -r "${COS_GLOBAL_REGION}" upload -r "${1}" "/packages/${arch_path}${3}/dists" } func_deb_upload() { @@ -152,8 +136,6 @@ func_deb_upload() { # We will only upload apisix and apisix-base, # so the directory is fixed: pool/main/a. # Regardless of other packages. - - export COS_CMD=$COS_CMD export arch_path=$arch_path export BUCKET=$2 export OS=$3 @@ -161,24 +143,27 @@ func_deb_upload() { find "${1}" -type f -name "apisix_*.deb" \ -exec echo "upload : {}" \; \ -exec sh -c 'file=$(basename {}); \ - $COS_CMD -e "${VAR_COS_ENDPOINT}" cp {} --part-size 512 "cos://${BUCKET}/packages/${arch_path}${OS}/pool/${CODENAME}/main/a/apisix/${file}"' \; + coscmd -b "${BUCKET}" -r "${COS_GLOBAL_REGION}" upload {} "/packages/${arch_path}${OS}/pool/${CODENAME}/main/a/apisix/${file}"' \; find "${1}" -type f -name "apisix-base*.deb" \ -exec echo "upload : {}" \; \ -exec sh -c 'file=$(basename {}); \ - $COS_CMD -e "${VAR_COS_ENDPOINT}" cp {} --part-size 512 "cos://${BUCKET}/packages/${arch_path}${OS}/pool/${CODENAME}/main/a/apisix-base/${file}"' \; + coscmd -b "${BUCKET}" -r "${COS_GLOBAL_REGION}" upload {} "/packages/${arch_path}${OS}/pool/${CODENAME}/main/a/apisix-base/${file}"' \; } func_repo_publish() { - $COS_CMD -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${2}/packages/${arch_path}${3}/dists" || true - $COS_CMD -e "${VAR_COS_ENDPOINT}" cp -r --part-size 512 "cos://${1}/packages/${arch_path}${3}/dists" "cos://${2}/packages/${arch_path}${3}/dists" + # ${1} - CI bucket + # ${2} - repo publish bucket + # ${3} - COS path + coscmd delete -r -f "/packages/${arch_path}${3}/dists" || true + coscmd -b "${2}" copy -r "${1}.${VAR_COS_REGION_DNS}/packages/${arch_path}${3}/dists" "/packages/${arch_path}${3}/dists" } func_repo_backup_remove() { # ${1} - bucket name # ${2} - COS path # ${3} - backup tag - $COS_CMD -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${1}/packages/${arch_path}backup/${2}_dists_${3}" || true + coscmd -b "${1}" delete -r -f "/packages/${arch_path}backup/${2}_dists_${3}" || true } # ======================================= @@ -188,8 +173,7 @@ case_opt=$1 case ${case_opt} in init_cos_utils) - func_cos_utils_install - func_cos_utils_credential_init "${VAR_COS_ENDPOINT}" "${TENCENT_COS_SECRETID}" "${TENCENT_COS_SECRETKEY}" + func_cos_utils_credential_init "${TENCENT_COS_SECRETID}" "${TENCENT_COS_SECRETKEY}" "${VAR_COS_BUCKET_REPO}" ;; init_freight_utils) func_freight_utils_install @@ -210,8 +194,10 @@ repo_clone) repo_rebuild) func_dists_rebuild "/tmp/old_pool" "/tmp/freight" ${VAR_DEB_WORKBENCH_DIR} ${VAR_CODENAME} ;; -repo_upload) +repo_ci_upload) func_dists_upload_ci_repo "/tmp/freight/cache/dists" "${VAR_COS_BUCKET_CI}" "${VAR_OS}" + ;; +repo_upload) func_deb_upload "${VAR_DEB_WORKBENCH_DIR}" "${VAR_COS_BUCKET_REPO}" "${VAR_OS}" "${VAR_CODENAME}" ;; repo_publish) diff --git a/utils/publish-rpm.sh b/utils/publish-rpm.sh index 7753aea56..e530bbe30 100755 --- a/utils/publish-rpm.sh +++ b/utils/publish-rpm.sh @@ -9,12 +9,17 @@ env # ======================================= # Runtime default config # ======================================= -VAR_TENCENT_COS_UTILS_VERSION=${VAR_TENCENT_COS_UTILS_VERSION:-v0.11.0-beta} VAR_RPM_WORKBENCH_DIR=${VAR_RPM_WORKBENCH_DIR:-/tmp/output} VAR_GPG_PRIV_KET=${VAR_GPG_PRIV_KET:-/tmp/rpm-gpg-publish.private} VAR_GPG_PASSPHRASE=${VAR_GPG_PASSPHRASE:-/tmp/rpm-gpg-publish.passphrase} ARCH=${ARCH:-`(uname -m | tr '[:upper:]' '[:lower:]')`} +COS_REGION=${COS_REGION:-"ap-guangzhou"} +COS_GLOBAL_REGION=${COS_GLOBAL_REGION:-"accelerate"} +COS_PART_SIZE=${COS_PART_SIZE:-"10"} +VAR_COS_REGION_DNS="cos.${COS_REGION}.myqcloud.com" +VAR_COS_GLOBAL_REGION_DNS="cos.${COS_GLOBAL_REGION}.myqcloud.com" + # ======================================= # GPG extension # ======================================= @@ -42,24 +47,11 @@ func_gpg_key_load() { # ======================================= # COS extension # ======================================= -func_cos_utils_install() { - # ${1} - COS util version - curl -o /usr/bin/coscli -L "https://github.com/tencentyun/coscli/releases/download/${1}/coscli-linux" - chmod 755 /usr/bin/coscli -} - func_cos_utils_credential_init() { - # ${1} - COS endpoint - # ${2} - COS SECRET_ID - # ${3} - COS SECRET_KEY - cat > "${HOME}/.cos.yaml" <<_EOC_ -cos: - base: - secretid: ${2} - secretkey: ${3} - sessiontoken: "" - protocol: https -_EOC_ + # ${1} - COS SECRET_ID + # ${2} - COS SECRET_KEY + # ${3} - COS bucket name + coscmd config -a "${1}" -s "${2}" -b "${3}" -r ${COS_REGION} -p ${COS_PART_SIZE} } # ======================================= @@ -77,25 +69,25 @@ func_repo_clone() { # ${3} - target path # --part-size indicates the file chunk size. - # when the file is larger than --part-size, coscli will chunk the file by --part-size. + # when the file is larger than --part-size, coscmd will chunk the file by --part-size. # when uploading/downloading the file in chunks, it will enable breakpoint transfer by default, # which will generate cosresumabletask file and interfere with the file integrity. # ref: https://cloud.tencent.com/document/product/436/63669 - coscli -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${2}" "${3}" + coscmd -b "${1}" -r "${COS_GLOBAL_REGION}" download -r "/packages/${2}" "${3}" } func_repo_backup() { # ${1} - bucket name # ${2} - COS path # ${3} - backup tag - coscli -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${2}" "cos://${1}/packages/backup/${2}_${3}" + coscmd copy -r "${1}.${VAR_COS_REGION_DNS}/packages/${2}" "/packages/backup/${2}_${3}" } func_repo_backup_remove() { # ${1} - bucket name # ${2} - COS path # ${3} - backup tag - coscli -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${1}/packages/backup/${2}_${3}" + coscmd -b "${1}" delete -r -f "/packages/backup/${2}_${3}" } func_repo_repodata_rebuild() { @@ -117,16 +109,16 @@ func_repo_upload() { # ${1} - local path # ${2} - bucket name # ${3} - COS path - coscli -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${2}/packages/${3}" || true - coscli -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "${1}" "cos://${2}/packages/${3}" + coscmd -b "${2}" delete -r -f "/packages/${3}" || true + coscmd -b "${2}" -r ${COS_GLOBAL_REGION} upload -r "${1}" "/packages/${3}" } func_repo_publish() { # ${1} - CI bucket # ${2} - repo publish bucket # ${3} - COS path - coscli -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${2}/packages/${3}" || true - coscli -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${3}" "cos://${2}/packages/${3}" + coscmd delete -r -f "/packages/${3}" || true + coscmd -b "${2}" copy -r "${1}.${VAR_COS_REGION_DNS}/packages/${3}" "packages/${3}" } # ======================================= @@ -135,8 +127,7 @@ func_repo_publish() { case_opt=$1 case ${case_opt} in init_cos_utils) - func_cos_utils_install "${VAR_TENCENT_COS_UTILS_VERSION}" - func_cos_utils_credential_init "${VAR_COS_ENDPOINT}" "${TENCENT_COS_SECRETID}" "${TENCENT_COS_SECRETKEY}" + func_cos_utils_credential_init "${TENCENT_COS_SECRETID}" "${TENCENT_COS_SECRETKEY}" "${VAR_COS_BUCKET_REPO}" ;; repo_init) # create basic repo directory structure