chore(ci): replace coscli with coscmd (#324)

api7 · Aug 11, 2023 · 2a1b948 · 2a1b948
1 parent 740ec67
commit 2a1b948
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 69 deletions.
diff --git a/.github/workflows/publish-deb.yml b/.github/workflows/publish-deb.yml
@@ -1,7 +1,7 @@
 name: Build and Publish Package
 
 on:
-  create
+  create:
 
 jobs:
   publish_apisix:
@@ -10,6 +10,8 @@ jobs:
     timeout-minutes: 60
     env:
       VAR_DEB_WORKBENCH_DIR: /tmp/output
+      VAR_COS_BUCKET_REPO: ${{ secrets.VAR_COS_BUCKET_REPO }}
+      VAR_COS_BUCKET_CI: ${{ secrets.VAR_COS_BUCKET_CI }}
       VAR_OS: debian
       VAR_CODENAME: bullseye
       VAR_OS_RELEASE: bullseye-slim
@@ -73,3 +75,40 @@ jobs:
           name: "apisix-base_${{ steps.tag_env.outputs.version}}-0~${{ env.VAR_OS }}${{ env.VAR_OS_RELEASE }}_amd64.deb"
           path: "${{ env.VAR_DEB_WORKBENCH_DIR }}/apisix-base_${{ steps.tag_env.outputs.version}}-0~${{ env.VAR_OS }}${{ env.VAR_OS_RELEASE }}_amd64.deb"
 
+      - name: DEB repo deps init
+        env:
+          TENCENT_COS_SECRETID: ${{ secrets.TENCENT_COS_SECRETID }}
+          TENCENT_COS_SECRETKEY: ${{ secrets.TENCENT_COS_SECRETKEY }}
+        run: |
+          sudo pip install coscmd
+          sudo -E ./utils/publish-deb.sh init_cos_utils
+
+      - name: DEB repo init for debian
+        run: |
+          sudo -E ./utils/publish-deb.sh repo_clone
+
+      - name: DEB repo package update
+        env:
+          DEB_GPG_MAIL: ${{ secrets.DEB_GPG_MAIL }}
+          GPG_NAME: ${{ secrets.GPG_NAME }}
+          GPG_MAIL: ${{ secrets.GPG_MAIL }}
+        run: |
+          echo "${{ secrets.DEB_GPG_PRIV_KEY }}" >> /tmp/deb-gpg-publish.private
+          echo "${{ secrets.DEB_GPG_PASSPHRASE }}" >> /tmp/deb-gpg-publish.passphrase
+          sudo -E ./utils/publish-deb.sh init_freight_utils
+          sudo -E ./utils/publish-deb.sh init_gpg
+
+      - name: DEB repo backup for debian
+        run: |
+          sudo -E ./utils/publish-deb.sh dists_backup
+
+      - name: DEB repo refresh for debian
+        run: |
+          sudo -E ./utils/publish-deb.sh repo_rebuild
+          sudo -E ./utils/publish-deb.sh repo_ci_upload
+
+      - name: DEB repo publish for debian
+        run: |
+          sudo -E ./utils/publish-deb.sh repo_upload
+          sudo -E ./utils/publish-rpm.sh repo_publish
+
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -130,6 +130,7 @@ jobs:
           TENCENT_COS_SECRETID: ${{ secrets.TENCENT_COS_SECRETID }}
           TENCENT_COS_SECRETKEY: ${{ secrets.TENCENT_COS_SECRETKEY }}
         run: |
+          sudo pip install coscmd
           sudo -E ./utils/publish-rpm.sh init_cos_utils
 
       - name: RPM repo init for centos and redhat

diff --git a/utils/publish-deb.sh b/utils/publish-deb.sh
@@ -10,12 +10,16 @@ env
 # Runtime default config
 # =======================================
 VAR_FREIGHT_UTILS_VERSION=${VAR_FREIGHT_UTILS_VERSION:-v0.3.13}
-VAR_TENCENT_COS_UTILS_VERSION=${VAR_TENCENT_COS_UTILS_VERSION:-v0.11.0-beta}
 VAR_DEB_WORKBENCH_DIR=${VAR_DEB_WORKBENCH_DIR:-/tmp/output}
 VAR_GPG_PRIV_KET=${VAR_GPG_PRIV_KET:-/tmp/deb-gpg-publish.private}
 VAR_GPG_PASSPHRASE=${VAR_GPG_PASSPHRASE:-/tmp/deb-gpg-publish.passphrase}
 
-COS_CMD=coscli
+COS_REGION=${COS_REGION:-"ap-guangzhou"}
+COS_GLOBAL_REGION=${COS_GLOBAL_REGION:-"accelerate"}
+COS_PART_SIZE=${COS_PART_SIZE:-"10"}
+VAR_COS_REGION_DNS="cos.${COS_REGION}.myqcloud.com"
+VAR_COS_GLOBAL_REGION_DNS="cos.${COS_GLOBAL_REGION}.myqcloud.com"
+
 TAG_DATE=$(date +%Y%m%d)
 ARCH=${ARCH:-`(uname -m | tr '[:upper:]' '[:lower:]')`}
 arch_path=""
@@ -41,30 +45,11 @@ _EOC_
 # =======================================
 # COS extension
 # =======================================
-func_cos_utils_install() {
-    if [[ $ARCH == "arm64" ]] || [[ $ARCH == "aarch64" ]]; then
-        wget https://github.com/tencentyun/coscli/archive/refs/tags/${VAR_TENCENT_COS_UTILS_VERSION}.tar.gz
-        tar -zxvf ${VAR_TENCENT_COS_UTILS_VERSION}.tar.gz
-        cd coscli-* && go build
-        mv coscli ../
-    else
-        sudo curl -o /usr/bin/coscli -L "https://github.com/tencentyun/coscli/releases/download/${VAR_TENCENT_COS_UTILS_VERSION}/coscli-linux"
-        sudo chmod 755 /usr/bin/coscli
-    fi
-}
-
 func_cos_utils_credential_init() {
-    # ${1} - COS endpoint
-    # ${2} - COS SECRET_ID
-    # ${3} - COS SECRET_KEY
-    cat > "${HOME}/.cos.yaml" <<_EOC_
-cos:
-  base:
-    secretid: ${2}
-    secretkey: ${3}
-    sessiontoken: ""
-    protocol: https
-_EOC_
+    # ${1} - COS SECRET_ID
+    # ${2} - COS SECRET_KEY
+    # ${3} - COS bucket name
+    coscmd config -a "${1}" -s "${2}" -b "${3}" -r ${COS_REGION} -p ${COS_PART_SIZE}
 }
 
 
@@ -91,21 +76,20 @@ func_dists_backup() {
     # ${1} - bucket name
     # ${2} - COS path
     # ${3} - backup tag
-    $COS_CMD -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${arch_path}${2}/dists" "cos://${1}/packages/${arch_path}backup/${2}_dists_${3}" || true
+    coscmd copy -r "${1}.${VAR_COS_REGION_DNS}/packages/${arch_path}${2}/dists" "/packages/${arch_path}backup/${2}_dists_${3}"
 }
 
 func_pool_clone() {
     # ${1} - bucket name
     # ${2} - COS path
     # ${3} - local pool path
-
     mkdir -p ${3}
     # --part-size indicates the file chunk size.
     # when the file is larger than --part-size, coscli will chunk the file by --part-size.
     # when uploading/downloading the file in chunks, it will enable breakpoint transfer by default,
     # which will generate cosresumabletask file and interfere with the file integrity.
     # ref: https://cloud.tencent.com/document/product/436/63669
-    $COS_CMD -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${arch_path}${2}/pool" "${3}"
+    coscmd -b "${1}" -r "${COS_GLOBAL_REGION}" download -r "/packages/${arch_path}${2}/pool" "${3}"
 }
 
 func_dists_rebuild() {
@@ -139,8 +123,8 @@ func_dists_rebuild() {
 }
 
 func_dists_upload_ci_repo() {
-    $COS_CMD -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${2}/packages/${arch_path}${3}" || true
-    $COS_CMD -e "${VAR_COS_ENDPOINT}" cp -r --part-size 512 "${1}" "cos://${2}/packages/${arch_path}${3}/dists/"
+    coscmd -b "${2}" delete -r -f "/packages/${arch_path}${3}" || true
+    coscmd -b "${2}" -r "${COS_GLOBAL_REGION}" upload -r "${1}" "/packages/${arch_path}${3}/dists"
 }
 
 func_deb_upload() {
@@ -152,33 +136,34 @@ func_deb_upload() {
     # We will only upload apisix and apisix-base,
     # so the directory is fixed: pool/main/a.
     # Regardless of other packages.
-
-    export COS_CMD=$COS_CMD
     export arch_path=$arch_path
     export BUCKET=$2
     export OS=$3
     export CODENAME=$4
     find "${1}" -type f -name "apisix_*.deb" \
         -exec echo "upload : {}" \; \
         -exec sh -c 'file=$(basename {}); \
-                    $COS_CMD -e "${VAR_COS_ENDPOINT}" cp {} --part-size 512 "cos://${BUCKET}/packages/${arch_path}${OS}/pool/${CODENAME}/main/a/apisix/${file}"' \;
+                    coscmd -b "${BUCKET}" -r "${COS_GLOBAL_REGION}" upload {} "/packages/${arch_path}${OS}/pool/${CODENAME}/main/a/apisix/${file}"' \;
 
     find "${1}" -type f -name "apisix-base*.deb" \
         -exec echo "upload : {}" \; \
         -exec sh -c 'file=$(basename {}); \
-                    $COS_CMD -e "${VAR_COS_ENDPOINT}" cp {} --part-size 512 "cos://${BUCKET}/packages/${arch_path}${OS}/pool/${CODENAME}/main/a/apisix-base/${file}"' \;
+                    coscmd -b "${BUCKET}" -r "${COS_GLOBAL_REGION}" upload {} "/packages/${arch_path}${OS}/pool/${CODENAME}/main/a/apisix-base/${file}"' \;
 }
 
 func_repo_publish() {
-    $COS_CMD -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${2}/packages/${arch_path}${3}/dists" || true
-    $COS_CMD -e "${VAR_COS_ENDPOINT}" cp -r --part-size 512 "cos://${1}/packages/${arch_path}${3}/dists" "cos://${2}/packages/${arch_path}${3}/dists"
+    # ${1} - CI bucket
+    # ${2} - repo publish bucket
+    # ${3} - COS path
+    coscmd delete -r -f "/packages/${arch_path}${3}/dists" || true
+    coscmd -b "${2}" copy -r "${1}.${VAR_COS_REGION_DNS}/packages/${arch_path}${3}/dists" "/packages/${arch_path}${3}/dists"
 }
 
 func_repo_backup_remove() {
     # ${1} - bucket name
     # ${2} - COS path
     # ${3} - backup tag
-    $COS_CMD -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${1}/packages/${arch_path}backup/${2}_dists_${3}" || true
+    coscmd -b "${1}" delete -r -f "/packages/${arch_path}backup/${2}_dists_${3}" || true
 }
 
 # =======================================
@@ -188,8 +173,7 @@ case_opt=$1
 
 case ${case_opt} in
 init_cos_utils)
-    func_cos_utils_install
-    func_cos_utils_credential_init "${VAR_COS_ENDPOINT}" "${TENCENT_COS_SECRETID}" "${TENCENT_COS_SECRETKEY}"
+    func_cos_utils_credential_init "${TENCENT_COS_SECRETID}" "${TENCENT_COS_SECRETKEY}" "${VAR_COS_BUCKET_REPO}"
     ;;
 init_freight_utils)
     func_freight_utils_install
@@ -210,8 +194,10 @@ repo_clone)
 repo_rebuild)
     func_dists_rebuild "/tmp/old_pool" "/tmp/freight" ${VAR_DEB_WORKBENCH_DIR} ${VAR_CODENAME}
     ;;
-repo_upload)
+repo_ci_upload)
     func_dists_upload_ci_repo "/tmp/freight/cache/dists" "${VAR_COS_BUCKET_CI}" "${VAR_OS}"
+    ;;
+repo_upload)
     func_deb_upload "${VAR_DEB_WORKBENCH_DIR}" "${VAR_COS_BUCKET_REPO}" "${VAR_OS}" "${VAR_CODENAME}"
     ;;
 repo_publish)

diff --git a/utils/publish-rpm.sh b/utils/publish-rpm.sh
@@ -9,12 +9,17 @@ env
 # =======================================
 # Runtime default config
 # =======================================
-VAR_TENCENT_COS_UTILS_VERSION=${VAR_TENCENT_COS_UTILS_VERSION:-v0.11.0-beta}
 VAR_RPM_WORKBENCH_DIR=${VAR_RPM_WORKBENCH_DIR:-/tmp/output}
 VAR_GPG_PRIV_KET=${VAR_GPG_PRIV_KET:-/tmp/rpm-gpg-publish.private}
 VAR_GPG_PASSPHRASE=${VAR_GPG_PASSPHRASE:-/tmp/rpm-gpg-publish.passphrase}
 ARCH=${ARCH:-`(uname -m | tr '[:upper:]' '[:lower:]')`}
 
+COS_REGION=${COS_REGION:-"ap-guangzhou"}
+COS_GLOBAL_REGION=${COS_GLOBAL_REGION:-"accelerate"}
+COS_PART_SIZE=${COS_PART_SIZE:-"10"}
+VAR_COS_REGION_DNS="cos.${COS_REGION}.myqcloud.com"
+VAR_COS_GLOBAL_REGION_DNS="cos.${COS_GLOBAL_REGION}.myqcloud.com"
+
 # =======================================
 # GPG extension
 # =======================================
@@ -42,24 +47,11 @@ func_gpg_key_load() {
 # =======================================
 # COS extension
 # =======================================
-func_cos_utils_install() {
-    # ${1} - COS util version
-    curl -o /usr/bin/coscli -L "https://github.com/tencentyun/coscli/releases/download/${1}/coscli-linux"
-    chmod 755 /usr/bin/coscli
-}
-
 func_cos_utils_credential_init() {
-    # ${1} - COS endpoint
-    # ${2} - COS SECRET_ID
-    # ${3} - COS SECRET_KEY
-    cat > "${HOME}/.cos.yaml" <<_EOC_
-cos:
-  base:
-    secretid: ${2}
-    secretkey: ${3}
-    sessiontoken: ""
-    protocol: https
-_EOC_
+    # ${1} - COS SECRET_ID
+    # ${2} - COS SECRET_KEY
+    # ${3} - COS bucket name
+    coscmd config -a "${1}" -s "${2}" -b "${3}" -r ${COS_REGION} -p ${COS_PART_SIZE}
 }
 
 # =======================================
@@ -77,25 +69,25 @@ func_repo_clone() {
     # ${3} - target path
 
     # --part-size indicates the file chunk size.
-    # when the file is larger than --part-size, coscli will chunk the file by --part-size.
+    # when the file is larger than --part-size, coscmd will chunk the file by --part-size.
     # when uploading/downloading the file in chunks, it will enable breakpoint transfer by default,
     # which will generate cosresumabletask file and interfere with the file integrity.
     # ref: https://cloud.tencent.com/document/product/436/63669
-    coscli -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${2}" "${3}"
+    coscmd -b "${1}"  -r "${COS_GLOBAL_REGION}" download -r "/packages/${2}" "${3}"
 }
 
 func_repo_backup() {
     # ${1} - bucket name
     # ${2} - COS path
     # ${3} - backup tag
-    coscli -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${2}" "cos://${1}/packages/backup/${2}_${3}"
+    coscmd copy -r "${1}.${VAR_COS_REGION_DNS}/packages/${2}" "/packages/backup/${2}_${3}"
 }
 
 func_repo_backup_remove() {
     # ${1} - bucket name
     # ${2} - COS path
     # ${3} - backup tag
-    coscli -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${1}/packages/backup/${2}_${3}"
+    coscmd -b "${1}" delete -r -f "/packages/backup/${2}_${3}"
 }
 
 func_repo_repodata_rebuild() {
@@ -117,16 +109,16 @@ func_repo_upload() {
     # ${1} - local path
     # ${2} - bucket name
     # ${3} - COS path
-    coscli -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${2}/packages/${3}" || true
-    coscli -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "${1}" "cos://${2}/packages/${3}"
+    coscmd -b "${2}"  delete -r -f "/packages/${3}" || true
+    coscmd -b "${2}" -r ${COS_GLOBAL_REGION} upload -r "${1}" "/packages/${3}"
 }
 
 func_repo_publish() {
     # ${1} - CI bucket
     # ${2} - repo publish bucket
     # ${3} - COS path
-    coscli -e "${VAR_COS_ENDPOINT}" rm -r -f "cos://${2}/packages/${3}" || true
-    coscli -e "${VAR_COS_ENDPOINT}" cp -r --part-size 1000 "cos://${1}/packages/${3}" "cos://${2}/packages/${3}"
+    coscmd delete -r -f "/packages/${3}" || true
+    coscmd -b "${2}" copy -r "${1}.${VAR_COS_REGION_DNS}/packages/${3}" "packages/${3}"
 }
 
 # =======================================
@@ -135,8 +127,7 @@ func_repo_publish() {
 case_opt=$1
 case ${case_opt} in
 init_cos_utils)
-    func_cos_utils_install "${VAR_TENCENT_COS_UTILS_VERSION}"
-    func_cos_utils_credential_init "${VAR_COS_ENDPOINT}" "${TENCENT_COS_SECRETID}" "${TENCENT_COS_SECRETKEY}"
+    func_cos_utils_credential_init "${TENCENT_COS_SECRETID}" "${TENCENT_COS_SECRETKEY}" "${VAR_COS_BUCKET_REPO}"
     ;;
 repo_init)
     # create basic repo directory structure