Automate release process

.github/workflows/doc.yaml

@@ -45,14 +45,7 @@ jobs:
             libgirepository1.0-dev \
             ninja-build \
             valac
-          pip install \
-            Jinja2 \
-            Pygments \
-            Sphinx \
-            breathe \
-            gi-docgen \
-            meson \
-            pydata_sphinx_theme
+          pip install -r requirements.txt
           case ${GITHUB_REF} in
             refs/tags/*)
               version=${GITHUB_REF#refs/tags/}
@@ -96,17 +89,15 @@ jobs:
           path: html/
       - uses: actions/checkout@v3
         if: |
-          github.event_name == 'push' &&
-            (startsWith(github.ref, 'refs/tags/') ||
-             github.ref == 'refs/heads/main')
+          startsWith(github.ref, 'refs/tags/') ||
+            github.ref == 'refs/heads/main'
         with:
           ref: gh-pages
           path: gh-pages
       - name: Deploy
         if: |
-          github.event_name == 'push' &&
-            (startsWith(github.ref, 'refs/tags/') ||
-             github.ref == 'refs/heads/main')
+          startsWith(github.ref, 'refs/tags/') ||
+            github.ref == 'refs/heads/main'
         run: |
           rm -rf gh-pages/${VERSION}
           cp -a build/doc/html/ gh-pages/${VERSION}

.github/workflows/package.yaml

@@ -23,6 +23,47 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  source:
+    name: Source
+    runs-on: ubuntu-latest
+    steps:
+      - name: Prepare
+        run: |
+          case ${GITHUB_REF} in
+            refs/tags/*)
+              version=${GITHUB_REF#refs/tags/}
+              ;;
+            *)
+              version=${GITHUB_SHA}
+              ;;
+          esac
+          echo "VERSION=${version}" >> ${GITHUB_ENV}
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          path: datafusion-c-${{ env.VERSION }}
+      - name: Archive
+        run: |
+          rm -rf datafusion-c-${VERSION}/.git
+          tar cvzf datafusion-c-${VERSION}.tar.gz datafusion-c-${VERSION}
+          zip -r datafusion-c-${VERSION}.zip datafusion-c-${VERSION}
+      - uses: actions/upload-artifact@v3
+        with:
+          name: tar.gz
+          path: datafusion-c-${{ env.VERSION }}.tar.gz
+      - uses: actions/upload-artifact@v3
+        with:
+          name: zip
+          path: datafusion-c-${{ env.VERSION }}.zip
+      - uses: softprops/action-gh-release@v1
+        if: |
+          startsWith(github.ref, 'refs/tags/')
+        with:
+          body_path: datafusion-c-${{ env.VERSION }}/doc/source/news/${{ env.VERSION }}.md
+          files: |
+            datafusion-c-${{ env.VERSION }}.tar.gz
+            datafusion-c-${{ env.VERSION }}.zip
+
   build:
     name: Build
     runs-on: ubuntu-22.04
@@ -84,31 +125,29 @@ jobs:
           #   target: debian-bookworm-arm64
           #   test-docker-image: arm64v8/debian:bookworm
           - label: Ubuntu Focal amd64
-            id: debian-focal-amd64
+            id: ubuntu-focal-amd64
             task-namespace: apt
             target: ubuntu-focal
             test-docker-image: ubuntu:focal
           # - label: Ubuntu Focal arm64
-          #   id: debian-focal-arm64
+          #   id: ubuntu-focal-arm64
           #   task-namespace: apt
           #   target: ubuntu-focal-arm64
           #   test-docker-image: arm64v8/ubuntu:focal
           - label: Ubuntu Jammy amd64
-            id: debian-jammy-amd64
+            id: ubuntu-jammy-amd64
             task-namespace: apt
             target: ubuntu-jammy
             test-docker-image: ubuntu:jammy
           # - label: Ubuntu Jammy arm64
-          #   id: debian-jammy-arm64
+          #   id: ubuntu-jammy-arm64
           #   task-namespace: apt
           #   target: ubuntu-jammy-arm64
           #   test-docker-image: arm64v8/ubuntu:jammy
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: true
-      - name: Checkout Arrow
-        uses: actions/checkout@v3
       - name: Install dependencies
         run: |
           sudo apt update
@@ -118,8 +157,7 @@ jobs:
             ruby
       - name: Update version
         if: |
-          !startsWith(github.ref, 'refs/tags/') &&
-          !startsWith(github.ref, 'refs/heads/maintenance/')
+          !startsWith(github.ref, 'refs/tags/')
         run: |
           cd package
           rake version:update RELEASE_DATE=$(date +%Y-%m-%d)
@@ -145,13 +183,16 @@ jobs:
           startsWith(github.ref, 'refs/tags/')
         run: |
           mkdir -p ${{ matrix.id }}
+          shopt -s globstar
           cp -a package/${{ matrix.task-namespace }}/repositories/**/*.* ${{ matrix.id }}/
           tar czf ${{ matrix.id }}.tar.gz ${{ matrix.id }}
+          echo "VERSION=${GITHUB_REF#refs/tags/}" >> ${GITHUB_ENV}
       - name: Upload to release
         uses: softprops/action-gh-release@v1
         if: |
           startsWith(github.ref, 'refs/tags/')
         with:
+          body_path: doc/source/news/${{ env.VERSION }}.md
           files: |
             ${{ matrix.id }}.tar.gz
       - name: Test
@@ -160,4 +201,5 @@ jobs:
             --rm \
             --volume ${PWD}:/host:ro \
             ${{ matrix.test-docker-image }} \
-            /host/package/${{ matrix.task-namespace }}/test.sh
+            /host/package/${{ matrix.task-namespace }}/test.sh \
+            local

.prettierignore

@@ -0,0 +1,15 @@
+# Copyright 2022 Sutou Kouhei <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+/vendor/

dev/release/prepare.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# Copyright 2022 Sutou Kouhei <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+if [ $# -lt 1 ]; then
+  echo "Usage: $0 version"
+  echo " e.g.: $0 11.0.0"
+  exit 1
+fi
+
+version=$1
+major_version=$(echo ${version} | grep -o -E '^[0-9]+')
+
+source_dir="$(dirname "$0")/../.."
+
+git fetch origin --tags
+
+branch="prepare-${version}"
+git branch -D ${branch} || :
+git checkout -b ${branch} origin/main
+
+pushd "${source_dir}"
+sed -i'.bak' \
+  -e "s/^version = \".*\"/version = \"${version}\"/" \
+  Cargo.toml
+rm Cargo.toml.bak
+git add Cargo.toml
+
+sed -i'.bak' \
+  -e "s/^version = \'.*\'/version = \"${version}\"/" \
+  meson.build
+rm meson.build.bak
+git add meson.build
+popd
+
+pushd "${source_dir}/package/debian/"
+sed -i'.bak' -E \
+  -e "s/libdatafusion[0-9]+/libdatafusion${major_version}/g" \
+  -e "s/libdatafusion-glib[0-9]+/libdatafusion-glib${major_version}/g" \
+  -e "s/gir1\.2-datafusion-[0-9]+/gir1.2-datafusion-${major_version}/g" \
+  control
+rm control.bak
+git add control
+git mv gir1.2-datafusion-*.install \
+       gir1.2-datafusion-${major_version}.0.install
+git mv libdatafusion-glib[[:alnum:]]*.install \
+       libdatafusion-glib${major_version}.install
+git mv libdatafusion[[:alnum:]]*.install \
+       libdatafusion${major_version}.install
+popd
+
+pushd "${source_dir}/doc/"
+latest_news_md=$(ls source/news/*.md | sort -n -r | head -n1)
+sed \
+  -e "s/^# .*\$/# ${version} - $(date +%Y-%m-%d)/" \
+  ${latest_news_md} > source/news/${version}.md
+git add source/news/${version}.md
+sed -i'.bak' \
+  -e "N; /^\.\. toctree::/ a \ \ \ news/${version}" \
+  source/news.rst
+rm source/news.rst.bak
+git add source/news.rst
+sed -i'.bak' \
+  -e "\,^  'source' / 'news\.rst', i \ \ 'source' / 'news' / '${version}.md'," \
+  meson.build
+rm meson.build.bak
+git add meson.build
+popd
+
+pushd "${source_dir}/package"
+rake version:update
+git add .
+popd

dev/release/tag.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2022 Sutou Kouhei <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+source_dir="$(dirname "$0")/../.."
+
+version=$(grep "^version" "${source_dir}/Cargo.toml" | \
+            head -n1 | \
+            grep -o "[0-9.]*")
+git tag -a -m "Release ${version}!!!" ${version}
+git push origin ${version}

dev/release/verify-packages.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Copyright 2022 Sutou Kouhei <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+if [ $# -lt 1 ]; then
+  echo "Usage: $0 rc"
+  echo "       $0 staging-rc"
+  echo "       $0 release"
+  echo "       $0 staging-release"
+  echo " e.g.: $0 rc              # Verify RC packages"
+  echo " e.g.: $0 staging-rc      # Verify RC packages on staging"
+  echo " e.g.: $0 release         # Verify release packages"
+  echo " e.g.: $0 staging-release # Verify release packages on staging"
+  exit 1
+fi
+
+verify_type="$1"
+
+source_dir="$(dirname "$0")/../.."
+
+pushd "${source_dir}"
+for dir in package/{apt,yum}/*; do
+  if [ ! -d "${dir}" ]; then
+    continue
+  fi
+  base_name=${dir##*/}
+  distribution=
+  code_name=
+  case ${base_name} in
+    *-aarch64|*-arm64)
+      ;;
+    almalinux-*|debian-*|ubuntu-*)
+      distribution=${base_name%-*}
+      code_name=${base_name#*-}
+      ;;
+  esac
+  if [ "${distribution}" = "" -o "#{code_name}" = "" ]; then
+    continue
+  fi
+  package_type=$(basename "$(dirname "${dir}")")
+  docker run \
+         --rm \
+         --volume "$PWD:/host" \
+         -it \
+         ${distribution}:${code_name} \
+         /host/package/${package_type}/test.sh \
+         "${verify_type}"
+done
+popd

doc/meson.build

@@ -23,6 +23,8 @@ depend_files = files(
   'source' / 'index.rst',
   'source' / 'install.rst',
   'source' / 'introduction.rst',
+  'source' / 'news' / '10.0.0.md',
+  'source' / 'news.rst',
   'source' / 'raw-c-api.rst',
 )
 depend_files += [gi_docgen_toml]

doc/source/conf.py

@@ -26,7 +26,12 @@
 
 extensions = [
     'breathe',
+    'myst_parser',
 ]
+source_suffix = {
+    '.rst': 'restructuredtext',
+    '.md': 'markdown',
+}
 
 breathe_default_project = 'datafusion-c'
 breathe_domain_by_extension = {'h' : 'c'}

doc/source/developer.rst

@@ -0,0 +1,19 @@
+.. Copyright 2022 Sutou Kouhei <kou@clear-code.com>
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+
+Developer
+=========
+
+.. toctree::
+
+   developer/release