From 09c0dea59ab783da34455f65cc2c580885a8b62b Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Fri, 7 Feb 2025 10:11:47 +0000 Subject: [PATCH 01/29] Make push optional for set-version action --- .github/workflows/set_version.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/set_version.yml b/.github/workflows/set_version.yml index 2e734e9b5..3ad6b60b0 100644 --- a/.github/workflows/set_version.yml +++ b/.github/workflows/set_version.yml @@ -11,6 +11,11 @@ on: description: "New version number" required: true type: string + push: + description: "Push commit to branch" + required: true + type: boolean + default: false jobs: set-version: @@ -79,6 +84,7 @@ jobs: git diff HEAD^ - name: "Push commit to branch" + if: ${{ inputs.push }} shell: bash -l {0} run: | # Verify that ref is a branch (as opposed to a tag or hash) From b41ae28f5670d75c82a468b7d471bb15be4dd550 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Fri, 7 Feb 2025 10:15:04 +0000 Subject: [PATCH 02/29] Enable workflow_call for set-version action --- .github/workflows/set_version.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/set_version.yml b/.github/workflows/set_version.yml index 3ad6b60b0..b02466eeb 100644 --- a/.github/workflows/set_version.yml +++ b/.github/workflows/set_version.yml @@ -17,6 +17,22 @@ on: type: boolean default: false + workflow_call: + inputs: + ref: + description: "Target branch" + required: true + type: string + version: + description: "New version number" + required: true + type: string + push: + description: "Push commit to branch" + type: boolean + default: false + required: false + jobs: set-version: runs-on: ubuntu-latest From 2d1084a4bf3e7d18d998d2359f846fd798d7f197 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Fri, 7 Feb 2025 10:29:13 +0000 Subject: [PATCH 03/29] Allow workflow call to skip checkout --- .github/workflows/set_version.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/set_version.yml b/.github/workflows/set_version.yml index b02466eeb..06bf68f6b 100644 --- a/.github/workflows/set_version.yml +++ b/.github/workflows/set_version.yml @@ -21,8 +21,9 @@ on: inputs: ref: description: "Target branch" - required: true + required: false type: string + default: "" version: description: "New version number" required: true @@ -53,7 +54,10 @@ jobs: print(f"VERSION_NUMBER={version_number}", file=github_env) print(f"VERSION_STRING={version_string}", file=github_env) + # From a workflow call we might already have working copy; in that + # case inputs.ref should be an empty string. Otherwise, checkout. - uses: actions/checkout@v4 + if: ${{ inputs.ref }} with: ref: ${{ inputs.ref }} fetch-tags: true From 87afa6936977f43ad1d04fcd395dc129f9157a09 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Fri, 7 Feb 2025 15:28:35 +0000 Subject: [PATCH 04/29] Implement CHANGELOG parser; no bumping yet --- build_utils/bump_changelog.py | 82 +++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100755 build_utils/bump_changelog.py diff --git a/build_utils/bump_changelog.py b/build_utils/bump_changelog.py new file mode 100755 index 000000000..ad9ed5c19 --- /dev/null +++ b/build_utils/bump_changelog.py @@ -0,0 +1,82 @@ +#! /usr/bin/env python + +"""Bump the current version number in CHANGELOG.rst + +Unreleased changes will be moved into a section for the new version number +""" +from argparse import ArgumentParser +from dataclasses import dataclass +from itertools import batched +from pathlib import Path +import re +from typing import NamedTuple + +REPOSITORY_ADDRESS = "https://github.com/pace-neutrons/Euphonic" + + +@dataclass +class Block: + tag: str + previous_tag: str + content: str + + +def parse_changelog(changelog_file: Path) -> list[Block]: + """Read all sections from changelog file + + Each section should begin with two lines in format + + `TAG `_ + -------------------------------------------------- + + except for 'Unreleased' which compares with HEAD + """ + + with open(changelog_file) as fd: + split_text = re.split(r"`(\S+) <\S+/compare/(\S+)\.\.\.\S+>`_\n-+", + fd.read()) + + # First item is always empty? + split_text = split_text[1:] + + blocks = [Block(*block_data) + for block_data in batched(split_text, n=3)] + + for block in blocks: + block.content = block.content.strip() + + return blocks + + +def tag_to_header(tag: str, previous_tag: str) -> str: + compare_tag = "HEAD" if tag == "Unreleased" else tag + header = f"`{tag} <{REPOSITORY_ADDRESS}/compare/{previous_tag}...{compare_tag}>`_" + return header + "\n" + len(header) * "-" + + +def to_text(blocks: list[Block]) -> str: + return "\n\n".join( + tag_to_header(block.tag, block.previous_tag) + "\n\n" + block.content + for block in blocks + ) + + +def get_parser() -> ArgumentParser: + parser = ArgumentParser() + parser.add_argument("filename", type=Path, help="Input CHANGELOG.rst file") + parser.add_argument("tag", type=str, help="Tag for new/updated version") + parser.add_argument("--replace", action="store_true", help="Write to existing file") + + return parser + +if __name__ == "__main__": + args = get_parser().parse_args() + + blocks = parse_changelog(args.filename) + + if args.replace: + with open(args.filename, "w") as fd: + print(to_text(blocks), file=fd) + + else: + print(to_text(blocks)) From e4ed5832759f793929dbe3fbd1864db1dd40e7da Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Fri, 7 Feb 2025 16:24:41 +0000 Subject: [PATCH 05/29] Implement CHANGELOG bumping script; minor re-render of CHANGELOG Now this script can be used in the following ways: - without "--replace" argument the new CHANGELOG is printed to stdout - If there is Unreleased content and the requested tag already exists right after Unreleased, we assume that a failed release is being fixed. Content from Unreleased is moved into that section and appended to what is already there. - If the tags match but there are no Unreleased changelog notes, nothing much should change. (Maybe some line breaks.) This might be the case if we are on a release branch and already manually updated the CHANGELOG. - Otherwise, we create a new release section under Unreleased, move content from Unreleased into that section, and update the "compare" links. While working on the parser I ran into a broken link and some inconsistent spacing in the existing CHANGELOG.rst file. --- CHANGELOG.rst | 6 ++---- build_utils/bump_changelog.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9c4019d72..f863d5c04 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -186,7 +186,6 @@ as well as new features and performance enhancements. - Filter out spglib deprecation warnings caused by SeeK-path. - `v1.3.1 `_ ----------------------------------------------------------------------------- @@ -246,7 +245,6 @@ as well as new features and performance enhancements. accessible at this Q value. This gives much more intuitive behaviour, especially for narrow angle ranges. - `v1.2.1 `_ ----------------------------------------------------------------------------- @@ -500,8 +498,8 @@ as well as new features and performance enhancements. S(Q,w) will be scaled by 2.72e4/0.1 = 2.72e5. The original structure factors can now be correctly recovered by multiplying S(Q,w) by the energy bin width. -`v0.6.0 `_ ----------------------------------------------------------------------------- +`v0.6.0 `_ +----------------------------------------------------------------------------- - Euphonic can now calculate neutron-weighted partial density of states, and has new ``Spectra`` features to handle PDOS data: diff --git a/build_utils/bump_changelog.py b/build_utils/bump_changelog.py index ad9ed5c19..f3def70cd 100755 --- a/build_utils/bump_changelog.py +++ b/build_utils/bump_changelog.py @@ -8,6 +8,7 @@ from dataclasses import dataclass from itertools import batched from pathlib import Path +from packaging.version import Version import re from typing import NamedTuple @@ -33,14 +34,12 @@ def parse_changelog(changelog_file: Path) -> list[Block]: """ with open(changelog_file) as fd: - split_text = re.split(r"`(\S+) <\S+/compare/(\S+)\.\.\.\S+>`_\n-+", - fd.read()) + split_text = re.split(r"`(\S+) <\S+/compare/(\S+)\.\.\.\S+>`_\n-+", fd.read()) # First item is always empty? split_text = split_text[1:] - blocks = [Block(*block_data) - for block_data in batched(split_text, n=3)] + blocks = [Block(*block_data) for block_data in batched(split_text, n=3)] for block in blocks: block.content = block.content.strip() @@ -48,6 +47,25 @@ def parse_changelog(changelog_file: Path) -> list[Block]: return blocks +def bump_version(blocks: list[Block], tag: str) -> None: + """Create or update a new or existing block with content from "Unreleased" """ + + all_tags = [block.tag for block in blocks] + + if all_tags[0] != "Unreleased": + raise ValueError("CHANGELOG should always begin with Unreleased") + + if all_tags[1] != tag: + # This version is not in the CHANGELOG yet + previous_tag = all_tags[1] + blocks.insert(1, Block(tag, previous_tag, "")) + + blocks[1].content = "\n\n".join( + [txt for txt in (blocks[0].content, blocks[1].content) if txt] + ) + blocks[0].content = "" + + def tag_to_header(tag: str, previous_tag: str) -> str: compare_tag = "HEAD" if tag == "Unreleased" else tag header = f"`{tag} <{REPOSITORY_ADDRESS}/compare/{previous_tag}...{compare_tag}>`_" @@ -56,7 +74,8 @@ def tag_to_header(tag: str, previous_tag: str) -> str: def to_text(blocks: list[Block]) -> str: return "\n\n".join( - tag_to_header(block.tag, block.previous_tag) + "\n\n" + block.content + tag_to_header(block.tag, block.previous_tag) + + (f"\n\n{block.content}" if block.content else "") for block in blocks ) @@ -69,10 +88,12 @@ def get_parser() -> ArgumentParser: return parser + if __name__ == "__main__": args = get_parser().parse_args() blocks = parse_changelog(args.filename) + bump_version(blocks, args.tag) if args.replace: with open(args.filename, "w") as fd: From 157288e4f84617c8ca5d0680d48ef0d52ae8cb51 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Fri, 7 Feb 2025 18:25:44 +0000 Subject: [PATCH 06/29] Tidy up bump_changelog.py --- build_utils/bump_changelog.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build_utils/bump_changelog.py b/build_utils/bump_changelog.py index f3def70cd..4984eea2a 100755 --- a/build_utils/bump_changelog.py +++ b/build_utils/bump_changelog.py @@ -8,15 +8,15 @@ from dataclasses import dataclass from itertools import batched from pathlib import Path -from packaging.version import Version import re -from typing import NamedTuple + REPOSITORY_ADDRESS = "https://github.com/pace-neutrons/Euphonic" @dataclass class Block: + """CHANGELOG data corresponding to a single release""" tag: str previous_tag: str content: str @@ -67,12 +67,14 @@ def bump_version(blocks: list[Block], tag: str) -> None: def tag_to_header(tag: str, previous_tag: str) -> str: + """Produce header including github diff link""" compare_tag = "HEAD" if tag == "Unreleased" else tag header = f"`{tag} <{REPOSITORY_ADDRESS}/compare/{previous_tag}...{compare_tag}>`_" return header + "\n" + len(header) * "-" def to_text(blocks: list[Block]) -> str: + """Dump blocks to single multiline string""" return "\n\n".join( tag_to_header(block.tag, block.previous_tag) + (f"\n\n{block.content}" if block.content else "") From c3d8d9dcb6e579f8d628631de47b0757b6a67cd3 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Fri, 7 Feb 2025 22:21:04 +0000 Subject: [PATCH 07/29] Bump CHANGELOG from set_version action This could be done in a _slightly_ more modular fashion but would add a bit of complexity as well to pass around the file changes across actions without making multiple commits in the history. The nice thing about this setup is that bump_changelog.py doesn't mind at all if you already sorted out the CHANGELOG by hand, so if there are any difficult/edge cases they can be handled by using a release branch and running the script locally before using the release action. --- .github/workflows/set_version.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/set_version.yml b/.github/workflows/set_version.yml index 06bf68f6b..24c5a1125 100644 --- a/.github/workflows/set_version.yml +++ b/.github/workflows/set_version.yml @@ -75,11 +75,15 @@ jobs: sed -i "s/^version:\ .*/version: ${VERSION_NUMBER}/" CITATION.cff sed -i "s/^date-released:\ .*/date-released: $(date -I)/" CITATION.cff + - name: "Update CHANGELOG.rst" + shell: bash -l {0} + run: python build_utils/bump_changelog.py --replace CHANGELOG.rst ${VERSION_NUMBER} + - name: "Create commit" run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add euphonic/version.py CITATION.cff + git add euphonic/version.py CITATION.cff CHANGELOG.rst if git diff --cached --quiet then From 17e8b21396d6eee8612edf0287d97d63419cf4e6 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Fri, 7 Feb 2025 22:49:17 +0000 Subject: [PATCH 08/29] More linting, replace itertools.batched with itertoolz.partition itertools.batched is from Python 3.12. Not a complete dealbreaker, but makes life simpler as developers are likely running the script from a Euphonic dev environment which is 3.10 with toolz available. --- .github/workflows/set_version.yml | 7 +++++++ build_utils/bump_changelog.py | 17 ++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/set_version.yml b/.github/workflows/set_version.yml index 24c5a1125..e44084201 100644 --- a/.github/workflows/set_version.yml +++ b/.github/workflows/set_version.yml @@ -39,6 +39,13 @@ jobs: runs-on: ubuntu-latest steps: + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: python -m pip install toolz packaging + - name: Validate and normalise version number shell: python run : | diff --git a/build_utils/bump_changelog.py b/build_utils/bump_changelog.py index 4984eea2a..1d05f225f 100755 --- a/build_utils/bump_changelog.py +++ b/build_utils/bump_changelog.py @@ -6,10 +6,10 @@ """ from argparse import ArgumentParser from dataclasses import dataclass -from itertools import batched from pathlib import Path import re +from toolz.itertoolz import partition REPOSITORY_ADDRESS = "https://github.com/pace-neutrons/Euphonic" @@ -33,13 +33,14 @@ def parse_changelog(changelog_file: Path) -> list[Block]: except for 'Unreleased' which compares with HEAD """ - with open(changelog_file) as fd: + with open(changelog_file, "rt", encoding="utf8") as fd: split_text = re.split(r"`(\S+) <\S+/compare/(\S+)\.\.\.\S+>`_\n-+", fd.read()) # First item is always empty? split_text = split_text[1:] - blocks = [Block(*block_data) for block_data in batched(split_text, n=3)] + # From Python 3.12 can replace partition with itertools.batched + blocks = [Block(*block_data) for block_data in partition(3, split_text)] for block in blocks: block.content = block.content.strip() @@ -83,6 +84,7 @@ def to_text(blocks: list[Block]) -> str: def get_parser() -> ArgumentParser: + """Use argparse to get user input""" parser = ArgumentParser() parser.add_argument("filename", type=Path, help="Input CHANGELOG.rst file") parser.add_argument("tag", type=str, help="Tag for new/updated version") @@ -91,15 +93,20 @@ def get_parser() -> ArgumentParser: return parser -if __name__ == "__main__": +def main() -> None: + """Entrypoint""" args = get_parser().parse_args() blocks = parse_changelog(args.filename) bump_version(blocks, args.tag) if args.replace: - with open(args.filename, "w") as fd: + with open(args.filename, "w", encoding="utf8") as fd: print(to_text(blocks), file=fd) else: print(to_text(blocks)) + + +if __name__ == "__main__": + main() From a0089fa40f3571ee6ceb7ebeaee830a8eb1ce55a Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Fri, 7 Feb 2025 23:02:51 +0000 Subject: [PATCH 09/29] Update "Unreleased" diff URL when updating CHANGELOG --- build_utils/bump_changelog.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build_utils/bump_changelog.py b/build_utils/bump_changelog.py index 1d05f225f..97aa479fb 100755 --- a/build_utils/bump_changelog.py +++ b/build_utils/bump_changelog.py @@ -64,7 +64,10 @@ def bump_version(blocks: list[Block], tag: str) -> None: blocks[1].content = "\n\n".join( [txt for txt in (blocks[0].content, blocks[1].content) if txt] ) + + # Update "Unreleased" section blocks[0].content = "" + blocks[0].previous_tag = tag def tag_to_header(tag: str, previous_tag: str) -> str: From 848ab79ef5ce194de97ddd6d5f5733140929c967 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Mon, 10 Feb 2025 10:31:55 +0000 Subject: [PATCH 10/29] Developing release action: split up build actions, no release yet Change of plan: we _do_ need to push the version/tag commit in order to have a sensible multi-job workflow. We do want that multi-job workflow because versioning tweaks should happen on one OS but building/testing needs to be matrixed. But this should generally be done on a release branch, so if the release fails it will be possible to clean up these commits/tags without messing up the master branch history. It seems a good idea to provide standalone build/test actions for debugging. By separating the sdist and wheel builds they can be called in parallel as part of overall release workflow. A couple of details still to work out at this point: - The OS matrix should be moved up to release.yml and passed as an option to build_wheels, so it is possible to troubleshoot one OS at a time. - We may need to store the normalised release tag to environmnent and use in subsequent steps instead of inputs.ref. That way, small formatting changes like adding "v" will be carried forward correctly. https://docs.github.com/en/actions/sharing-automations/reusing-workflows#using-outputs-from-a-reusable-workflow --- .github/workflows/build_sdist.yml | 96 ++++++++++++++++++++++++ .github/workflows/build_wheels.yml | 113 +++++++++++++++++++++++++++++ .github/workflows/release.yml | 77 +++++++++++++++++++- 3 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build_sdist.yml create mode 100644 .github/workflows/build_wheels.yml diff --git a/.github/workflows/build_sdist.yml b/.github/workflows/build_sdist.yml new file mode 100644 index 000000000..84df5a108 --- /dev/null +++ b/.github/workflows/build_sdist.yml @@ -0,0 +1,96 @@ +name: Build and test sdist + +on: + workflow_dispatch: + inputs: + ref: + description: "Target branch / tag / commit" + required: true + type: string + workflow_call: + inputs: + ref: + description: "Target branch / tag / commit" + required: true + type: string + +jobs: + build-sdist: + name: Build sdist + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + fetch-tags: true + + - name: Create source distribution + shell: bash -l {0} + run: | + pipx run build --sdist . + + - name: Upload source dist as build artifact + uses: actions/upload-artifact@v4 + with: + name: python-source-distribution + path: dist/ + if-no-files-found: error + + test-sdist: + needs: build-sdist + name: Test build from sdist on Windows + runs-on: windows-latest + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Find MSVC and set environment variables + shell: bash -l {0} + env: + MSVC_PREFIX: 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\MSVC' + run: | + echo "Available MSVC installations:" + ls "$MSVC_PREFIX" + + MSVC_BIN=$(ls "$MSVC_PREFIX" | tail -n 1)\\bin\\HostX64\\x64 + CC="$MSVC_PREFIX\\$MSVC_BIN\\cl.exe" + echo "CC: $CC" + echo "CC=$CC" >> $GITHUB_ENV + + CC_LD="$MSVC_PREFIX\\$MSVC_BIN\\link.exe" + echo "CC_LD: $CC_LD" + echo "CC_LD=$CC_LD" >> $GITHUB_ENV + + - name: Download sdist + uses: actions/download-artifact@v4 + with: + path: dist/ + merge-multiple: true + + - name: List downloaded sdist + run: | + ls -R dist/ + + - name: Update pip + shell: bash -l {0} + run: | + python -m pip install --upgrade pip + + - name: Install from sdist + shell: bash -l {0} + run: python -m pip install $(find dist -name 'euphonic-*.tar.gz')[matplotlib,phonopy-reader,brille,test] + + - name: Checkout repository (for tests and test data) + uses: actions/checkout@v4 + + - name: Delete source (to ensure we are testing INSTALLED version) + shell: bash -l {0} + run: rm -rf euphonic + + - name: run tests + shell: bash -l {0} + run: python tests_and_analysis/test/run_tests.py --report diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml new file mode 100644 index 000000000..37a1ae94b --- /dev/null +++ b/.github/workflows/build_wheels.yml @@ -0,0 +1,113 @@ +name: Build and test Python wheels + +on: + workflow_dispatch: + inputs: + ref: + description: "Target branch / tag / commit" + required: true + type: string + workflow_call: + inputs: + ref: + description: "Target branch / tag / commit" + required: true + type: string + +jobs: + build-wheels: + strategy: + matrix: + os: [windows-latest, macos-13, macos-latest, ubuntu-latest] + python-version: ['3.10', '3.11', '3.12'] + include: + - os: windows-latest + wheelname: win + cibw_archs: "AMD64" + - os: macos-13 + wheelname: macosx + cibw_archs: "x86_64" + - os: macos-latest + wheelname: macosx + cibw_archs: "arm64" + - os: ubuntu-latest + wheelname: manylinux + cibw_archs: "x86_64" + - python-version: '3.10' + version-tag: cp310 + - python-version: '3.11' + version-tag: cp311 + - python-version: '3.12' + version-tag: cp312 + fail-fast: false + runs-on: ${{ matrix.os }} + steps: + - name: Checkout project + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + fetch-tags: true + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install llvm on Macos + if: startsWith(matrix.os, 'macos') + shell: bash -l {0} + env: + # Homebrew location is different on Intel Mac + LLVM_DIR: ${{ (matrix.os == 'macos-13') && '/usr/local/opt/llvm' || '/opt/homebrew/opt/llvm' }} + run: | + brew install llvm + echo CC="${LLVM_DIR}/bin/clang" >> $GITHUB_ENV + echo LDFLAGS="-L${LLVM_DIR}/lib $LDFLAGS" >> $GITHUB_ENV + echo CPPFLAGS="-I${LLVM_DIR}/include $CPPFLAGS" >> $GITHUB_ENV + + - name: Windows - find MSVC and set environment variables + if: startsWith(matrix.os, 'windows') + shell: bash -l {0} + env: + MSVC_PREFIX: 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\MSVC' + run: | + echo "Available MSVC installations:" + ls "$MSVC_PREFIX" + + MSVC_BIN=$(ls "$MSVC_PREFIX" | tail -n 1)\\bin\\HostX64\\x64 + CC="$MSVC_PREFIX\\$MSVC_BIN\\cl.exe" + echo "CC: $CC" + echo "CC=$CC" >> $GITHUB_ENV + + CC_LD="$MSVC_PREFIX\\$MSVC_BIN\\link.exe" + echo "CC_LD: $CC_LD" + echo "CC_LD=$CC_LD" >> $GITHUB_ENV + + - name: Update Python pip, build, wheel, and twine + shell: bash -l {0} + run: | + python -m pip install --upgrade pip build wheel twine + + - name: Build wheels from git checkout + uses: pypa/cibuildwheel@v2.21.3 + env: + CIBW_BUILD_FRONTEND: build + CIBW_BUILD: ${{ matrix.version-tag }}-* + CIBW_ARCHS: ${{ matrix.cibw_archs }} + CIBW_SKIP: "*-musllinux*" + + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" + + CIBW_TEST_EXTRAS: "test,brille,phonopy-reader,matplotlib" + CIBW_TEST_COMMAND: python {package}/tests_and_analysis/test/run_tests.py + + with: + output-dir: wheelhouse + + - name: Upload wheels as build artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.wheelname }}-${{ matrix.python-version }}-${{ matrix.cibw_archs }} + path: wheelhouse/*-${{ matrix.wheelname }}*_${{ matrix.cibw_archs }}.whl + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 09dd4e6ab..2edce5f9b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,4 +12,79 @@ on: required: true type: string push: - description: "Push commit, tag and release" + description: "Make release and push to PyPI" + required: true + type: boolean + default: false + +jobs: + prepare: + runs-on: ubuntu-latest + + steps: + - name: Bump version number, push changes and tag + uses: ./.github/workflows/set_version.yml + with: + ref: ${{ inputs.ref }} + version: ${{ inputs.version }} + push: true + + build-wheels: + needs: prepare + runs-on: ubuntu-latest + + steps: + - name: Build and test wheels + uses: ./.github/workflows/build_wheels.yml + with: + ref: ${{ inputs.ref }} + + build-sdist: + needs: prepare + runs-on: ubuntu-latest + + steps: + - name: Build and test sdist + uses: ./.github/workflows/build_sdist.yml + with: + ref: ${{ inputs.ref }} + + release: + if: ${{ inputs.push }} + needs: [build-wheels,build-sdist] + runs-on: ubuntu-latest + + steps: + - name: Do nothing + shell: python + run : | + raise NotImplemented() + + publish: + if: ${{ inputs.push }} + needs: release + name: Upload release to PyPI + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/euphonic + permissions: + id-token: write + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 # This is possibly unnecessary? + + - name: Download artifacts to Ubuntu environment + uses: actions/download-artifact@v4 + with: + path: dist/ + merge-multiple: true + + - name: List Files + run: ls -R dist/ + + # - name: Upload wheels to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 From 4e08cf6e22c55c43e4f737d92e9d5ed96a8d7036 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Mon, 10 Feb 2025 10:55:26 +0000 Subject: [PATCH 11/29] Need to checkout or the actions can't be found --- .github/workflows/release.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2edce5f9b..6d0dd4907 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,10 +22,16 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + fetch-depth: 1 + - name: Bump version number, push changes and tag uses: ./.github/workflows/set_version.yml with: - ref: ${{ inputs.ref }} + ref: "" version: ${{ inputs.version }} push: true @@ -34,6 +40,11 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + - name: Build and test wheels uses: ./.github/workflows/build_wheels.yml with: @@ -44,6 +55,12 @@ jobs: runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + fetch-depth: 1 + - name: Build and test sdist uses: ./.github/workflows/build_sdist.yml with: From 488a013efddc0d8ee71ae97256f72ff309265d4a Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Mon, 10 Feb 2025 11:10:50 +0000 Subject: [PATCH 12/29] Reusable workflows: call at job level, not step --- .github/workflows/release.yml | 53 +++++++++-------------------------- 1 file changed, 14 insertions(+), 39 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6d0dd4907..507c3128e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,52 +19,27 @@ on: jobs: prepare: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref }} - fetch-depth: 1 - - - name: Bump version number, push changes and tag - uses: ./.github/workflows/set_version.yml - with: - ref: "" - version: ${{ inputs.version }} - push: true + name: Bump version number + uses: ./.github/workflows/set_version.yml + with: + ref: ${{ inputs.ref }} + version: ${{ inputs.version }} + push: true build-wheels: + name: Build and test wheels needs: prepare - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref }} - - name: Build and test wheels - uses: ./.github/workflows/build_wheels.yml - with: - ref: ${{ inputs.ref }} + uses: ./.github/workflows/build_wheels.yml + with: + ref: ${{ inputs.ref }} build-sdist: needs: prepare - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ inputs.ref }} - fetch-depth: 1 - - - name: Build and test sdist - uses: ./.github/workflows/build_sdist.yml - with: - ref: ${{ inputs.ref }} + name: Build and test sdist + uses: ./.github/workflows/build_sdist.yml + with: + ref: ${{ inputs.ref }} release: if: ${{ inputs.push }} From 7f4ca077d69c83b1defb69159bd48e51f20f6175 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Mon, 10 Feb 2025 11:20:49 +0000 Subject: [PATCH 13/29] Maybe swapping these steps will improve Github visualisation The workflow visualisation tends to put the "test sdist" step right on top of the dependency line from build-wheels to release. This is quite misleading as it looks like test sdist depends on build-wheels and yet it can start running before build-wheels is done. Perhaps if we swap the jobs they will be stacked in a different order and it will look clearer? --- .github/workflows/release.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 507c3128e..6af0e3b5d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,18 +26,18 @@ jobs: version: ${{ inputs.version }} push: true - build-wheels: - name: Build and test wheels + build-sdist: needs: prepare - - uses: ./.github/workflows/build_wheels.yml + name: Build and test sdist + uses: ./.github/workflows/build_sdist.yml with: ref: ${{ inputs.ref }} - build-sdist: + build-wheels: + name: Build and test wheels needs: prepare - name: Build and test sdist - uses: ./.github/workflows/build_sdist.yml + + uses: ./.github/workflows/build_wheels.yml with: ref: ${{ inputs.ref }} From 51e6a48bb7923bbe3469b29fb09421af7fab23db Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Mon, 10 Feb 2025 14:58:49 +0000 Subject: [PATCH 14/29] Call release.py from release.yml if the builds look good We could streamline things a bit by moving the code directly into release.yml, but for now let's keep options open. --- .github/workflows/release.yml | 13 +++++++++---- build_utils/release.py | 5 +++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6af0e3b5d..d22dde268 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,15 +42,20 @@ jobs: ref: ${{ inputs.ref }} release: - if: ${{ inputs.push }} + name: Github release needs: [build-wheels,build-sdist] runs-on: ubuntu-latest steps: - - name: Do nothing - shell: python + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + fetch-depth: 1 + + - name: run : | - raise NotImplemented() + python build_utils/release.py --github ${{ inputs.push && '--notest' || '' }} publish: if: ${{ inputs.push }} diff --git a/build_utils/release.py b/build_utils/release.py index 530f09450..21aa8ecf6 100644 --- a/build_utils/release.py +++ b/build_utils/release.py @@ -47,6 +47,11 @@ def release_github(test=True): } if test: print(payload) + + if "GITHUB_TOKEN" in os.environ: + print("Found GITHUB_TOKEN") + else: + print("No GITHUB_TOKEN set") else: response = requests.post( 'https://api.github.com/repos/pace-neutrons/euphonic/releases', From 10af9b175ec892c9fdf9521bcd3d2a69a4e1c0e7 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Mon, 10 Feb 2025 15:44:09 +0000 Subject: [PATCH 15/29] release.py expects Euphonic to be installed already This seems like overkill, we have already been quite careful to create consistent version information. But it wouldn't hurt to have these checks running for an automated release cycle before we consider pruning them. --- .github/workflows/release.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d22dde268..c8ae7d56d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,7 +53,16 @@ jobs: ref: ${{ inputs.ref }} fetch-depth: 1 - - name: + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Basic installation to check euphonic.__version__ + shell: bash -l {0} + run: | + python -m pip install . + + - name: Check version consistency, generate Github release page run : | python build_utils/release.py --github ${{ inputs.push && '--notest' || '' }} From c93133e38b7b1248d9dbff8b981443e3d4d86746 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Mon, 10 Feb 2025 16:30:24 +0000 Subject: [PATCH 16/29] release.py dependencies, linting --- .github/workflows/release.yml | 1 + build_utils/release.py | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c8ae7d56d..4f05fb33f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,6 +60,7 @@ jobs: - name: Basic installation to check euphonic.__version__ shell: bash -l {0} run: | + python -m pip install requests pyyaml python -m pip install . - name: Check version consistency, generate Github release page diff --git a/build_utils/release.py b/build_utils/release.py index 21aa8ecf6..3f5d1a5dd 100644 --- a/build_utils/release.py +++ b/build_utils/release.py @@ -2,9 +2,10 @@ import json import os import re + import requests -import subprocess import yaml + from euphonic import __version__ @@ -18,9 +19,9 @@ def main(): def release_github(test=True): - with open('CHANGELOG.rst') as f: + with open('CHANGELOG.rst', "rt", encoding="utf8") as f: changelog = f.read() - with open('CITATION.cff') as f: + with open('CITATION.cff', "rt", encoding="utf8") as f: citation = yaml.safe_load(f) euphonic_ver = __version__ @@ -30,7 +31,7 @@ def release_github(test=True): version_dict['CITATION.cff'] = 'v' + citation['version'] for ver_name, ver in version_dict.items(): if euphonic_ver != ver: - raise Exception(( + raise ValueError(( f'euphonic.__version__/{ver_name} version mismatch! ' f'euphonic.__version__: {euphonic_ver} {ver_name}: ' f'{ver}')) From c53b94f5e3df6c032925ba78c42aeb9a15f1802e Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Mon, 10 Feb 2025 16:47:40 +0000 Subject: [PATCH 17/29] release.py troubleshooting: check git log release.py is complaining that CHANGELOG is out of sync. How can that be? --- .github/workflows/release.yml | 35 +++++++++++++++++++++-------------- build_utils/release.py | 4 ++++ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4f05fb33f..c00a36f65 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,24 +26,25 @@ jobs: version: ${{ inputs.version }} push: true - build-sdist: - needs: prepare - name: Build and test sdist - uses: ./.github/workflows/build_sdist.yml - with: - ref: ${{ inputs.ref }} + # build-sdist: + # needs: prepare + # name: Build and test sdist + # uses: ./.github/workflows/build_sdist.yml + # with: + # ref: ${{ inputs.ref }} - build-wheels: - name: Build and test wheels - needs: prepare + # build-wheels: + # name: Build and test wheels + # needs: prepare - uses: ./.github/workflows/build_wheels.yml - with: - ref: ${{ inputs.ref }} + # uses: ./.github/workflows/build_wheels.yml + # with: + # ref: ${{ inputs.ref }} release: name: Github release - needs: [build-wheels,build-sdist] + # needs: [build-wheels,build-sdist] + needs: prepare runs-on: ubuntu-latest steps: @@ -51,7 +52,13 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ inputs.ref }} - fetch-depth: 1 + # fetch-depth: 1 + + - name: Git log, preview CHANGELOG + shell: bash -l {0} + run: | + git log | head -n 50 + head CHANGELOG.rst - uses: actions/setup-python@v5 with: diff --git a/build_utils/release.py b/build_utils/release.py index 3f5d1a5dd..234ce996a 100644 --- a/build_utils/release.py +++ b/build_utils/release.py @@ -24,6 +24,10 @@ def release_github(test=True): with open('CITATION.cff', "rt", encoding="utf8") as f: citation = yaml.safe_load(f) + print(changelog) + + print([tag for tag in re.findall(r'\n`(v\d+\.\d+\.\S+)\s', changelog)]) + euphonic_ver = __version__ version_dict = {} version_dict['CHANGELOG.rst'] = re.findall(r'\n`(v\d+\.\d+\.\S+)\s', From 389376d6cdad886617c6542fb54b5867d4e37444 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Mon, 10 Feb 2025 17:01:54 +0000 Subject: [PATCH 18/29] Correct version number format to CHANGELOG bumper Release script correctly picked up that the version number was written to changelog without the leading "v": this is inconsistent! --- .github/workflows/set_version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/set_version.yml b/.github/workflows/set_version.yml index e44084201..c9e103135 100644 --- a/.github/workflows/set_version.yml +++ b/.github/workflows/set_version.yml @@ -84,7 +84,7 @@ jobs: - name: "Update CHANGELOG.rst" shell: bash -l {0} - run: python build_utils/bump_changelog.py --replace CHANGELOG.rst ${VERSION_NUMBER} + run: python build_utils/bump_changelog.py --replace CHANGELOG.rst ${VERSION_STRING} - name: "Create commit" run: | From 1c4861d1bb0ee59803b78268dc7f4b653dd39143 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Mon, 10 Feb 2025 17:20:41 +0000 Subject: [PATCH 19/29] Detect prerelease, supply GITHUB_TOKEN --- .github/workflows/release.yml | 8 ++------ build_utils/release.py | 3 ++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c00a36f65..42e054454 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,12 +54,6 @@ jobs: ref: ${{ inputs.ref }} # fetch-depth: 1 - - name: Git log, preview CHANGELOG - shell: bash -l {0} - run: | - git log | head -n 50 - head CHANGELOG.rst - - uses: actions/setup-python@v5 with: python-version: "3.10" @@ -71,6 +65,8 @@ jobs: python -m pip install . - name: Check version consistency, generate Github release page + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run : | python build_utils/release.py --github ${{ inputs.push && '--notest' || '' }} diff --git a/build_utils/release.py b/build_utils/release.py index 234ce996a..915f0d4c2 100644 --- a/build_utils/release.py +++ b/build_utils/release.py @@ -3,6 +3,7 @@ import os import re +from packaging.version import Version import requests import yaml @@ -48,7 +49,7 @@ def release_github(test=True): "name": euphonic_ver, "body": desc, "draft": False, - "prerelease": False + "prerelease": Version(euphonic_ver).is_prerelease } if test: print(payload) From 4ca78110f09a12045d41e702654029add6f1328f Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Mon, 10 Feb 2025 17:26:40 +0000 Subject: [PATCH 20/29] Remove debugging bits, restore build/test step --- .github/workflows/release.yml | 29 ++++++++++++++--------------- build_utils/release.py | 4 ---- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 42e054454..c9c0f1fc7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,25 +26,24 @@ jobs: version: ${{ inputs.version }} push: true - # build-sdist: - # needs: prepare - # name: Build and test sdist - # uses: ./.github/workflows/build_sdist.yml - # with: - # ref: ${{ inputs.ref }} + build-sdist: + needs: prepare + name: Build and test sdist + uses: ./.github/workflows/build_sdist.yml + with: + ref: ${{ inputs.ref }} - # build-wheels: - # name: Build and test wheels - # needs: prepare + build-wheels: + name: Build and test wheels + needs: prepare - # uses: ./.github/workflows/build_wheels.yml - # with: - # ref: ${{ inputs.ref }} + uses: ./.github/workflows/build_wheels.yml + with: + ref: ${{ inputs.ref }} release: name: Github release - # needs: [build-wheels,build-sdist] - needs: prepare + needs: [build-wheels,build-sdist] runs-on: ubuntu-latest steps: @@ -52,7 +51,7 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ inputs.ref }} - # fetch-depth: 1 + fetch-depth: 1 - uses: actions/setup-python@v5 with: diff --git a/build_utils/release.py b/build_utils/release.py index 915f0d4c2..a6762da41 100644 --- a/build_utils/release.py +++ b/build_utils/release.py @@ -25,10 +25,6 @@ def release_github(test=True): with open('CITATION.cff', "rt", encoding="utf8") as f: citation = yaml.safe_load(f) - print(changelog) - - print([tag for tag in re.findall(r'\n`(v\d+\.\d+\.\S+)\s', changelog)]) - euphonic_ver = __version__ version_dict = {} version_dict['CHANGELOG.rst'] = re.findall(r'\n`(v\d+\.\d+\.\S+)\s', From c04b6cc059910e3bbe2046f2477716bc6a4c841e Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Tue, 11 Feb 2025 10:01:07 +0000 Subject: [PATCH 21/29] Build landing page from version.yml --- .github/workflows/create-landing-page.yml | 16 +++++++++++----- .github/workflows/release.yml | 16 ++++++++++++++++ .github/workflows/set_version.yml | 16 ++++++++++++---- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/.github/workflows/create-landing-page.yml b/.github/workflows/create-landing-page.yml index 30aab5019..1eecec854 100644 --- a/.github/workflows/create-landing-page.yml +++ b/.github/workflows/create-landing-page.yml @@ -5,8 +5,14 @@ on: - master paths: - 'CITATION.cff' - release: - types: [published] + + workflow_call: + inputs: + ref: + description: "Target: 'latest' or tag" + required: true + type: string + default: latest jobs: update-landing-page: @@ -26,9 +32,9 @@ jobs: - name: Set env var based on (master) push trigger if: github.event_name == 'push' run: echo "PAGE_TYPE=latest" >> $GITHUB_ENV - - name: Set env var based on release - if: github.event_name == 'release' - run: echo "PAGE_TYPE=${{ github.event.release.name }}" >> $GITHUB_ENV + - name: Set env var based on workflow call + if: github.event_name == 'workflow_call' + run: echo "PAGE_TYPE=${{ inputs.ref }}" >> $GITHUB_ENV - name: Create landing page run: python write_doi_landing_page.py ${{ env.PAGE_TYPE }} - uses: EndBug/add-and-commit@v9 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c9c0f1fc7..9e7889f53 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,6 +26,14 @@ jobs: version: ${{ inputs.version }} push: true + show_tag: + needs: prepare + runs-on: ubuntu-latest + steps: + - shell: bash -l {0} + run: | + echo NORMALISED TAG: ${{ needs.prepare.outputs.tag }} + build-sdist: needs: prepare name: Build and test sdist @@ -97,3 +105,11 @@ jobs: # - name: Upload wheels to PyPI # uses: pypa/gh-action-pypi-publish@release/v1 + + landing_page: + needs: [prepare,release] + if: ${{ inputs.push }} + name: Create landing page + uses: ./.github/workflows/create-landing-page.yml + with: + ref: ${{ needs.prepare.outputs.tag }} diff --git a/.github/workflows/set_version.yml b/.github/workflows/set_version.yml index c9e103135..8eb0eedc1 100644 --- a/.github/workflows/set_version.yml +++ b/.github/workflows/set_version.yml @@ -33,10 +33,16 @@ on: type: boolean default: false required: false + outputs: + tag: + description: "Normalised version tag" + value: ${{ jobs.set_version.outputs.tag }} jobs: - set-version: + set_version: runs-on: ubuntu-latest + outputs: + tag: ${{ steps.normalise.outputs.VERSION_STRING }} steps: - uses: actions/setup-python@v5 @@ -47,6 +53,7 @@ jobs: run: python -m pip install toolz packaging - name: Validate and normalise version number + id: normalise shell: python run : | import os @@ -57,9 +64,10 @@ jobs: print(f"Normalised version string: {version_string}") - with open(os.environ["GITHUB_ENV"], "a") as github_env: - print(f"VERSION_NUMBER={version_number}", file=github_env) - print(f"VERSION_STRING={version_string}", file=github_env) + for github_target in "GITHUB_ENV", "GITHUB_OUTPUT": + with open(os.environ[github_target], "a") as target: + print(f"VERSION_NUMBER={version_number}", file=target) + print(f"VERSION_STRING={version_string}", file=target) # From a workflow call we might already have working copy; in that # case inputs.ref should be an empty string. Otherwise, checkout. From 156dacd0ecaa08cb5c04fb32d4dc27702a0dd673 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Tue, 11 Feb 2025 10:31:34 +0000 Subject: [PATCH 22/29] Reusable workflow doesn't know it's a workflow_call This is a pretty confusing Actions behaviour: the github.event_name is inherited from the calling workflow so create-landing-page thinks it is a workflow_dispatch when called from release.yml --- .github/workflows/create-landing-page.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-landing-page.yml b/.github/workflows/create-landing-page.yml index 1eecec854..6570d815b 100644 --- a/.github/workflows/create-landing-page.yml +++ b/.github/workflows/create-landing-page.yml @@ -33,7 +33,7 @@ jobs: if: github.event_name == 'push' run: echo "PAGE_TYPE=latest" >> $GITHUB_ENV - name: Set env var based on workflow call - if: github.event_name == 'workflow_call' + if: (github.event_name == 'workflow_call') || (github.event_name == 'workflow_dispatch') run: echo "PAGE_TYPE=${{ inputs.ref }}" >> $GITHUB_ENV - name: Create landing page run: python write_doi_landing_page.py ${{ env.PAGE_TYPE }} From 03460f60aa1d62ab64fc2e9629098781b77a3c35 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Tue, 11 Feb 2025 11:38:36 +0000 Subject: [PATCH 23/29] Remove debug and activate PyPI push in Release action --- .github/workflows/release.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9e7889f53..4450845db 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,14 +26,6 @@ jobs: version: ${{ inputs.version }} push: true - show_tag: - needs: prepare - runs-on: ubuntu-latest - steps: - - shell: bash -l {0} - run: | - echo NORMALISED TAG: ${{ needs.prepare.outputs.tag }} - build-sdist: needs: prepare name: Build and test sdist @@ -103,8 +95,8 @@ jobs: - name: List Files run: ls -R dist/ - # - name: Upload wheels to PyPI - # uses: pypa/gh-action-pypi-publish@release/v1 + - name: Upload wheels to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 landing_page: needs: [prepare,release] From 699501d4c314cd09d868c0bd09d36085ad3023dc Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Tue, 11 Feb 2025 11:42:36 +0000 Subject: [PATCH 24/29] CHANGELOG update --- CHANGELOG.rst | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f863d5c04..d16f054b8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,18 @@ ill-defined cases, ForceConstants now sets both Born charges and dielectric tensor to None if only one was provided. + - An optional parameter is provided to change how bin edges are + obtained from bin centres: previously the bin edges were constrained + to the initial data range, but this can lead to incorrect scaling + when performing broadening. Variable-width broadening schemes are + now allowed to extrapolate the bin edges in order to get the correct + width scaling. + + Outside of broadening, the default behaviour is unchanged in order + to maintain backward compatibility. This is likely to be changed + in the next "major version" (i.e. API-breaking release) of + Euphonic. + - Maintenance - The euphonic.spectra module has been broken up into a subpackage @@ -24,20 +36,9 @@ external file. For clarity and safety, this field is now explicitly read by the Euphonic .castep_bin parser, but remains unused. - -- Bug fixes - - - An optional parameter is provided to change how bin edges are - obtained from bin centres: previously the bin edges were constrained - to the initial data range, but this can lead to incorrect scaling - when performing broadening. Variable-width broadening schemes are - now allowed to extrapolate the bin edges in order to get the correct - width scaling. - - Outside of broadening, the default behaviour is unchanged in order - to maintain backward compatibility. This is likely to be changed - in the next "major version" (i.e. API-breaking release) of - Euphonic. + - The release process has been reworked to reduce manual steps: a + top-level "release" action will now sequence most of the + steps. (Post-release testing is still separate.) `v1.4.0.post1 `_ ----------------------------------------------------------------------------------------- From e3c6661cdd3dadd385bf9747dacb03aa1f4ad0ec Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Tue, 11 Feb 2025 11:44:31 +0000 Subject: [PATCH 25/29] Remove build-upload action, this has been broken into smaller parts --- .../workflows/build_upload_pypi_wheels.yml | 211 ------------------ 1 file changed, 211 deletions(-) delete mode 100644 .github/workflows/build_upload_pypi_wheels.yml diff --git a/.github/workflows/build_upload_pypi_wheels.yml b/.github/workflows/build_upload_pypi_wheels.yml deleted file mode 100644 index 7243bd3f0..000000000 --- a/.github/workflows/build_upload_pypi_wheels.yml +++ /dev/null @@ -1,211 +0,0 @@ -name: Build and upload PyPI wheels and source dist - -on: - release: - types: [published] - workflow_dispatch: - -jobs: - build-wheels: - strategy: - matrix: - os: [windows-latest, macos-13, macos-latest, ubuntu-latest] - python-version: ['3.10', '3.11', '3.12'] - include: - - os: windows-latest - wheelname: win - cibw_archs: "AMD64" - - os: macos-13 - wheelname: macosx - cibw_archs: "x86_64" - - os: macos-latest - wheelname: macosx - cibw_archs: "arm64" - - os: ubuntu-latest - wheelname: manylinux - cibw_archs: "x86_64" - - python-version: '3.10' - version-tag: cp310 - - python-version: '3.11' - version-tag: cp311 - - python-version: '3.12' - version-tag: cp312 - fail-fast: false - runs-on: ${{ matrix.os }} - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install llvm on Macos - if: startsWith(matrix.os, 'macos') - shell: bash -l {0} - env: - # Homebrew location is different on Intel Mac - LLVM_DIR: ${{ (matrix.os == 'macos-13') && '/usr/local/opt/llvm' || '/opt/homebrew/opt/llvm' }} - run: | - brew install llvm - echo CC="${LLVM_DIR}/bin/clang" >> $GITHUB_ENV - echo LDFLAGS="-L${LLVM_DIR}/lib $LDFLAGS" >> $GITHUB_ENV - echo CPPFLAGS="-I${LLVM_DIR}/include $CPPFLAGS" >> $GITHUB_ENV - - - name: Windows - find MSVC and set environment variables - if: startsWith(matrix.os, 'windows') - shell: bash -l {0} - env: - MSVC_PREFIX: 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\MSVC' - run: | - echo "Available MSVC installations:" - ls "$MSVC_PREFIX" - - MSVC_BIN=$(ls "$MSVC_PREFIX" | tail -n 1)\\bin\\HostX64\\x64 - CC="$MSVC_PREFIX\\$MSVC_BIN\\cl.exe" - echo "CC: $CC" - echo "CC=$CC" >> $GITHUB_ENV - - CC_LD="$MSVC_PREFIX\\$MSVC_BIN\\link.exe" - echo "CC_LD: $CC_LD" - echo "CC_LD=$CC_LD" >> $GITHUB_ENV - - - name: Update Python pip, build, wheel, and twine - shell: bash -l {0} - run: | - python -m pip install --upgrade pip build wheel twine - - - name: Build wheels from git checkout - uses: pypa/cibuildwheel@v2.21.3 - env: - CIBW_BUILD_FRONTEND: build - CIBW_BUILD: ${{ matrix.version-tag }}-* - CIBW_ARCHS: ${{ matrix.cibw_archs }} - CIBW_SKIP: "*-musllinux*" - - CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" - - CIBW_TEST_EXTRAS: "test,brille,phonopy-reader,matplotlib" - CIBW_TEST_COMMAND: python {package}/tests_and_analysis/test/run_tests.py - - with: - output-dir: wheelhouse - - - name: Upload wheels as build artifacts - uses: actions/upload-artifact@v4 - with: - name: wheel-${{ matrix.wheelname }}-${{ matrix.python-version }}-${{ matrix.cibw_archs }} - path: wheelhouse/*-${{ matrix.wheelname }}*_${{ matrix.cibw_archs }}.whl - if-no-files-found: error - - build-sdist: - name: Build sdist - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - - - name: Create source distribution - shell: bash -l {0} - run: | - pipx run build --sdist . - - - name: Upload source dist as build artifact - uses: actions/upload-artifact@v4 - with: - name: python-source-distribution - path: dist/ - if-no-files-found: error - - test-sdist: - needs: build-sdist - name: Test build from sdist on Windows - runs-on: windows-latest - steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: 3.11 - - - name: Find MSVC and set environment variables - shell: bash -l {0} - env: - MSVC_PREFIX: 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\MSVC' - run: | - echo "Available MSVC installations:" - ls "$MSVC_PREFIX" - - MSVC_BIN=$(ls "$MSVC_PREFIX" | tail -n 1)\\bin\\HostX64\\x64 - CC="$MSVC_PREFIX\\$MSVC_BIN\\cl.exe" - echo "CC: $CC" - echo "CC=$CC" >> $GITHUB_ENV - - CC_LD="$MSVC_PREFIX\\$MSVC_BIN\\link.exe" - echo "CC_LD: $CC_LD" - echo "CC_LD=$CC_LD" >> $GITHUB_ENV - - - name: Download sdist - uses: actions/download-artifact@v4 - with: - path: dist/ - merge-multiple: true - - - name: List downloaded sdist - run: | - ls -R dist/ - - - name: Update pip - shell: bash -l {0} - run: | - python -m pip install --upgrade pip - - - name: Install from sdist - shell: bash -l {0} - run: python -m pip install $(find dist -name 'euphonic-*.tar.gz')[matplotlib,phonopy-reader,brille,test] - - - name: Checkout repository (for tests and test data) - uses: actions/checkout@v4 - - - name: Delete source (to ensure we are testing INSTALLED version) - shell: bash -l {0} - run: rm -rf euphonic - - - name: run tests - shell: bash -l {0} - run: python tests_and_analysis/test/run_tests.py --report - - publish: - if: github.event_name == 'release' - needs: [build-wheels,test-sdist] - name: Upload release to PyPI - runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/p/euphonic - permissions: - id-token: write - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 # This is possibly unnecessary? - - - name: Download artifacts to Ubuntu environment - uses: actions/download-artifact@v4 - with: - path: dist/ - merge-multiple: true - - - name: List Files - run: ls -R dist/ - - - name: Upload wheels to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 From ed16103db18064170891adcd8f53c588363c0f66 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Tue, 11 Feb 2025 14:48:18 +0000 Subject: [PATCH 26/29] Apply suggestions from code review Co-authored-by: Jacob Wilkins <46597752+oerc0122@users.noreply.github.com> --- build_utils/bump_changelog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_utils/bump_changelog.py b/build_utils/bump_changelog.py index 97aa479fb..a23a57c68 100755 --- a/build_utils/bump_changelog.py +++ b/build_utils/bump_changelog.py @@ -40,7 +40,7 @@ def parse_changelog(changelog_file: Path) -> list[Block]: split_text = split_text[1:] # From Python 3.12 can replace partition with itertools.batched - blocks = [Block(*block_data) for block_data in partition(3, split_text)] + blocks = [Block(tag, prev_tag, content) for tag, prev_tag, content in partition(3, split_text)] for block in blocks: block.content = block.content.strip() @@ -62,7 +62,7 @@ def bump_version(blocks: list[Block], tag: str) -> None: blocks.insert(1, Block(tag, previous_tag, "")) blocks[1].content = "\n\n".join( - [txt for txt in (blocks[0].content, blocks[1].content) if txt] + txt for txt in (blocks[0].content, blocks[1].content) if txt ) # Update "Unreleased" section From 33ed3dde799490de91bdb9b8627cc19b590fd075 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Tue, 11 Feb 2025 15:08:50 +0000 Subject: [PATCH 27/29] Refactor bump_changelog: define link string in one place --- build_utils/bump_changelog.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/build_utils/bump_changelog.py b/build_utils/bump_changelog.py index a23a57c68..d39094de1 100755 --- a/build_utils/bump_changelog.py +++ b/build_utils/bump_changelog.py @@ -12,6 +12,7 @@ from toolz.itertoolz import partition REPOSITORY_ADDRESS = "https://github.com/pace-neutrons/Euphonic" +LINK_STRING = "`{tag} <" + REPOSITORY_ADDRESS + "/compare/{previous_tag}...{compare_tag}>`_" @dataclass @@ -34,7 +35,13 @@ def parse_changelog(changelog_file: Path) -> list[Block]: """ with open(changelog_file, "rt", encoding="utf8") as fd: - split_text = re.split(r"`(\S+) <\S+/compare/(\S+)\.\.\.\S+>`_\n-+", fd.read()) + split_text = re.split(LINK_STRING + .replace('.', r'\.') + .format(tag=r"(\S+)", + previous_tag=r"(\S+)", + compare_tag=r"\S+" + ) + r"\n-+", + fd.read()) # First item is always empty? split_text = split_text[1:] @@ -73,7 +80,7 @@ def bump_version(blocks: list[Block], tag: str) -> None: def tag_to_header(tag: str, previous_tag: str) -> str: """Produce header including github diff link""" compare_tag = "HEAD" if tag == "Unreleased" else tag - header = f"`{tag} <{REPOSITORY_ADDRESS}/compare/{previous_tag}...{compare_tag}>`_" + header = LINK_STRING.format(tag=tag, previous_tag=previous_tag, compare_tag=compare_tag) return header + "\n" + len(header) * "-" From 414dce552431dfeddbb086a73908b01a86ea578f Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Tue, 11 Feb 2025 15:17:53 +0000 Subject: [PATCH 28/29] Refactor bump_changelog: block printing to __str__ method --- build_utils/bump_changelog.py | 43 +++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/build_utils/bump_changelog.py b/build_utils/bump_changelog.py index d39094de1..fbe3b4a58 100755 --- a/build_utils/bump_changelog.py +++ b/build_utils/bump_changelog.py @@ -22,6 +22,22 @@ class Block: previous_tag: str content: str + @staticmethod + def tag_to_header(tag: str, previous_tag: str) -> str: + """Produce header including github diff link""" + compare_tag = "HEAD" if tag == "Unreleased" else tag + header = LINK_STRING.format(tag=tag, + previous_tag=previous_tag, + compare_tag=compare_tag) + return header + "\n" + len(header) * "-" + + def __str__(self) -> str: + txt = self.tag_to_header(self.tag, self.previous_tag) + if self.content: + txt += f"\n\n{self.content}" + + return txt + def parse_changelog(changelog_file: Path) -> list[Block]: """Read all sections from changelog file @@ -35,13 +51,13 @@ def parse_changelog(changelog_file: Path) -> list[Block]: """ with open(changelog_file, "rt", encoding="utf8") as fd: - split_text = re.split(LINK_STRING - .replace('.', r'\.') - .format(tag=r"(\S+)", - previous_tag=r"(\S+)", - compare_tag=r"\S+" - ) + r"\n-+", - fd.read()) + split_text = re.split( + LINK_STRING.replace(".", r"\.").format( + tag=r"(\S+)", previous_tag=r"(\S+)", compare_tag=r"\S+" + ) + + r"\n-+", + fd.read(), + ) # First item is always empty? split_text = split_text[1:] @@ -77,20 +93,9 @@ def bump_version(blocks: list[Block], tag: str) -> None: blocks[0].previous_tag = tag -def tag_to_header(tag: str, previous_tag: str) -> str: - """Produce header including github diff link""" - compare_tag = "HEAD" if tag == "Unreleased" else tag - header = LINK_STRING.format(tag=tag, previous_tag=previous_tag, compare_tag=compare_tag) - return header + "\n" + len(header) * "-" - - def to_text(blocks: list[Block]) -> str: """Dump blocks to single multiline string""" - return "\n\n".join( - tag_to_header(block.tag, block.previous_tag) - + (f"\n\n{block.content}" if block.content else "") - for block in blocks - ) + return "\n\n".join(map(str, blocks)) def get_parser() -> ArgumentParser: From c1f259b876b9df90c5e5f40ec11b4dec35d81e34 Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Tue, 11 Feb 2025 15:29:23 +0000 Subject: [PATCH 29/29] Update build_utils/bump_changelog.py Co-authored-by: Jacob Wilkins <46597752+oerc0122@users.noreply.github.com> --- build_utils/bump_changelog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_utils/bump_changelog.py b/build_utils/bump_changelog.py index fbe3b4a58..a76a3b160 100755 --- a/build_utils/bump_changelog.py +++ b/build_utils/bump_changelog.py @@ -50,7 +50,7 @@ def parse_changelog(changelog_file: Path) -> list[Block]: except for 'Unreleased' which compares with HEAD """ - with open(changelog_file, "rt", encoding="utf8") as fd: + with changelog_file.open(encoding="utf8") as fd: split_text = re.split( LINK_STRING.replace(".", r"\.").format( tag=r"(\S+)", previous_tag=r"(\S+)", compare_tag=r"\S+"