From 3e89c679f6713e16185ccbbf4d7c4ca1e5682369 Mon Sep 17 00:00:00 2001 From: Ilia Kurenkov Date: Thu, 26 Oct 2023 14:32:46 +0200 Subject: [PATCH] Generate changelogs from fragments using towncrier. (#15983) --- .github/workflows/pr-quick-check.yml | 1 + changelog_template.jinja | 13 + datadog_checks_dev/CHANGELOG.md | 2 +- datadog_checks_dev/changelog.d/15983.changed | 3 + .../dev/tooling/commands/release/changelog.py | 125 +--- .../dev/tooling/commands/release/make.py | 11 +- .../tooling/commands/release/show/changes.py | 48 +- .../datadog_checks/dev/tooling/constants.py | 12 +- datadog_checks_dev/pyproject.toml | 2 + ddev/CHANGELOG.md | 2 +- ddev/changelog.d/15983.changed | 3 + ddev/src/ddev/cli/release/changelog/fix.py | 12 +- ddev/src/ddev/cli/release/changelog/new.py | 73 +-- ddev/src/ddev/release/constants.py | 2 +- ddev/src/ddev/utils/scripts/check_pr.py | 110 +++- ddev/tests/cli/release/test_changelog.py | 601 +++++------------- towncrier.toml | 42 ++ 17 files changed, 418 insertions(+), 644 deletions(-) create mode 100644 changelog_template.jinja create mode 100644 datadog_checks_dev/changelog.d/15983.changed create mode 100644 ddev/changelog.d/15983.changed create mode 100644 towncrier.toml diff --git a/.github/workflows/pr-quick-check.yml b/.github/workflows/pr-quick-check.yml index a5f18257ce5a6..393cf03276292 100644 --- a/.github/workflows/pr-quick-check.yml +++ b/.github/workflows/pr-quick-check.yml @@ -57,3 +57,4 @@ jobs: --diff-file /tmp/diff --pr-file "$GITHUB_EVENT_PATH" ${{ github.event.pull_request.base.repo.private && '--private' || '' }} + --repo "${{ inputs.repo }}" diff --git a/changelog_template.jinja b/changelog_template.jinja new file mode 100644 index 0000000000000..ab0e397ed0f93 --- /dev/null +++ b/changelog_template.jinja @@ -0,0 +1,13 @@ +{% if sections[""] %} +{% for category, val in definitions.items() if category in sections[""] %} +***{{ definitions[category]['name'] }}***: + +{% for text, values in sections[""][category].items() %} +* {{ text }} {{ values|join(', ') }} +{% endfor %} + +{% endfor %} +{% else %} +No significant changes. + +{% endif %} diff --git a/datadog_checks_dev/CHANGELOG.md b/datadog_checks_dev/CHANGELOG.md index ab15055c773ea..670de0edc88b6 100644 --- a/datadog_checks_dev/CHANGELOG.md +++ b/datadog_checks_dev/CHANGELOG.md @@ -1,6 +1,6 @@ # CHANGELOG - Datadog Checks Dev -## Unreleased + ## 27.0.1 / 2023-10-26 diff --git a/datadog_checks_dev/changelog.d/15983.changed b/datadog_checks_dev/changelog.d/15983.changed new file mode 100644 index 0000000000000..4a86ec4a0f36c --- /dev/null +++ b/datadog_checks_dev/changelog.d/15983.changed @@ -0,0 +1,3 @@ +Generate changelogs from fragment files using towncrier. +There are no changes to the ddev commands, only to their outputs. +We are making this change to avoid merge conflicts in high-traffic packages where people used to have to modify one CHANGELOG.md file. diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/release/changelog.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/release/changelog.py index e7eca59059a6d..1b223bee71b3c 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/commands/release/changelog.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/release/changelog.py @@ -2,52 +2,53 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) import os +import sys from collections import namedtuple -from datetime import date, datetime -from io import StringIO import click from semver import VersionInfo -from ....fs import stream_file_lines, write_file -from ...constants import get_root +from datadog_checks.dev.tooling.constants import get_root + from ...utils import complete_testable_checks, get_valid_checks, get_version_string -from ..console import CONTEXT_SETTINGS, abort, echo_info, echo_success, validate_check_arg +from ..console import CONTEXT_SETTINGS, abort, echo_info, run_or_abort, validate_check_arg ChangelogEntry = namedtuple('ChangelogEntry', 'number, title, url, author, author_url, from_contributor') +def towncrier(target_dir, cmd, *cmd_args): + ''' + Run towncrier command with its arguments in target_dir. + ''' + tc_res = run_or_abort( + [ + sys.executable, + "-m", + "towncrier", + cmd, + "--config", + os.path.join(get_root(), "towncrier.toml"), + "--dir", + target_dir, + *cmd_args, + ], + capture='both', + ) + echo_info(tc_res.stdout.rstrip()) + return tc_res + + @click.command(context_settings=CONTEXT_SETTINGS, short_help='Update the changelog for a check') @click.argument('check', shell_complete=complete_testable_checks, callback=validate_check_arg) @click.argument('version') @click.argument('old_version', required=False) -@click.option('--end') -@click.option('--initial', is_flag=True) -@click.option('--organization', '-r', default='DataDog') @click.option('--quiet', '-q', is_flag=True) @click.option('--dry-run', '-n', is_flag=True) -@click.option('--output-file', '-o', default='CHANGELOG.md', show_default=True) @click.option('--tag-pattern', default=None, hidden=True) @click.option('--tag-prefix', '-tp', default='v', show_default=True) @click.option('--no-semver', '-ns', default=False, is_flag=True) -@click.option('--exclude-branch', default=None, help="Exclude changes comming from a specific branch") -@click.pass_context -def changelog( - ctx, - check, - version, - old_version, - end, - initial, - quiet, - dry_run, - output_file, - tag_pattern, - tag_prefix, - no_semver, - organization, - exclude_branch, -): +@click.option('--date', default=None) +def changelog(check, version, old_version, quiet, dry_run, tag_pattern, tag_prefix, no_semver, date): """Perform the operations needed to update the changelog. This method is supposed to be used by other tasks and not directly. @@ -71,69 +72,9 @@ def changelog( if not quiet: echo_info(f'Current version of check {check}: {cur_version}, bumping to: {version}') - # read the old contents - if check: - changelog_path = os.path.join(get_root(), check, output_file) - else: - changelog_path = os.path.join(get_root(), output_file) - old = list(stream_file_lines(changelog_path)) - - if initial: - # For initial releases, just keep the ddev generated CHANGELOG but update the date to today - for idx, line in enumerate(old): - if line.startswith("## 1.0.0"): - old[idx] = f"## 1.0.0 / {date.today()}\n" - break - write_result(dry_run, changelog_path, ''.join(old), num_changes=1) - return - - # find the first header below the Unreleased section - header_index = 2 - for index in range(2, len(old)): - if old[index].startswith("##") and "## Unreleased" not in old[index]: - header_index = index - break - - # get text from the unreleased section - if header_index == 4: - abort('There are no changes for this integration') - - changelogs = old[4:header_index] - num_changelogs = 0 - for line in changelogs: - if line.startswith('* '): - num_changelogs += 1 - - # the header contains version and date - header = f"## {version} / {datetime.utcnow().strftime('%Y-%m-%d')}\n" - - # store the new changelog in memory - new_entry = StringIO() - new_entry.write(header) - new_entry.write('\n') - - # write the new changelog in memory - changelog_buffer = StringIO() - - # preserve the title and unreleased section - changelog_buffer.write(''.join(old[:4])) - - # prepend the new changelog to the old contents - # make the command idempotent - if header not in old: - changelog_buffer.write(new_entry.getvalue()) - - # append the rest of the old changelog - changelog_buffer.write(''.join(old[4:])) - - write_result(dry_run, changelog_path, changelog_buffer.getvalue(), num_changelogs) - - -def write_result(dry_run, changelog_path, final_output, num_changes): - # print on the standard out in case of a dry run + build_args = ["--yes", "--version", version] if dry_run: - echo_info(final_output) - else: - # overwrite the old changelog - write_file(changelog_path, final_output) - echo_success(f"Successfully generated {num_changes} change{'s' if num_changes > 1 else ''}") + build_args.append("--draft") + if date: + build_args.extend(["--date", date]) + towncrier(os.path.join(get_root(), check), "build", *build_args) diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/release/make.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/release/make.py index 08f3ad8538446..5cb0a111ff7bc 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/commands/release/make.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/release/make.py @@ -132,10 +132,10 @@ def make(ctx, checks, version, end, initial_release, skip_sign, sign_only, exclu if check == 'ddev': cur_version = get_version_string(check) _, changelog_types = ctx.invoke( - changes, check=check, tag_pattern='ddev-v.+', tag_prefix='ddev-v', end=end, dry_run=True + changes, check=check, tag_pattern='ddev-v.+', tag_prefix='ddev-v', dry_run=True ) else: - cur_version, changelog_types = ctx.invoke(changes, check=check, end=end, dry_run=True) + cur_version, changelog_types = ctx.invoke(changes, check=check, dry_run=True) echo_debug(f'Current version: {cur_version}. Changes: {changelog_types}') if not changelog_types: @@ -149,18 +149,15 @@ def make(ctx, checks, version, end, initial_release, skip_sign, sign_only, exclu # update the CHANGELOG echo_waiting('Updating the changelog... ', nl=False) - # TODO: Avoid double GitHub API calls when bumping all checks at once ctx.invoke( changelog, check=check, version=version, old_version=None if check == 'ddev' else cur_version, - end=end, - initial=initial_release, - tag_pattern='ddev-v.+' if check == 'ddev' else None, - tag_prefix='ddev-v' if check == 'ddev' else 'v', quiet=True, dry_run=False, + tag_pattern='ddev-v.+' if check == 'ddev' else None, + tag_prefix='ddev-v' if check == 'ddev' else 'v', ) echo_success('success!') diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/release/show/changes.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/release/show/changes.py index f51776da184b5..d8061c345320d 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/commands/release/show/changes.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/release/show/changes.py @@ -5,22 +5,18 @@ import click -from .....fs import stream_file_lines from ....constants import get_root from ....utils import complete_valid_checks, get_valid_checks, get_version_string from ...console import ( CONTEXT_SETTINGS, abort, - echo_failure, echo_info, - echo_success, validate_check_arg, ) @click.command(context_settings=CONTEXT_SETTINGS, short_help='Show all the pending PRs for a given check.') @click.argument('check', shell_complete=complete_valid_checks, callback=validate_check_arg) -@click.option('--organization', '-r', default='DataDog', help="The Github organization the repository belongs to") @click.option( '--tag-pattern', default=None, @@ -33,10 +29,7 @@ @click.option( '--since', default=None, help="The git ref to use instead of auto-detecting the tag to view changes since" ) -@click.option('--end') -@click.option('--exclude-branch', default=None, help="Exclude changes comming from a specific branch") -@click.pass_context -def changes(ctx, check, tag_pattern, tag_prefix, dry_run, organization, since, end, exclude_branch): +def changes(check, tag_pattern, tag_prefix, dry_run, since): """Show all the pending PRs for a given check.""" if not dry_run and check and check not in get_valid_checks(): abort(f'Check `{check}` is not an Agent-based Integration') @@ -49,31 +42,18 @@ def changes(ctx, check, tag_pattern, tag_prefix, dry_run, organization, since, e 'following SemVer and matches the provided tag_prefix and/or tag_pattern.' ) - if check: - changelog_path = os.path.join(get_root(), check, 'CHANGELOG.md') - else: - changelog_path = os.path.join(get_root(), 'CHANGELOG.md') - log = list(stream_file_lines(changelog_path)) - - header_index = 2 - for index in range(2, len(log)): - if log[index].startswith("##") and "## Unreleased" not in log[index]: - header_index = index - break - - if header_index == 4: - echo_failure('There are no changes for this integration') - return cur_version, [] - - unreleased = log[4:header_index] - applicable_changelog_types = [] - - for line in unreleased: - if line.startswith('***'): - applicable_changelog_types.append(line[3:-5]) - echo_success(line) - - elif line.strip(): - echo_info(line) + applicable_changelog_types = set() + fragment_dir = os.path.join(get_root(), check, 'changelog.d') + if not os.path.exists(fragment_dir): + echo_info('No changes for this check.') + return cur_version, applicable_changelog_types + changes_to_report = [] + for fname in os.listdir(fragment_dir): + applicable_changelog_types.add(fname.split(".")[1]) + changes_to_report.append(f'{os.path.join(check, "changelog.d", fname)}:') + fpath = os.path.join(fragment_dir, fname) + with open(fpath, mode='r') as fh: + changes_to_report.append(fh.read() + '\n') + echo_info('\n'.join(changes_to_report).rstrip()) return cur_version, applicable_changelog_types diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/constants.py b/datadog_checks_dev/datadog_checks/dev/tooling/constants.py index 8f68096cab2fb..593e6ea17ad55 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/constants.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/constants.py @@ -37,12 +37,12 @@ } VERSION_BUMP = { - 'Added': semver.bump_minor, - 'Changed': semver.bump_major, - 'Deprecated': semver.bump_minor, - 'Fixed': semver.bump_patch, - 'Removed': semver.bump_major, - 'Security': semver.bump_minor, + 'added': semver.bump_minor, + 'changed': semver.bump_major, + 'deprecated': semver.bump_minor, + 'fixed': semver.bump_patch, + 'removed': semver.bump_major, + 'security': semver.bump_minor, 'major': semver.bump_major, 'minor': semver.bump_minor, 'patch': semver.bump_patch, diff --git a/datadog_checks_dev/pyproject.toml b/datadog_checks_dev/pyproject.toml index 7acf291a1c6ec..d8c658e5928e1 100644 --- a/datadog_checks_dev/pyproject.toml +++ b/datadog_checks_dev/pyproject.toml @@ -85,6 +85,8 @@ cli = [ "tomli>=1.1.0", "tomli-w>=1.0.0", "tox>=3.12.1, <4.0.0", + # Move towncrier dependency to ddev once we migrate changelog-related commands. + "towncrier==23.10.0", "twine>=1.11.0", "virtualenv<20.22.0", # TODO: Remove once every check has a pyproject.toml diff --git a/ddev/CHANGELOG.md b/ddev/CHANGELOG.md index f2cd8c2fbc6b9..3fe5f401abdb0 100644 --- a/ddev/CHANGELOG.md +++ b/ddev/CHANGELOG.md @@ -1,6 +1,6 @@ # CHANGELOG - ddev -## Unreleased + ## 5.3.0 / 2023-10-26 diff --git a/ddev/changelog.d/15983.changed b/ddev/changelog.d/15983.changed new file mode 100644 index 0000000000000..4a86ec4a0f36c --- /dev/null +++ b/ddev/changelog.d/15983.changed @@ -0,0 +1,3 @@ +Generate changelogs from fragment files using towncrier. +There are no changes to the ddev commands, only to their outputs. +We are making this change to avoid merge conflicts in high-traffic packages where people used to have to modify one CHANGELOG.md file. diff --git a/ddev/src/ddev/cli/release/changelog/fix.py b/ddev/src/ddev/cli/release/changelog/fix.py index 503989cdd3859..5d6cefa2ec2af 100644 --- a/ddev/src/ddev/cli/release/changelog/fix.py +++ b/ddev/src/ddev/cli/release/changelog/fix.py @@ -11,14 +11,20 @@ from ddev.cli.application import Application -@click.command(short_help='Fix changelog entries') +@click.command @click.pass_obj def fix(app: Application): """ + Fix changelog entries. + + This command is only needed if you are manually writing to the changelog. + For instance for marketplace and extras integrations. + Don't use this in integrations-core because the changelogs there are generated automatically. + The first line of every new changelog entry must include the PR number in which the change occurred. This command will apply this suffix to manually added entries if it is missing. """ - from ddev.utils.scripts.check_pr import changelog_entry_suffix, get_changelog_errors + from ddev.utils.scripts.check_pr import changelog_entry_suffix, get_noncore_repo_changelog_errors latest_commit = app.repo.git.latest_commit pr = app.github.get_pull_request(latest_commit.sha) @@ -37,7 +43,7 @@ def fix(app: Application): expected_suffix = changelog_entry_suffix(pr_number, pr_url) fixed = 0 - for path, line_number, _ in get_changelog_errors(git_diff, expected_suffix): + for path, line_number, _ in get_noncore_repo_changelog_errors(git_diff, expected_suffix): if line_number == 1: continue diff --git a/ddev/src/ddev/cli/release/changelog/new.py b/ddev/src/ddev/cli/release/changelog/new.py index a5dd5ea570b6e..9ceb56cd32a9d 100644 --- a/ddev/src/ddev/cli/release/changelog/new.py +++ b/ddev/src/ddev/cli/release/changelog/new.py @@ -28,81 +28,34 @@ def new(app: Application, entry_type: str | None, targets: tuple[str], message: By default, changelog entries will be created for all integrations that have changed code. To create entries only for specific targets, you may pass them as additional arguments after the entry type. """ + from datadog_checks.dev.tooling.commands.release.changelog import towncrier + from ddev.release.constants import ENTRY_TYPES - from ddev.utils.scripts.check_pr import changelog_entry_suffix - derive_message = message is None latest_commit = app.repo.git.latest_commit pr = app.github.get_pull_request(latest_commit.sha) + message_based_on_git = '' if pr is not None: pr_number = pr.number - pr_url = pr.html_url - if message is None: - message = pr.title + message_based_on_git = pr.title else: pr_number = app.github.get_next_issue_number() - pr_url = f'https://github.com/{app.github.repo_id}/pull/{pr_number}' - if message is None: - message = latest_commit.subject + message_based_on_git = latest_commit.subject if entry_type is not None: - entry_type = entry_type.capitalize() if entry_type not in ENTRY_TYPES: app.abort(f'Unknown entry type: {entry_type}') else: entry_type = click.prompt('Entry type?', type=click.Choice(ENTRY_TYPES, case_sensitive=False)) - expected_suffix = changelog_entry_suffix(pr_number, pr_url) - entry = f'* {message.rstrip()}{expected_suffix}' - if derive_message and (new_entry := click.edit(entry)) is not None: - entry = new_entry - entry = entry.strip() - - entry_priority = ENTRY_TYPES.index(entry_type) + create_cmd = [ + 'create', + '--content', + message or click.edit(text=message_based_on_git, require_save=False) or message_based_on_git, + f'{pr_number}.{entry_type}', + ] edited = 0 - - for target in app.repo.integrations.iter_changed_code(targets): - changelog = target.path / 'CHANGELOG.md' - lines = changelog.read_text().splitlines() - - unreleased = False - current_entry_type: str | None = None - i = 0 - for i, line in enumerate(lines): - if line == '## Unreleased': - unreleased = True - continue - elif unreleased and line.startswith('## '): - break - elif line.startswith('***'): - # e.g. ***Added***: - current_entry_type = line[3:-4] - - try: - current_entry_priority = ENTRY_TYPES.index(current_entry_type) - except ValueError: - app.abort( - f'{changelog.relative_to(app.repo.path)}, line {i}: unknown entry type {current_entry_type}' - ) - - if current_entry_priority > entry_priority: - break - - if current_entry_type is None or current_entry_type != entry_type: - for line in reversed( - ( - f'***{entry_type}***:', - '', - entry, - '', - ) - ): - lines.insert(i, line) - else: - lines.insert(i - 1, entry) - - lines.append('') - changelog.write_text('\n'.join(lines)) + for check in app.repo.integrations.iter_changed_code(targets): + towncrier(check.path, *create_cmd) edited += 1 - app.display_success(f'Added {edited} changelog entr{"ies" if edited > 1 else "y"}') diff --git a/ddev/src/ddev/release/constants.py b/ddev/src/ddev/release/constants.py index 3da15eb710097..8903a4f7c4e16 100644 --- a/ddev/src/ddev/release/constants.py +++ b/ddev/src/ddev/release/constants.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -ENTRY_TYPES = ('Removed', 'Changed', 'Security', 'Deprecated', 'Added', 'Fixed') +ENTRY_TYPES = ('removed', 'changed', 'security', 'deprecated', 'added', 'fixed') diff --git a/ddev/src/ddev/utils/scripts/check_pr.py b/ddev/src/ddev/utils/scripts/check_pr.py index c351765a48c0e..81834415f9587 100644 --- a/ddev/src/ddev/utils/scripts/check_pr.py +++ b/ddev/src/ddev/utils/scripts/check_pr.py @@ -9,6 +9,7 @@ import os import re import sys +from collections import defaultdict from typing import Iterator @@ -98,7 +99,12 @@ def get_added_lines(git_diff: str) -> dict[str, dict[int, str]]: return files -def get_changelog_errors(git_diff: str, suffix: str, private: bool = False) -> list[tuple[str, int, str]]: +def get_noncore_repo_changelog_errors(git_diff: str, suffix: str, private: bool = False) -> list[tuple[str, int, str]]: + ''' + Extras and Marketplace repos manage their changelogs as a single file. + + We make sure that what contributors write to it follows are formatting conventions. + ''' targets: dict[str, dict[str, dict[int, str]]] = {} for filename, lines in get_added_lines(git_diff).items(): target, _, path = filename.partition('/') @@ -144,7 +150,89 @@ def get_changelog_errors(git_diff: str, suffix: str, private: bool = False) -> l return errors -def changelog_impl(*, ref: str, diff_file: str, pr_file: str, private: bool) -> None: +def extract_filenames(git_diff: str) -> Iterator[str]: + for modification in re.split(r'^diff --git ', git_diff, flags=re.MULTILINE): + if not modification: + continue + + # a/file b/file + # new file mode 100644 + # index 0000000000..089fd64579 + # --- a/file + # +++ b/file + metadata, *_ = re.split(r'^@@ ', modification, flags=re.MULTILINE) + *_, before, after = metadata.strip().splitlines() + + # Binary files /dev/null and b/foo/archive.tar.gz differ + binary_indicator = 'Binary files ' + if after.startswith(binary_indicator): + line = after[len(binary_indicator) :].rsplit(maxsplit=1)[0] + if line.startswith('/dev/null and '): + filename = line.split(maxsplit=2)[-1][2:] + elif line.endswith(' and /dev/null'): + filename = line.split(maxsplit=2)[0][2:] + else: + _, _, filename = line.partition(' and b/') + + yield filename + continue + + # --- a/file + # +++ /dev/null + before = before.split(maxsplit=1)[1] + after = after.split(maxsplit=1)[1] + filename = before[2:] if after == '/dev/null' else after[2:] + yield filename + + +def get_core_repo_changelog_errors(git_diff: str, pr_number: int) -> list[str]: + ''' + The integrations-core repo uses towncrier to stitch a release changelog from entry files. + + The validation reflects this so it's different from extras and marketplace. + ''' + targets: defaultdict[str, list[str]] = defaultdict(list) + for filename in extract_filenames(git_diff): + target, _, path = filename.partition('/') + if path: + targets[target].append(path) + + fragments_dir = 'changelog.d' + errors: list[str] = [] + for target, files in sorted(targets.items()): + if not requires_changelog(target, iter(files)): + continue + changelog_entries = [f for f in files if f.startswith(fragments_dir)] + if not changelog_entries: + errors.append( + f'Package "{target}" is missing a changelog entry for the following changes:\n' + + '\n'.join(f'- {f}' for f in files) + + 'Please run `ddev release changelog new` to add missing changelog entries.' + ) + continue + for entry_path in changelog_entries: + entry_parents, entry_fname = os.path.split(entry_path) + entry_pr_num, _, entry_fname_rest = entry_fname.partition(".") + if int(entry_pr_num) != pr_number: + correct_entry_path = os.path.join(entry_parents, f'{pr_number}.{entry_fname_rest}') + errors.append( + f'Please rename changelog entry file "{target}/{entry_path}" to "{correct_entry_path}". ' + + 'This way your changelog entry matches the PR number.' + ) + + return errors + + +def convert_to_messages(errors, on_ci): + for relative_path, line_number, message in errors: + yield ( + f'::error file={relative_path},line={line_number}::{message}' + if on_ci + else f'{relative_path}, line {line_number}: {message}' + ) + + +def changelog_impl(*, ref: str, diff_file: str, pr_file: str, private: bool, repo: str) -> None: import json on_ci = os.environ.get('GITHUB_ACTIONS') == 'true' @@ -168,17 +256,16 @@ def changelog_impl(*, ref: str, diff_file: str, pr_file: str, private: bool) -> print('No changelog entries required (changelog/no-changelog label found)') return - errors = get_changelog_errors(git_diff, changelog_entry_suffix(pr_number, pr_url), private=private) + errors = ( + get_core_repo_changelog_errors(git_diff, pr_number) + if repo == 'core' + else get_noncore_repo_changelog_errors(git_diff, changelog_entry_suffix(pr_number, pr_url), private=private) + ) if not errors: return - elif os.environ.get('GITHUB_ACTIONS') == 'true': - for relative_path, line_number, message in errors: - message = '%0A'.join(message.splitlines()) - print(f'::error file={relative_path},line={line_number}::{message}') - else: - for relative_path, line_number, message in errors: - print(f'{relative_path}, line {line_number}: {message}') - + for message in errors if repo == "core" else convert_to_messages(errors, on_ci): + formatted = '%0A'.join(message.splitlines()) if on_ci else message + print(formatted) sys.exit(1) @@ -188,6 +275,7 @@ def changelog_command(subparsers) -> None: parser.add_argument('--diff-file') parser.add_argument('--pr-file') parser.add_argument('--private', action='store_true') + parser.add_argument('--repo', default='core') parser.set_defaults(func=changelog_impl) diff --git a/ddev/tests/cli/release/test_changelog.py b/ddev/tests/cli/release/test_changelog.py index 05bf7ed25d601..e47f7d39eb8ed 100644 --- a/ddev/tests/cli/release/test_changelog.py +++ b/ddev/tests/cli/release/test_changelog.py @@ -1,6 +1,10 @@ # (C) Datadog, Inc. 2023-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +from functools import partial + +import pytest + from ddev.repo.core import Repository @@ -166,26 +170,84 @@ def test_no_pr(self, ddev, repository, helpers, network_replay, mocker): ) -class TestNew: - def test_start(self, ddev, repository, helpers, network_replay, mocker): - network_replay('release/changelog/fix_no_pr.yaml') +@pytest.fixture +def repo_with_towncrier(repository, helpers): + (repository.path / 'towncrier.toml').write_text( + helpers.dedent( + r''' + [tool.towncrier] + # If you change the values for directory or filename, make sure to look for them in the code as well. + directory = "changelog.d" + filename = "CHANGELOG.md" + start_string = "\n" + underlines = ["", "", ""] + template = "changelog_template.jinja" + title_format = "## {version} / {project_date}" + # We automatically link to PRs, but towncrier only has an issue template so we abuse that. + issue_format = "([#{issue}](https://github.com/DataDog/integrations-core/pull/{issue}))" + + # The order of entries matters! It controls the order in which changelog sections are displayed. + # https://towncrier.readthedocs.io/en/stable/configuration.html#use-a-toml-array-defined-order + [[tool.towncrier.type]] + directory="removed" + name = "Removed" + showcontent = true + + [[tool.towncrier.type]] + directory="changed" + name = "Changed" + showcontent = true + + [[tool.towncrier.type]] + directory="security" + name = "Security" + showcontent = true + + [[tool.towncrier.type]] + directory="deprecated" + name = "Deprecated" + showcontent = true + + [[tool.towncrier.type]] + directory="added" + name = "Added" + showcontent = true + + [[tool.towncrier.type]] + directory="fixed" + name = "Fixed" + showcontent = true + ''' + ) + ) + (repository.path / 'changelog_template.jinja').write_text( + helpers.dedent( + ''' + {% if sections[""] %} + {% for category, val in definitions.items() if category in sections[""] %} + ***{{ definitions[category]['name'] }}***: + + {% for text, values in sections[""][category].items() %} + * {{ text }} {{ values|join(', ') }} + {% endfor %} + + {% endfor %} + {% else %} + No significant changes. + + + {% endif %} + ''' + ) + ) + return repository - repo = Repository(repository.path.name, str(repository.path)) - changelog = repository.path / 'ddev' / 'CHANGELOG.md' - changelog.write_text( - helpers.dedent( - """ - # CHANGELOG - ddev - - ## Unreleased - - ## 3.3.0 / 2023-07-20 - - ***Added***: - """ - ) - ) +class TestNew: + @pytest.fixture + def fragments_dir(self, repo_with_towncrier, network_replay, mocker): + network_replay('release/changelog/fix_no_pr.yaml') + repo = Repository(repo_with_towncrier.path.name, str(repo_with_towncrier.path)) repo.git.capture('add', '.') repo.git.capture('commit', '-m', 'test') mocker.patch( @@ -197,498 +259,181 @@ def test_start(self, ddev, repository, helpers, network_replay, mocker): '', ], ) + return repo_with_towncrier.path / 'ddev' / 'changelog.d' + + def test_start(self, ddev, fragments_dir, helpers, mocker): mocker.patch('click.edit', return_value=None) + fragment_file = fragments_dir / '15476.added' result = ddev('release', 'changelog', 'new', 'added') assert result.exit_code == 0, result.output assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ + f''' + Created news fragment at {fragment_file} Added 1 changelog entry - """ + ''' ) + assert fragment_file.read_text() == "Foo" - assert changelog.read_text() == helpers.dedent( - """ - # CHANGELOG - ddev - - ## Unreleased - - ***Added***: - - * Foo ([#15476](https://github.com/DataDog/integrations-core/pull/15476)) - - ## 3.3.0 / 2023-07-20 - - ***Added***: - """ - ) - - def test_append(self, ddev, repository, helpers, network_replay, mocker): - network_replay('release/changelog/fix_no_pr.yaml') - - repo = Repository(repository.path.name, str(repository.path)) - - changelog = repository.path / 'ddev' / 'CHANGELOG.md' - changelog.write_text( - helpers.dedent( - """ - # CHANGELOG - ddev - - ## Unreleased - - ***Added***: - - * Over (#9000) - - ## 3.3.0 / 2023-07-20 - - ***Added***: - """ - ) - ) - repo.git.capture('add', '.') - repo.git.capture('commit', '-m', 'test') - mocker.patch( - 'ddev.utils.git.GitManager.capture', - side_effect=[ - '0000000000000000000000000000000000000000\nFoo', - 'M ddev/pyproject.toml', - '', - '', - ], - ) + def test_explicit_message(self, ddev, fragments_dir, helpers, mocker): mocker.patch('click.edit', return_value=None) + fragment_file = fragments_dir / '15476.added' - result = ddev('release', 'changelog', 'new', 'added') + result = ddev('release', 'changelog', 'new', 'added', '-m', 'Bar') assert result.exit_code == 0, result.output assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ + f''' + Created news fragment at {fragment_file} Added 1 changelog entry - """ + ''' ) + assert fragment_file.read_text() == "Bar" - assert changelog.read_text() == helpers.dedent( - """ - # CHANGELOG - ddev - - ## Unreleased - - ***Added***: - - * Over (#9000) - * Foo ([#15476](https://github.com/DataDog/integrations-core/pull/15476)) - - ## 3.3.0 / 2023-07-20 - - ***Added***: - """ - ) - - def test_before(self, ddev, repository, helpers, network_replay, mocker): - network_replay('release/changelog/fix_no_pr.yaml') - - repo = Repository(repository.path.name, str(repository.path)) - - changelog = repository.path / 'ddev' / 'CHANGELOG.md' - changelog.write_text( - helpers.dedent( - """ - # CHANGELOG - ddev - - ## Unreleased - - ***Added***: - - * Over (#9000) - - ## 3.3.0 / 2023-07-20 - - ***Added***: - """ - ) - ) - repo.git.capture('add', '.') - repo.git.capture('commit', '-m', 'test') - mocker.patch( - 'ddev.utils.git.GitManager.capture', - side_effect=[ - '0000000000000000000000000000000000000000\nFoo', - 'M ddev/pyproject.toml', - '', - '', - ], - ) + def test_prompt_for_entry_type(self, ddev, fragments_dir, helpers, mocker): mocker.patch('click.edit', return_value=None) + fragment_file = fragments_dir / '15476.added' - result = ddev('release', 'changelog', 'new', 'changed') + result = ddev('release', 'changelog', 'new', input='added') assert result.exit_code == 0, result.output assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ + f''' + Entry type? (removed, changed, security, deprecated, added, fixed): added + Created news fragment at {fragment_file} Added 1 changelog entry - """ + ''' ) + assert fragment_file.read_text() == "Foo" - assert changelog.read_text() == helpers.dedent( - """ - # CHANGELOG - ddev - - ## Unreleased - - ***Changed***: - - * Foo ([#15476](https://github.com/DataDog/integrations-core/pull/15476)) - - ***Added***: - - * Over (#9000) - - ## 3.3.0 / 2023-07-20 - - ***Added***: - """ - ) - - def test_after(self, ddev, repository, helpers, network_replay, mocker): - network_replay('release/changelog/fix_no_pr.yaml') - - repo = Repository(repository.path.name, str(repository.path)) - - changelog = repository.path / 'ddev' / 'CHANGELOG.md' - changelog.write_text( - helpers.dedent( - """ - # CHANGELOG - ddev - - ## Unreleased - - ***Added***: - - * Over (#9000) - - ## 3.3.0 / 2023-07-20 - - ***Added***: - """ - ) - ) - repo.git.capture('add', '.') - repo.git.capture('commit', '-m', 'test') + def test_edit_entry(self, ddev, fragments_dir, helpers, mocker): + message = 'Foo \n\n Bar' mocker.patch( - 'ddev.utils.git.GitManager.capture', - side_effect=[ - '0000000000000000000000000000000000000000\nFoo', - 'M ddev/pyproject.toml', - '', - '', - ], + 'click.edit', + return_value=message, ) - mocker.patch('click.edit', return_value=None) + fragment_file = fragments_dir / '15476.added' - result = ddev('release', 'changelog', 'new', 'fixed') + result = ddev('release', 'changelog', 'new', 'added') assert result.exit_code == 0, result.output assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ + f''' + Created news fragment at {fragment_file} Added 1 changelog entry - """ + ''' ) + assert fragment_file.read_text() == message - assert changelog.read_text() == helpers.dedent( - """ - # CHANGELOG - ddev - - ## Unreleased - - ***Added***: - - * Over (#9000) - - ***Fixed***: - - * Foo ([#15476](https://github.com/DataDog/integrations-core/pull/15476)) - ## 3.3.0 / 2023-07-20 +class TestBuild: + @pytest.fixture + def build_changelog(self, repo_with_towncrier): + ''' + We explicitly import and setup the command that only generates the changelog. - ***Added***: - """ - ) + This is needed because the "release make" command does too much. + ''' + from click.testing import CliRunner + from datadog_checks.dev.tooling.commands.release.changelog import changelog + from datadog_checks.dev.tooling.constants import set_root - def test_multiple(self, ddev, repository, helpers, network_replay, mocker): - network_replay('release/changelog/fix_no_pr.yaml') + set_root(repo_with_towncrier.path) - repo = Repository(repository.path.name, str(repository.path)) + return partial(CliRunner().invoke, changelog, catch_exceptions=False) - changelog1 = repository.path / 'ddev' / 'CHANGELOG.md' - changelog1.write_text( + @pytest.fixture + def setup_changelog_build(self, repo_with_towncrier, helpers): + changelog = repo_with_towncrier.path / 'ddev' / 'CHANGELOG.md' + changelog.write_text( helpers.dedent( - """ + ''' # CHANGELOG - ddev - ## Unreleased - - ***Added***: - - * Over (#9000) + ## 3.3.0 / 2023-07-20 ***Added***: - """ + ''' ) ) - changelog2 = repository.path / 'postgres' / 'CHANGELOG.md' - changelog2.write_text( - helpers.dedent( - """ - # CHANGELOG - postgres + fragments_dir = repo_with_towncrier.path / 'ddev' / 'changelog.d' + fragments_dir.mkdir(parents=True) + return changelog, fragments_dir - ## Unreleased + def test_build(self, setup_changelog_build, helpers, build_changelog): + ''' + This example checks several properties of a successful changelog: - ***Added***: + - Entries of the same entry type should be sorted. + - Entry types should be sorted. + - Multiline entries should preserve what user entered + ''' + changelog, fragments_dir = setup_changelog_build + (fragments_dir / '1.added').write_text("Foo") + (fragments_dir / '2.fixed').write_text("Bar") + (fragments_dir / '3.added').write_text('Foo\n\n Bar') - * Over (#9000) - - ## 3.3.0 / 2023-07-20 - - ***Added***: - """ - ) - ) - repo.git.capture('add', '.') - repo.git.capture('commit', '-m', 'test') - mocker.patch( - 'ddev.utils.git.GitManager.capture', - side_effect=[ - '0000000000000000000000000000000000000000\nFoo', - 'M ddev/pyproject.toml\nM postgres/pyproject.toml', - '', - '', - ], - ) - mocker.patch('click.edit', return_value=None) - - result = ddev('release', 'changelog', 'new', 'added') + result = build_changelog(args=["ddev", "3.4.0", "--date", "2023-10-11"]) assert result.exit_code == 0, result.output - assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ - Added 2 changelog entries - """ - ) - - assert changelog1.read_text() == helpers.dedent( - """ + assert changelog.read_text() == helpers.dedent( + ''' # CHANGELOG - ddev - ## Unreleased - - ***Added***: - - * Over (#9000) - * Foo ([#15476](https://github.com/DataDog/integrations-core/pull/15476)) + - ## 3.3.0 / 2023-07-20 - - ***Added***: - """ - ) - assert changelog2.read_text() == helpers.dedent( - """ - # CHANGELOG - postgres - - ## Unreleased + ## 3.4.0 / 2023-10-11 ***Added***: - * Over (#9000) - * Foo ([#15476](https://github.com/DataDog/integrations-core/pull/15476)) + * Foo ([#1](https://github.com/DataDog/integrations-core/pull/1)) + * Foo - ## 3.3.0 / 2023-07-20 + Bar ([#3](https://github.com/DataDog/integrations-core/pull/3)) - ***Added***: - """ - ) - - def test_explicit_message(self, ddev, repository, helpers, network_replay, mocker): - network_replay('release/changelog/fix_no_pr.yaml') - - repo = Repository(repository.path.name, str(repository.path)) - - changelog = repository.path / 'ddev' / 'CHANGELOG.md' - changelog.write_text( - helpers.dedent( - """ - # CHANGELOG - ddev - - ## Unreleased - - ## 3.3.0 / 2023-07-20 - - ***Added***: - """ - ) - ) - repo.git.capture('add', '.') - repo.git.capture('commit', '-m', 'test') - mocker.patch( - 'ddev.utils.git.GitManager.capture', - side_effect=[ - '0000000000000000000000000000000000000000\nFoo', - 'M ddev/pyproject.toml', - '', - '', - ], - ) - - result = ddev('release', 'changelog', 'new', 'added', '-m', 'Bar') - - assert result.exit_code == 0, result.output - assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ - Added 1 changelog entry - """ - ) - - assert changelog.read_text() == helpers.dedent( - """ - # CHANGELOG - ddev - - ## Unreleased - - ***Added***: + ***Fixed***: - * Bar ([#15476](https://github.com/DataDog/integrations-core/pull/15476)) + * Bar ([#2](https://github.com/DataDog/integrations-core/pull/2)) ## 3.3.0 / 2023-07-20 ***Added***: - """ + ''' ) - def test_prompt_for_entry_type(self, ddev, repository, helpers, network_replay, mocker): - network_replay('release/changelog/fix_no_pr.yaml') - - repo = Repository(repository.path.name, str(repository.path)) + def test_build_dry_run(self, setup_changelog_build, helpers, build_changelog): + changelog, fragments_dir = setup_changelog_build - changelog = repository.path / 'ddev' / 'CHANGELOG.md' - changelog.write_text( - helpers.dedent( - """ - # CHANGELOG - ddev + (fragments_dir / '1.added').write_text("Foo") - ## Unreleased - - ***Added***: - - * Over (#9000) - - ## 3.3.0 / 2023-07-20 - - ***Added***: - """ - ) - ) - repo.git.capture('add', '.') - repo.git.capture('commit', '-m', 'test') - mocker.patch( - 'ddev.utils.git.GitManager.capture', - side_effect=[ - '0000000000000000000000000000000000000000\nFoo', - 'M ddev/pyproject.toml', - '', - '', - ], - ) - mocker.patch('click.edit', return_value=None) - - result = ddev('release', 'changelog', 'new', input='added') + result = build_changelog(args=["ddev", "3.4.0", "--date", "2023-10-11", "--dry-run"]) assert result.exit_code == 0, result.output - assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ - Entry type? (Removed, Changed, Security, Deprecated, Added, Fixed): added - Added 1 changelog entry - """ - ) - - assert changelog.read_text() == helpers.dedent( - """ - # CHANGELOG - ddev - - ## Unreleased - - ***Added***: - - * Over (#9000) - * Foo ([#15476](https://github.com/DataDog/integrations-core/pull/15476)) - - ## 3.3.0 / 2023-07-20 - - ***Added***: - """ - ) - - def test_edit_entry(self, ddev, repository, helpers, network_replay, mocker): - network_replay('release/changelog/fix_no_pr.yaml') - - repo = Repository(repository.path.name, str(repository.path)) - - changelog = repository.path / 'ddev' / 'CHANGELOG.md' - changelog.write_text( + # The new changelog entry should appear in command output. + assert ( helpers.dedent( - """ - # CHANGELOG - ddev - - ## Unreleased - - ## 3.3.0 / 2023-07-20 + ''' + ## 3.4.0 / 2023-10-11 ***Added***: - """ - ) - ) - repo.git.capture('add', '.') - repo.git.capture('commit', '-m', 'test') - mocker.patch( - 'ddev.utils.git.GitManager.capture', - side_effect=[ - '0000000000000000000000000000000000000000\nFoo', - 'M ddev/pyproject.toml', - '', - '', - ], - ) - mocker.patch( - 'click.edit', - return_value='* Foo ([#15476](https://github.com/DataDog/integrations-core/pull/15476))\n\n Bar', - ) - - result = ddev('release', 'changelog', 'new', 'added') - assert result.exit_code == 0, result.output - assert helpers.remove_trailing_spaces(result.output) == helpers.dedent( - """ - Added 1 changelog entry - """ + * Foo ([#1](https://github.com/DataDog/integrations-core/pull/1)) + ''' + ) + in helpers.remove_trailing_spaces(result.output) ) - + # Make sure that we don't write anything to the changelog. assert changelog.read_text() == helpers.dedent( - """ + ''' # CHANGELOG - ddev - ## Unreleased - - ***Added***: - - * Foo ([#15476](https://github.com/DataDog/integrations-core/pull/15476)) - - Bar + ## 3.3.0 / 2023-07-20 ***Added***: - """ + ''' ) diff --git a/towncrier.toml b/towncrier.toml new file mode 100644 index 0000000000000..71cc4bae1ef38 --- /dev/null +++ b/towncrier.toml @@ -0,0 +1,42 @@ +[tool.towncrier] +# If you change the values for directory or filename, make sure to look for them in the code as well. +directory = "changelog.d" +filename = "CHANGELOG.md" +start_string = "\n" +underlines = ["", "", ""] +template = "changelog_template.jinja" +title_format = "## {version} / {project_date}" +# We automatically link to PRs, but towncrier only has an issue template so we abuse that. +issue_format = "([#{issue}](https://github.com/DataDog/integrations-core/pull/{issue}))" + +# The order of entries matters! It controls the order in which changelog sections are displayed. +# https://towncrier.readthedocs.io/en/stable/configuration.html#use-a-toml-array-defined-order +[[tool.towncrier.type]] +directory="removed" +name = "Removed" +showcontent = true + +[[tool.towncrier.type]] +directory="changed" +name = "Changed" +showcontent = true + +[[tool.towncrier.type]] +directory="security" +name = "Security" +showcontent = true + +[[tool.towncrier.type]] +directory="deprecated" +name = "Deprecated" +showcontent = true + +[[tool.towncrier.type]] +directory="added" +name = "Added" +showcontent = true + +[[tool.towncrier.type]] +directory="fixed" +name = "Fixed" +showcontent = true