Generate changelogs from fragments using towncrier. (#15983)

.github/workflows/pr-quick-check.yml

@@ -57,3 +57,4 @@ jobs:
         --diff-file /tmp/diff
         --pr-file "$GITHUB_EVENT_PATH"
         ${{ github.event.pull_request.base.repo.private && '--private' || '' }}
+        --repo "${{ inputs.repo }}"

changelog_template.jinja

@@ -0,0 +1,13 @@
+{% if sections[""] %}
+{% for category, val in definitions.items() if category in sections[""] %}
+***{{ definitions[category]['name'] }}***:
+
+{% for text, values in sections[""][category].items() %}
+* {{ text }} {{ values|join(', ') }}
+{% endfor %}
+
+{% endfor %}
+{% else %}
+No significant changes.
+
+{% endif %}

datadog_checks_dev/CHANGELOG.md

@@ -1,6 +1,6 @@
 # CHANGELOG - Datadog Checks Dev
 
-## Unreleased
+<!-- towncrier release notes start -->
 
 ## 27.0.1 / 2023-10-26
 

datadog_checks_dev/changelog.d/15983.changed

@@ -0,0 +1,3 @@
+Generate changelogs from fragment files using towncrier.
+There are no changes to the ddev commands, only to their outputs.
+We are making this change to avoid merge conflicts in high-traffic packages where people used to have to modify one CHANGELOG.md file.

datadog_checks_dev/datadog_checks/dev/tooling/commands/release/changelog.py

@@ -2,52 +2,53 @@
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import os
+import sys
 from collections import namedtuple
-from datetime import date, datetime
-from io import StringIO
 
 import click
 from semver import VersionInfo
 
-from ....fs import stream_file_lines, write_file
-from ...constants import get_root
+from datadog_checks.dev.tooling.constants import get_root
+
 from ...utils import complete_testable_checks, get_valid_checks, get_version_string
-from ..console import CONTEXT_SETTINGS, abort, echo_info, echo_success, validate_check_arg
+from ..console import CONTEXT_SETTINGS, abort, echo_info, run_or_abort, validate_check_arg
 
 ChangelogEntry = namedtuple('ChangelogEntry', 'number, title, url, author, author_url, from_contributor')
 
 
+def towncrier(target_dir, cmd, *cmd_args):
+    '''
+    Run towncrier command with its arguments in target_dir.
+    '''
+    tc_res = run_or_abort(
+        [
+            sys.executable,
+            "-m",
+            "towncrier",
+            cmd,
+            "--config",
+            os.path.join(get_root(), "towncrier.toml"),
+            "--dir",
+            target_dir,
+            *cmd_args,
+        ],
+        capture='both',
+    )
+    echo_info(tc_res.stdout.rstrip())
+    return tc_res
+
+
 @click.command(context_settings=CONTEXT_SETTINGS, short_help='Update the changelog for a check')
 @click.argument('check', shell_complete=complete_testable_checks, callback=validate_check_arg)
 @click.argument('version')
 @click.argument('old_version', required=False)
-@click.option('--end')
-@click.option('--initial', is_flag=True)
-@click.option('--organization', '-r', default='DataDog')
 @click.option('--quiet', '-q', is_flag=True)
 @click.option('--dry-run', '-n', is_flag=True)
-@click.option('--output-file', '-o', default='CHANGELOG.md', show_default=True)
 @click.option('--tag-pattern', default=None, hidden=True)
 @click.option('--tag-prefix', '-tp', default='v', show_default=True)
 @click.option('--no-semver', '-ns', default=False, is_flag=True)
-@click.option('--exclude-branch', default=None, help="Exclude changes comming from a specific branch")
-@click.pass_context
-def changelog(
-    ctx,
-    check,
-    version,
-    old_version,
-    end,
-    initial,
-    quiet,
-    dry_run,
-    output_file,
-    tag_pattern,
-    tag_prefix,
-    no_semver,
-    organization,
-    exclude_branch,
-):
+@click.option('--date', default=None)
+def changelog(check, version, old_version, quiet, dry_run, tag_pattern, tag_prefix, no_semver, date):
     """Perform the operations needed to update the changelog.
 
     This method is supposed to be used by other tasks and not directly.
@@ -71,69 +72,9 @@ def changelog(
     if not quiet:
         echo_info(f'Current version of check {check}: {cur_version}, bumping to: {version}')
 
-    # read the old contents
-    if check:
-        changelog_path = os.path.join(get_root(), check, output_file)
-    else:
-        changelog_path = os.path.join(get_root(), output_file)
-    old = list(stream_file_lines(changelog_path))
-
-    if initial:
-        # For initial releases, just keep the ddev generated CHANGELOG but update the date to today
-        for idx, line in enumerate(old):
-            if line.startswith("## 1.0.0"):
-                old[idx] = f"## 1.0.0 / {date.today()}\n"
-                break
-        write_result(dry_run, changelog_path, ''.join(old), num_changes=1)
-        return
-
-    # find the first header below the Unreleased section
-    header_index = 2
-    for index in range(2, len(old)):
-        if old[index].startswith("##") and "## Unreleased" not in old[index]:
-            header_index = index
-            break
-
-    # get text from the unreleased section
-    if header_index == 4:
-        abort('There are no changes for this integration')
-
-    changelogs = old[4:header_index]
-    num_changelogs = 0
-    for line in changelogs:
-        if line.startswith('* '):
-            num_changelogs += 1
-
-    # the header contains version and date
-    header = f"## {version} / {datetime.utcnow().strftime('%Y-%m-%d')}\n"
-
-    # store the new changelog in memory
-    new_entry = StringIO()
-    new_entry.write(header)
-    new_entry.write('\n')
-
-    # write the new changelog in memory
-    changelog_buffer = StringIO()
-
-    # preserve the title and unreleased section
-    changelog_buffer.write(''.join(old[:4]))
-
-    # prepend the new changelog to the old contents
-    # make the command idempotent
-    if header not in old:
-        changelog_buffer.write(new_entry.getvalue())
-
-    # append the rest of the old changelog
-    changelog_buffer.write(''.join(old[4:]))
-
-    write_result(dry_run, changelog_path, changelog_buffer.getvalue(), num_changelogs)
-
-
-def write_result(dry_run, changelog_path, final_output, num_changes):
-    # print on the standard out in case of a dry run
+    build_args = ["--yes", "--version", version]
     if dry_run:
-        echo_info(final_output)
-    else:
-        # overwrite the old changelog
-        write_file(changelog_path, final_output)
-        echo_success(f"Successfully generated {num_changes} change{'s' if num_changes > 1 else ''}")
+        build_args.append("--draft")
+    if date:
+        build_args.extend(["--date", date])
+    towncrier(os.path.join(get_root(), check), "build", *build_args)

datadog_checks_dev/datadog_checks/dev/tooling/commands/release/make.py

@@ -132,10 +132,10 @@ def make(ctx, checks, version, end, initial_release, skip_sign, sign_only, exclu
             if check == 'ddev':
                 cur_version = get_version_string(check)
                 _, changelog_types = ctx.invoke(
-                    changes, check=check, tag_pattern='ddev-v.+', tag_prefix='ddev-v', end=end, dry_run=True
+                    changes, check=check, tag_pattern='ddev-v.+', tag_prefix='ddev-v', dry_run=True
                 )
             else:
-                cur_version, changelog_types = ctx.invoke(changes, check=check, end=end, dry_run=True)
+                cur_version, changelog_types = ctx.invoke(changes, check=check, dry_run=True)
 
             echo_debug(f'Current version: {cur_version}. Changes: {changelog_types}')
             if not changelog_types:
@@ -149,18 +149,15 @@ def make(ctx, checks, version, end, initial_release, skip_sign, sign_only, exclu
 
         # update the CHANGELOG
         echo_waiting('Updating the changelog... ', nl=False)
-        # TODO: Avoid double GitHub API calls when bumping all checks at once
         ctx.invoke(
             changelog,
             check=check,
             version=version,
             old_version=None if check == 'ddev' else cur_version,
-            end=end,
-            initial=initial_release,
-            tag_pattern='ddev-v.+' if check == 'ddev' else None,
-            tag_prefix='ddev-v' if check == 'ddev' else 'v',
             quiet=True,
             dry_run=False,
+            tag_pattern='ddev-v.+' if check == 'ddev' else None,
+            tag_prefix='ddev-v' if check == 'ddev' else 'v',
         )
         echo_success('success!')
 

datadog_checks_dev/datadog_checks/dev/tooling/commands/release/show/changes.py

@@ -5,22 +5,18 @@
 
 import click
 
-from .....fs import stream_file_lines
 from ....constants import get_root
 from ....utils import complete_valid_checks, get_valid_checks, get_version_string
 from ...console import (
     CONTEXT_SETTINGS,
     abort,
-    echo_failure,
     echo_info,
-    echo_success,
     validate_check_arg,
 )
 
 
 @click.command(context_settings=CONTEXT_SETTINGS, short_help='Show all the pending PRs for a given check.')
 @click.argument('check', shell_complete=complete_valid_checks, callback=validate_check_arg)
-@click.option('--organization', '-r', default='DataDog', help="The Github organization the repository belongs to")
 @click.option(
     '--tag-pattern',
     default=None,
@@ -33,10 +29,7 @@
 @click.option(
     '--since', default=None, help="The git ref to use instead of auto-detecting the tag to view changes since"
 )
-@click.option('--end')
-@click.option('--exclude-branch', default=None, help="Exclude changes comming from a specific branch")
-@click.pass_context
-def changes(ctx, check, tag_pattern, tag_prefix, dry_run, organization, since, end, exclude_branch):
+def changes(check, tag_pattern, tag_prefix, dry_run, since):
     """Show all the pending PRs for a given check."""
     if not dry_run and check and check not in get_valid_checks():
         abort(f'Check `{check}` is not an Agent-based Integration')
@@ -49,31 +42,18 @@ def changes(ctx, check, tag_pattern, tag_prefix, dry_run, organization, since, e
             'following SemVer and matches the provided tag_prefix and/or tag_pattern.'
         )
 
-    if check:
-        changelog_path = os.path.join(get_root(), check, 'CHANGELOG.md')
-    else:
-        changelog_path = os.path.join(get_root(), 'CHANGELOG.md')
-    log = list(stream_file_lines(changelog_path))
-
-    header_index = 2
-    for index in range(2, len(log)):
-        if log[index].startswith("##") and "## Unreleased" not in log[index]:
-            header_index = index
-            break
-
-    if header_index == 4:
-        echo_failure('There are no changes for this integration')
-        return cur_version, []
-
-    unreleased = log[4:header_index]
-    applicable_changelog_types = []
-
-    for line in unreleased:
-        if line.startswith('***'):
-            applicable_changelog_types.append(line[3:-5])
-            echo_success(line)
-
-        elif line.strip():
-            echo_info(line)
+    applicable_changelog_types = set()
+    fragment_dir = os.path.join(get_root(), check, 'changelog.d')
+    if not os.path.exists(fragment_dir):
+        echo_info('No changes for this check.')
+        return cur_version, applicable_changelog_types
+    changes_to_report = []
+    for fname in os.listdir(fragment_dir):
+        applicable_changelog_types.add(fname.split(".")[1])
+        changes_to_report.append(f'{os.path.join(check, "changelog.d", fname)}:')
+        fpath = os.path.join(fragment_dir, fname)
+        with open(fpath, mode='r') as fh:
+            changes_to_report.append(fh.read() + '\n')
+    echo_info('\n'.join(changes_to_report).rstrip())
 
     return cur_version, applicable_changelog_types

datadog_checks_dev/datadog_checks/dev/tooling/constants.py

@@ -37,12 +37,12 @@
 }
 
 VERSION_BUMP = {
-    'Added': semver.bump_minor,
-    'Changed': semver.bump_major,
-    'Deprecated': semver.bump_minor,
-    'Fixed': semver.bump_patch,
-    'Removed': semver.bump_major,
-    'Security': semver.bump_minor,
+    'added': semver.bump_minor,
+    'changed': semver.bump_major,
+    'deprecated': semver.bump_minor,
+    'fixed': semver.bump_patch,
+    'removed': semver.bump_major,
+    'security': semver.bump_minor,
     'major': semver.bump_major,
     'minor': semver.bump_minor,
     'patch': semver.bump_patch,

datadog_checks_dev/pyproject.toml

@@ -85,6 +85,8 @@ cli = [
     "tomli>=1.1.0",
     "tomli-w>=1.0.0",
     "tox>=3.12.1, <4.0.0",
+    # Move towncrier dependency to ddev once we migrate changelog-related commands.
+    "towncrier==23.10.0",
     "twine>=1.11.0",
     "virtualenv<20.22.0",
     # TODO: Remove once every check has a pyproject.toml

ddev/CHANGELOG.md

@@ -1,6 +1,6 @@
 # CHANGELOG - ddev
 
-## Unreleased
+<!-- towncrier release notes start -->
 
 ## 5.3.0 / 2023-10-26
 

ddev/changelog.d/15983.changed

@@ -0,0 +1,3 @@
+Generate changelogs from fragment files using towncrier.
+There are no changes to the ddev commands, only to their outputs.
+We are making this change to avoid merge conflicts in high-traffic packages where people used to have to modify one CHANGELOG.md file.

ddev/src/ddev/cli/release/changelog/fix.py

@@ -11,14 +11,20 @@
     from ddev.cli.application import Application
 
 
-@click.command(short_help='Fix changelog entries')
+@click.command
 @click.pass_obj
 def fix(app: Application):
     """
+    Fix changelog entries.
+
+    This command is only needed if you are manually writing to the changelog.
+    For instance for marketplace and extras integrations.
+    Don't use this in integrations-core because the changelogs there are generated automatically.
+
     The first line of every new changelog entry must include the PR number in which the change
     occurred. This command will apply this suffix to manually added entries if it is missing.
     """
-    from ddev.utils.scripts.check_pr import changelog_entry_suffix, get_changelog_errors
+    from ddev.utils.scripts.check_pr import changelog_entry_suffix, get_noncore_repo_changelog_errors
 
     latest_commit = app.repo.git.latest_commit
     pr = app.github.get_pull_request(latest_commit.sha)
@@ -37,7 +43,7 @@ def fix(app: Application):
 
     expected_suffix = changelog_entry_suffix(pr_number, pr_url)
     fixed = 0
-    for path, line_number, _ in get_changelog_errors(git_diff, expected_suffix):
+    for path, line_number, _ in get_noncore_repo_changelog_errors(git_diff, expected_suffix):
         if line_number == 1:
             continue