Windows fixes

- Ignore errors when executing `shutil.rmtree()`, because it seems like it's common to fail when deleting git repos on Windows, and since these are temp files, we don't really care that much there's garbage left. Any good OS should clean the temp folders automatically.
- Always find Jinja templates in `PosixPath` mode.
- Ignore `OSError` when trying to enter a possibly git root directory. This is yielded by Windows when the path is a URL and we don't really care about it.
- Fix some tests with non-windows hardcoded stuff.
- Fix a test that was using a Bash script. Modified to be Python, which should work fine cross-system.
- Remove external python dependencies (yaml, plumbum) from test task/migration files. These are available on Linux because it gets the python env from the venv, but on Windows, it uses the main python interpreter and breaks. After all, that's not very important here.
- Do not modify EOL in CI.
- Use python executable in tests instead of python3.
- Update pre-commit versions to include pre-commit/pre-commit-hooks#509.
- Disable autorebasing.
- Disable pre-commit on CI on Windows.
- Require python 3.8 on Windows, where `tempfile` supports autoremoving directories with read-only files.

.github/workflows/automerge.yml

@@ -26,7 +26,7 @@ jobs:
           MERGE_LABELS: "automerge,!work in progress"
           MERGE_METHOD: "rebase"
           # Disable autorebasing PRs because they cannot retrigger checks
-          UPDATE_LABELS: ""
+          UPDATE_LABELS: "DISABLED"
           UPDATE_METHOD: "rebase"
           # Retry for 20m
           MERGE_RETRIES: 120

.github/workflows/ci.yml

@@ -6,19 +6,31 @@ on:
     branches: [master]
   release:
     types: [created]
+env:
+  LANG: "en_US.utf-8"
+  LC_ALL: "en_US.utf-8"
+  PYTHONIOENCODING: "UTF-8"
 
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
+        require-pre-commit: [true]
         os:
           - macos-latest
           - ubuntu-latest
-          - windows-latest
         python-version: [3.6, 3.7, 3.8]
+        include:
+          - os: windows-latest
+            python-version: 3.8
+            # FIXME Make pre-commit pass reliably on Windows, and remove this
+            require-pre-commit: false
     runs-on: ${{ matrix.os }}
     steps:
+      - run: git config --global user.name copier-ci
+      - run: git config --global user.email copier@copier
+      - run: git config --global core.autocrlf input
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v1
@@ -41,17 +53,14 @@ jobs:
           key:
             local-${{ env.PY }}|${{ hashFiles('pyproject.toml') }}|${{
             hashFiles('poetry.lock') }}|${{ hashFiles('.pre-commit-config.yaml') }}
-      - run: git config --global user.name copier-ci
-        shell: bash
-      - run: git config --global user.email copier@copier
-        shell: bash
       - run: python -m pip install poetry poetry-dynamic-versioning
         shell: bash
       - run: poetry install
         shell: bash
       - name: Run pre-commit
         shell: bash
         run: poetry run pre-commit run --all-files --color=always
+        continue-on-error: ${{ matrix.require-pre-commit }}
       - name: Run mypy
         shell: bash
         run: poetry run mypy --ignore-missing-imports .

.pre-commit-config.yaml

@@ -34,7 +34,7 @@ repos:
 
   # isorting our imports
   - repo: https://github.com/timothycrosley/isort
-    rev: 4.3.21
+    rev: 5.2.2
     hooks:
       - id: isort
         additional_dependencies:
@@ -52,7 +52,7 @@ repos:
 
   # miscellaneous hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.1.0
+    rev: v3.2.0
     hooks:
       - id: check-added-large-files
       - id: check-ast

README.md

@@ -20,6 +20,7 @@ A library for rendering project templates.
 
 ## Installation
 
+1. Install Python 3.6.1 or newer (3.8 or newer if you're on Windows).
 1. Install Git 2.24 or newer.
 1. To use as a CLI app: `pipx install copier`
 1. To use as a library: `pip install copier`

copier/main.py

@@ -148,7 +148,7 @@ def copy(
         raise
     finally:
         if is_update:
-            shutil.rmtree(conf.src_path)
+            shutil.rmtree(conf.src_path, ignore_errors=True)
 
 
 def copy_local(conf: ConfigData) -> None:

copier/tools.py

@@ -156,10 +156,8 @@ def __init__(self, conf: ConfigData) -> None:
         )
 
     def __call__(self, fullpath: StrOrPath) -> str:
-        relpath = (
-            str(fullpath).replace(str(self.conf.src_path), "", 1).lstrip(os.path.sep)
-        )
-        tmpl = self.env.get_template(relpath)
+        relpath = Path(fullpath).relative_to(self.conf.src_path).as_posix()
+        tmpl = self.env.get_template(str(relpath))
         return tmpl.render(**self.data)
 
     def string(self, string: StrOrPath) -> str:

copier/vcs.py

@@ -1,7 +1,6 @@
 """Utilities related to VCS."""
 
 import re
-import shutil
 import tempfile
 from pathlib import Path
 
@@ -27,8 +26,8 @@ def is_git_repo_root(path: Path) -> bool:
     """Indicate if a given path is a git repo root directory."""
     try:
         with local.cwd(path / ".git"):
-            return bool(git("rev-parse", "--is-inside-git-dir") == "true\n")
-    except (FileNotFoundError, NotADirectoryError):
+            return bool(git("rev-parse", "--is-inside-git-dir").strip() == "true")
+    except OSError:
         return False
 
 
@@ -74,7 +73,6 @@ def checkout_latest_tag(local_repo: StrOrPath) -> str:
 
 def clone(url: str, ref: str = "HEAD") -> str:
     location = tempfile.mkdtemp(prefix=f"{__name__}.clone.")
-    shutil.rmtree(location)  # Path must not exist
     git("clone", "--no-checkout", url, location)
     with local.cwd(location):
         git("checkout", ref)

tests/demo_migrations/copier.yaml

@@ -1,11 +1,11 @@
 _exclude:
-  - tasks.sh
+  - tasks.py
   - migrations.py
   - .git
 
 _tasks:
-  - "[[ _copier_conf.src_path / 'tasks.sh' ]] 1"
-  - ["[[ _copier_conf.src_path / 'tasks.sh' ]]", 2]
+  - "python [[ _copier_conf.src_path / 'tasks.py' ]] 1"
+  - [python, "[[ _copier_conf.src_path / 'tasks.py' ]]", 2]
 
 _migrations:
   # This migration is never executed because it's the 1st version copied, and

tests/demo_migrations/migrations.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python
 import json
 import os
 import sys

tests/demo_migrations/tasks.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+import os
+import os.path
+import sys
+from contextlib import suppress
+from subprocess import check_call
+
+with open("created-with-tasks.txt", "a", newline="\n") as cwt:
+    cwt.write(" ".join([os.environ["STAGE"]] + sys.argv[1:]) + "\n")
+    check_call(["git", "init"])
+    with suppress(FileNotFoundError):
+        os.unlink("delete-in-tasks.txt")

tests/test_config.py

@@ -1,4 +1,3 @@
-import re
 from pathlib import Path
 
 import pytest
@@ -45,11 +44,12 @@ def test_read_data(tmp_path, template):
 
 
 def test_invalid_yaml(capsys):
-    conf_path = Path("tests/demo_invalid/copier.yml")
+    conf_path = Path("tests", "demo_invalid", "copier.yml")
     with pytest.raises(InvalidConfigFileError):
         load_yaml_data(conf_path)
     out, _ = capsys.readouterr()
-    assert re.search(r"INVALID.*tests/demo_invalid/copier\.yml", out)
+    assert "INVALID CONFIG FILE" in out
+    assert str(conf_path) in out
 
 
 @pytest.mark.parametrize(

tests/test_migrations.py

@@ -1,8 +1,8 @@
+import json
 from glob import glob
 from pathlib import Path
 from shutil import copytree
 
-import py
 import yaml
 from plumbum import local
 from plumbum.cmd import git
@@ -14,10 +14,10 @@
 SRC = Path(f"{PROJECT_TEMPLATE}_migrations").absolute()
 
 
-def test_migrations_and_tasks(tmpdir: py.path.local):
+def test_migrations_and_tasks(tmp_path: Path):
     """Check migrations and tasks are run properly."""
     # Convert demo_migrations in a git repository with 2 versions
-    git_src, tmp_path = tmpdir / "src", tmpdir / "tmp_path"
+    git_src, dst = tmp_path / "src", tmp_path / "tmp_path"
     copytree(SRC, git_src)
     with local.cwd(git_src):
         git("init")
@@ -29,50 +29,50 @@ def test_migrations_and_tasks(tmpdir: py.path.local):
         git("commit", "--allow-empty", "-m2")
         git("tag", "v2.0")
     # Copy it in v1
-    copy(src_path=str(git_src), dst_path=str(tmp_path), vcs_ref="v1.0.0")
+    copy(src_path=str(git_src), dst_path=str(dst), vcs_ref="v1.0.0")
     # Check copy was OK
-    assert (tmp_path / "created-with-tasks.txt").read() == "task 1\ntask 2\n"
-    assert not (tmp_path / "delete-in-tasks.txt").exists()
-    assert (tmp_path / "delete-in-migration-v2.txt").isfile()
-    assert not (tmp_path / "migrations.py").exists()
-    assert not (tmp_path / "tasks.sh").exists()
-    assert not glob(str(tmp_path / "*-before.txt"))
-    assert not glob(str(tmp_path / "*-after.txt"))
-    answers = yaml.safe_load((tmp_path / ".copier-answers.yml").read())
+    assert (dst / "created-with-tasks.txt").read_text() == "task 1\ntask 2\n"
+    assert not (dst / "delete-in-tasks.txt").exists()
+    assert (dst / "delete-in-migration-v2.txt").is_file()
+    assert not (dst / "migrations.py").exists()
+    assert not (dst / "tasks.py").exists()
+    assert not glob(str(dst / "*-before.txt"))
+    assert not glob(str(dst / "*-after.txt"))
+    answers = yaml.safe_load((dst / ".copier-answers.yml").read_text())
     assert answers == {"_commit": "v1.0.0", "_src_path": str(git_src)}
     # Save changes in downstream repo
-    with local.cwd(tmp_path):
+    with local.cwd(dst):
         git("add", ".")
         git("config", "user.name", "Copier Test")
         git("config", "user.email", "test@copier")
         git("commit", "-m1")
     # Update it to v2
-    copy(dst_path=str(tmp_path), force=True)
+    copy(dst_path=str(dst), force=True)
     # Check update was OK
-    assert (tmp_path / "created-with-tasks.txt").read() == "task 1\ntask 2\n" * 2
-    assert not (tmp_path / "delete-in-tasks.txt").exists()
-    assert not (tmp_path / "delete-in-migration-v2.txt").exists()
-    assert not (tmp_path / "migrations.py").exists()
-    assert not (tmp_path / "tasks.sh").exists()
-    assert (tmp_path / "v1.0.0-v2-v2.0-before.json").isfile()
-    assert (tmp_path / "v1.0.0-v2-v2.0-after.json").isfile()
-    answers = yaml.safe_load((tmp_path / ".copier-answers.yml").read())
+    assert (dst / "created-with-tasks.txt").read_text() == "task 1\ntask 2\n" * 2
+    assert not (dst / "delete-in-tasks.txt").exists()
+    assert not (dst / "delete-in-migration-v2.txt").exists()
+    assert not (dst / "migrations.py").exists()
+    assert not (dst / "tasks.py").exists()
+    assert (dst / "v1.0.0-v2-v2.0-before.json").is_file()
+    assert (dst / "v1.0.0-v2-v2.0-after.json").is_file()
+    answers = yaml.safe_load((dst / ".copier-answers.yml").read_text())
     assert answers == {"_commit": "v2.0", "_src_path": str(git_src)}
 
 
 def test_pre_migration_modifies_answers(tmp_path_factory):
     """Test support for answers modifications in pre-migrations."""
     template = tmp_path_factory.mktemp("template")
     subproject = tmp_path_factory.mktemp("subproject")
-    # v1 of template asks for a favourite song and writes it to songs.yaml
+    # v1 of template asks for a favourite song and writes it to songs.json
     with local.cwd(template):
         build_file_tree(
             {
-                "[[ _copier_conf.answers_file ]].tmpl": "[[ _copier_answers|to_nice_yaml ]]",
+                "[[ _copier_conf.answers_file ]].tmpl": "[[ _copier_answers|tojson ]]",
                 "copier.yml": """\
                     best_song: la vie en rose
                     """,
-                "songs.yaml.tmpl": "- [[ best_song ]]",
+                "songs.json.tmpl": "[ [[ best_song|tojson ]] ]",
             }
         )
         git("init")
@@ -82,10 +82,10 @@ def test_pre_migration_modifies_answers(tmp_path_factory):
     # User copies v1 template into subproject
     with local.cwd(subproject):
         copy(src_path=str(template), force=True)
-        answers = yaml.safe_load(Path(".copier-answers.yml").read_text())
+        answers = json.loads(Path(".copier-answers.yml").read_text())
         assert answers["_commit"] == "v1"
         assert answers["best_song"] == "la vie en rose"
-        assert yaml.safe_load(Path("songs.yaml").read_text()) == ["la vie en rose"]
+        assert json.loads(Path("songs.json").read_text()) == ["la vie en rose"]
         git("init")
         git("add", ".")
         git("commit", "-m1")
@@ -100,18 +100,18 @@ def test_pre_migration_modifies_answers(tmp_path_factory):
                     _migrations:
                       - version: v2
                         before:
-                          - - python3
+                          - - python
                             - -c
                             - |
-                                import sys, yaml, pathlib
+                                import sys, json, pathlib
                                 answers_path = pathlib.Path(*sys.argv[1:])
-                                answers = yaml.safe_load(answers_path.read_text())
+                                answers = json.loads(answers_path.read_text())
                                 answers["best_song_list"] = [answers.pop("best_song")]
-                                answers_path.write_text(yaml.safe_dump(answers))
+                                answers_path.write_text(json.dumps(answers))
                             - "[[ _copier_conf.dst_path ]]"
                             - "[[ _copier_conf.answers_file ]]"
                     """,
-                "songs.yaml.tmpl": "[[ best_song_list|to_nice_yaml ]]",
+                "songs.json.tmpl": "[[ best_song_list|tojson ]]",
             }
         )
         git("add", ".")
@@ -120,8 +120,8 @@ def test_pre_migration_modifies_answers(tmp_path_factory):
     # User updates subproject to v2 template
     with local.cwd(subproject):
         copy(force=True)
-        answers = yaml.safe_load(Path(".copier-answers.yml").read_text())
+        answers = json.loads(Path(".copier-answers.yml").read_text())
         assert answers["_commit"] == "v2"
         assert "best_song" not in answers
         assert answers["best_song_list"] == ["la vie en rose"]
-        assert yaml.safe_load(Path("songs.yaml").read_text()) == ["la vie en rose"]
+        assert json.loads(Path("songs.json").read_text()) == ["la vie en rose"]

tests/test_output.py

@@ -8,15 +8,15 @@ def test_output(capsys, tmp_path):
     out, _ = capsys.readouterr()
     assert re.search(r"create[^\s]*  config\.py", out)
     assert re.search(r"create[^\s]*  pyproject\.toml", out)
-    assert re.search(r"create[^\s]*  doc/images/nslogo\.gif", out)
+    assert re.search(r"create[^\s]*  doc[/\\]images[/\\]nslogo\.gif", out)
 
 
 def test_output_pretend(capsys, tmp_path):
     render(tmp_path, quiet=False, pretend=True)
     out, _ = capsys.readouterr()
     assert re.search(r"create[^\s]*  config\.py", out)
     assert re.search(r"create[^\s]*  pyproject\.toml", out)
-    assert re.search(r"create[^\s]*  doc/images/nslogo\.gif", out)
+    assert re.search(r"create[^\s]*  doc[/\\]images[/\\]nslogo\.gif", out)
 
 
 def test_output_force(capsys, tmp_path):
@@ -27,7 +27,7 @@ def test_output_force(capsys, tmp_path):
     assert re.search(r"conflict[^\s]*  config\.py", out)
     assert re.search(r"force[^\s]*  config\.py", out)
     assert re.search(r"identical[^\s]*  pyproject\.toml", out)
-    assert re.search(r"identical[^\s]*  doc/images/nslogo\.gif", out)
+    assert re.search(r"identical[^\s]*  doc[/\\]images[/\\]nslogo\.gif", out)
 
 
 def test_output_skip(capsys, tmp_path):
@@ -38,7 +38,7 @@ def test_output_skip(capsys, tmp_path):
     assert re.search(r"conflict[^\s]*  config\.py", out)
     assert re.search(r"skip[^\s]*  config\.py", out)
     assert re.search(r"identical[^\s]*  pyproject\.toml", out)
-    assert re.search(r"identical[^\s]*  doc/images/nslogo\.gif", out)
+    assert re.search(r"identical[^\s]*  doc[/\\]images[/\\]nslogo\.gif", out)
 
 
 def test_output_quiet(capsys, tmp_path):

tests/test_updatediff.py

@@ -248,6 +248,7 @@ def test_commit_hooks_respected(tmp_path: Path):
                 """
             )
         )
+        git("add", ".")
         git("commit", "-am", "subproject is evolved")
         # Subproject re-updates just to change some values
         copy(data={"what": "study"}, force=True)

tests/test_vcs.py

@@ -45,4 +45,4 @@ def test_clone():
     tmp = vcs.clone("https://github.com/copier-org/copier.git")
     assert tmp
     assert exists(join(tmp, "README.md"))
-    shutil.rmtree(tmp)
+    shutil.rmtree(tmp, ignore_errors=True)