Test emoji sequences from Unicode test files

.github/workflows/rust.yml

@@ -43,6 +43,6 @@ jobs:
       with:
         python-version: '3.12'
     - name: Regen
-      run: rm tests/NormalizationTest.txt && cd scripts && python3 unicode.py
+      run: rm tests/emoji-test.txt && cd scripts && python3 unicode.py
     - name: Diff
       run: git update-index --refresh && git diff-index --quiet HEAD --

scripts/unicode.py

@@ -71,15 +71,19 @@ class OffsetType(enum.IntEnum):
 BitPos = int
 
 
-def fetch_open(filename: str, local_prefix: str = ""):
+def fetch_open(filename: str, local_prefix: str = "", emoji: bool = False):
     """Opens `filename` and return its corresponding file object. If `filename` isn't on disk,
     fetches it from `https://www.unicode.org/Public/`. Exits with code 1 on failure.
     """
     basename = os.path.basename(filename)
     localname = os.path.join(local_prefix, basename)
     if not os.path.exists(localname):
+        if emoji:
+            prefix = f"emoji/{UNICODE_VERSION[:-2]}"
+        else:
+            prefix = f"{UNICODE_VERSION}/ucd"
         urllib.request.urlretrieve(
-            f"https://www.unicode.org/Public/{UNICODE_VERSION}/ucd/{filename}",
+            f"https://www.unicode.org/Public/{prefix}/{filename}",
             localname,
         )
     try:
@@ -2055,6 +2059,8 @@ def main(module_path: str):
 
     normalization_tests = load_normalization_tests()
 
+    fetch_open("emoji-test.txt", "../tests", emoji=True)
+
     print("------------------------")
     total_size = 0
     for i, table in enumerate(tables):