Run CI tests on Windows, Macos and Linux (#444)

* Run unit_tests on windows and macos * Setup venv since macos really wants venv * windows venv path is different for some reason * Enable `ld_dependencies` and `create_asm_dependencies` on the basic_app yaml Also reduce some paths and change compiler to KMC * Fix silly paths on CI * Fix?? * CI FIX??? * ciiiiiiiii * idk * Stuff is hardcoded and I don't care enough to figure out how to fix it * maybe? * I'm dumb * Ensure Windows uses posix paths on dependency files and the linker script * changelog * fix? * Perform LF normalization on Windows * ??? * More endline normalization? * more changelog
ethteck · Feb 28, 2025 · a17a6dc · a17a6dc
1 parent 27b503d
commit a17a6dc
Show file tree

Hide file tree

Showing 20 changed files with 134 additions and 64 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,2 @@
+# Force LF on everyfile
+* text eol=lf
diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml
@@ -22,6 +22,7 @@ jobs:
         pip install mypy
         pip install -r requirements.txt
         pip install types-PyYAML
+        pip install -e .
 
     - name: mypy splat lib
       run: mypy --show-column-numbers --hide-error-context src/splat

diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -6,22 +6,55 @@ on:
 
 jobs:
   unit_tests:
-    runs-on: ubuntu-latest
+    name: unit_tests on ${{ matrix.os.name }}
+    runs-on: ${{ matrix.os.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [
+          {
+            name: linux,
+            runner: ubuntu-latest,
+            python_venv: .venv/bin/python3,
+          },
+          {
+            name: macos,
+            runner: macos-latest,
+            python_venv: .venv/bin/python3,
+          },
+          {
+            name: windows,
+            runner: windows-latest,
+            python_venv: .venv/Scripts/python3,
+          },
+        ]
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Install dependencies
-        run: sudo apt-get install -y build-essential make binutils-mips-linux-gnu python3 python3-pip wget
+        if: matrix.os.name == 'linux'
+        run: sudo apt-get install -y build-essential make binutils-mips-linux-gnu python3 python3-pip python3-venv wget
+
+      - name: Setup Python venv
+        run: |
+          python3 -m venv .venv
 
       - name: Install Python dependencies
-        run: python3 -m pip install -U -r requirements.txt
+        run: |
+          ${{ matrix.os.python_venv }} -m pip install -U -r requirements.txt
+          ${{ matrix.os.python_venv }} -m pip install -e .
 
-      - name: Build basic_app
+      - name: Build `basic_app` on ${{ matrix.os.name }}
+        if: matrix.os.name == 'linux'
+        # Linux CI checks if any of the test C code has changed without updating the generated binary
         run: |
           make -C test/basic_app clean
           make -C test/basic_app download_kmc
           make -C test/basic_app all
+          git diff --exit-code test/basic_app/build/basic_app.bin
 
       - name: Run the test
-        run: python3 test.py
+        run: |
+          ${{ matrix.os.python_venv }} test.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # splat Release Notes
 
+### 0.32.4
+
+- Fix splat on Windows not using forward slashes on generated paths.
+- Setup CI to be run on Windows and Macos too.
+- Fix generated `incbin`s segments not being compatible with SN64 compiler.
+
 ### 0.32.3
 
 - Fix "unrecognized YAML option" error if disassemble_all is provided via CLI and as a YAML option.

diff --git a/README.md b/README.md
@@ -21,7 +21,7 @@ The brackets corresponds to the optional dependencies to install while installin
 If you use a `requirements.txt` file in your repository, then you can add this library with the following line:
 
 ```txt
-splat64[mips]>=0.32.3,<1.0.0
+splat64[mips]>=0.32.4,<1.0.0
 ```
 
 ### Optional dependencies

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "splat64"
 # Should be synced with src/splat/__init__.py
-version = "0.32.3"
+version = "0.32.4"
 description = "A binary splitting tool to assist with decompilation and modding projects"
 readme = "README.md"
 license = {file = "LICENSE"}

diff --git a/src/splat/__init__.py b/src/splat/__init__.py
@@ -1,7 +1,7 @@
 __package_name__ = __name__
 
 # Should be synced with pyproject.toml
-__version__ = "0.32.3"
+__version__ = "0.32.4"
 __author__ = "ethteck"
 
 from . import util as util

diff --git a/src/splat/segtypes/common/c.py b/src/splat/segtypes/common/c.py
@@ -451,7 +451,7 @@ def create_c_file(
                     c_lines += self.get_c_lines_for_rodata_sym(rodata_sym, asm_out_dir)
 
         c_path.parent.mkdir(parents=True, exist_ok=True)
-        with c_path.open("w") as f:
+        with c_path.open("w", newline=options.opts.c_newline) as f:
             f.write("\n".join(c_lines))
         log.write(f"Wrote {self.name} to {c_path}")
 
@@ -475,12 +475,12 @@ def create_asm_dependencies_file(
 
         dep_path = build_path / c_path.with_suffix(".asmproc.d")
         dep_path.parent.mkdir(parents=True, exist_ok=True)
-        with dep_path.open("w") as f:
+        with dep_path.open("w", newline=options.opts.c_newline) as f:
             if options.opts.use_o_as_suffix:
                 o_path = build_path / c_path.with_suffix(".o")
             else:
                 o_path = build_path / c_path.with_suffix(c_path.suffix + ".o")
-            f.write(f"{o_path}:")
+            f.write(f"{o_path.as_posix()}:")
             depend_list = []
             for entry in symbols_entries:
                 if entry.function is not None:
@@ -491,7 +491,7 @@ def create_asm_dependencies_file(
                         outpath.parent.mkdir(parents=True, exist_ok=True)
 
                         depend_list.append(outpath)
-                        f.write(f" \\\n    {outpath}")
+                        f.write(f" \\\n    {outpath.as_posix()}")
                 else:
                     for rodata_sym in entry.rodataSyms:
                         rodata_name = rodata_sym.getName()
@@ -501,9 +501,9 @@ def create_asm_dependencies_file(
                             outpath.parent.mkdir(parents=True, exist_ok=True)
 
                             depend_list.append(outpath)
-                            f.write(f" \\\n    {outpath}")
+                            f.write(f" \\\n    {outpath.as_posix()}")
 
             f.write("\n")
 
             for depend_file in depend_list:
-                f.write(f"{depend_file}:\n")
+                f.write(f"{depend_file.as_posix()}:\n")
diff --git a/src/splat/segtypes/linker_entry.py b/src/splat/segtypes/linker_entry.py
@@ -51,7 +51,7 @@ def write_file_if_different(path: Path, new_content: str):
 
     if old_content != new_content:
         path.parent.mkdir(parents=True, exist_ok=True)
-        with path.open("w") as f:
+        with path.open("w", newline=options.opts.c_newline) as f:
             f.write(new_content)
 
 
@@ -522,18 +522,18 @@ def save_symbol_header(self):
             )
 
     def save_dependencies_file(self, output_path: Path, target_elf_path: Path):
-        output = f"{target_elf_path}:"
+        output = f"{target_elf_path.as_posix()}:"
 
         for entry in self.dependencies_entries:
             if entry.object_path is None:
                 continue
-            output += f" \\\n    {entry.object_path}"
+            output += f" \\\n    {entry.object_path.as_posix()}"
 
         output += "\n"
         for entry in self.dependencies_entries:
             if entry.object_path is None:
                 continue
-            output += f"{entry.object_path}:\n"
+            output += f"{entry.object_path.as_posix()}:\n"
         write_file_if_different(output_path, output)
 
     def _writeln(self, line: str):
@@ -561,7 +561,7 @@ def _write_symbol(self, symbol: str, value: Union[str, int]):
         self.header_symbols.add(symbol)
 
     def _write_object_path_section(self, object_path: Path, section: str):
-        self._writeln(f"{object_path}({section});")
+        self._writeln(f"{object_path.as_posix()}({section});")
 
     def _begin_segment(
         self, segment: Segment, seg_name: str, noload: bool, is_first: bool

diff --git a/test.py b/test.py
@@ -1,49 +1,52 @@
 #!/usr/bin/env python3
 
-from spimdisasm.common import FileSectionType
-
-from src.splat.scripts.split import *
-import unittest
-import io
+import difflib
 import filecmp
+import io
 import pathlib
-from src.splat.util import symbols, options
 import spimdisasm
+import unittest
+
+from src.splat import __version__
+from src.splat.scripts.split import *
+from src.splat.util import symbols, options
 from src.splat.segtypes.common.rodata import CommonSegRodata
 from src.splat.segtypes.common.code import CommonSegCode
 from src.splat.segtypes.common.c import CommonSegC
 from src.splat.segtypes.common.bss import CommonSegBss
-from src.splat import __version__
-import difflib
 
 
 class Testing(unittest.TestCase):
     def compare_files(self, test_path, ref_path):
         with io.open(test_path) as test_f, io.open(ref_path) as ref_f:
             self.assertListEqual(list(test_f), list(ref_f))
 
-    def get_same_files(self, dcmp, out):
+    def get_same_files(self, dcmp: filecmp.dircmp, out: List[Tuple[str, str, str]]):
         for name in dcmp.same_files:
             out.append((name, dcmp.left, dcmp.right))
 
         for sub_dcmp in dcmp.subdirs.values():
             self.get_same_files(sub_dcmp, out)
 
-    def get_diff_files(self, dcmp, out):
+    def get_diff_files(self, dcmp: filecmp.dircmp, out: List[Tuple[str, str, str]]):
         for name in dcmp.diff_files:
             out.append((name, dcmp.left, dcmp.right))
 
         for sub_dcmp in dcmp.subdirs.values():
             self.get_diff_files(sub_dcmp, out)
 
-    def get_left_only_files(self, dcmp, out):
+    def get_left_only_files(
+        self, dcmp: filecmp.dircmp, out: List[Tuple[str, str, str]]
+    ):
         for name in dcmp.left_only:
             out.append((name, dcmp.left, dcmp.right))
 
         for sub_dcmp in dcmp.subdirs.values():
             self.get_left_only_files(sub_dcmp, out)
 
-    def get_right_only_files(self, dcmp, out):
+    def get_right_only_files(
+        self, dcmp: filecmp.dircmp, out: List[Tuple[str, str, str]]
+    ):
         for name in dcmp.right_only:
             out.append((name, dcmp.left, dcmp.right))
 
@@ -188,7 +191,7 @@ def test_add_symbol_to_spim_section(self):
             vram=0x40000000,
             filename="test",
             words=[],
-            sectionType=FileSectionType.Text,
+            sectionType=spimdisasm.common.FileSectionType.Text,
             segmentVromStart=0x0,
             overlayCategory=None,
         )

diff --git a/test/basic_app/.gitignore b/test/basic_app/.gitignore
@@ -1,3 +1,4 @@
-build/
-split/
-gcc-2.7.2/
+/build/
+/split/
+/gcc-2.7.2/
+!/expected/
diff --git a/test/basic_app/build/basic_app.bin b/test/basic_app/build/basic_app.bin
diff --git a/test/basic_app/expected/.splache b/test/basic_app/expected/.splache
diff --git a/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s b/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s
@@ -1,6 +1,3 @@
-.set noat      /* allow manual use of $at */
-.set noreorder /* don't insert nops after branches */
-
 .section .rodata
 .align 3
 glabel jtbl_80000518

diff --git a/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s b/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s
@@ -1,6 +1,3 @@
-.set noat      /* allow manual use of $at */
-.set noreorder /* don't insert nops after branches */
-
 glabel func_800004A0
     /* 10A0 800004A0 27BDFFE8 */  addiu      $sp, $sp, -0x18
     /* 10A4 800004A4 AFBF0014 */  sw         $ra, 0x14($sp)

diff --git a/test/basic_app/expected/basic_app.d b/test/basic_app/expected/basic_app.d
@@ -0,0 +1,21 @@
+test/basic_app/split/build/basic_app_target.elf: \
+    build/asm/header.o \
+    build/assets/dummy_ipl3.o \
+    build/src/main.o \
+    build/asm/handwritten.o \
+    build/asm/data/main.data.o \
+    build/asm/handwritten.o \
+    build/src/main.o \
+    build/asm/handwritten.o \
+    build/asm/data/main.bss.o \
+    build/asm/handwritten.o
+build/asm/header.o:
+build/assets/dummy_ipl3.o:
+build/src/main.o:
+build/asm/handwritten.o:
+build/asm/data/main.data.o:
+build/asm/handwritten.o:
+build/src/main.o:
+build/asm/handwritten.o:
+build/asm/data/main.bss.o:
+build/asm/handwritten.o:
diff --git a/test/basic_app/expected/basic_app.ld b/test/basic_app/expected/basic_app.ld
@@ -7,7 +7,7 @@ SECTIONS
     {
         FILL(0x00000000);
         header_DATA_START = .;
-        build/split/asm/header.o(.data);
+        build/asm/header.o(.data);
         header_DATA_END = .;
         header_DATA_SIZE = ABSOLUTE(header_DATA_END - header_DATA_START);
     }
@@ -21,7 +21,7 @@ SECTIONS
     {
         FILL(0x00000000);
         dummy_ipl3_DATA_START = .;
-        build/split/assets/dummy_ipl3.o(.data);
+        build/assets/dummy_ipl3.o(.data);
         . = ALIGN(., 16);
         dummy_ipl3_DATA_END = .;
         dummy_ipl3_DATA_SIZE = ABSOLUTE(dummy_ipl3_DATA_END - dummy_ipl3_DATA_START);
@@ -38,20 +38,20 @@ SECTIONS
     {
         FILL(0x00000000);
         boot_TEXT_START = .;
-        build/split/src/main.o(.text);
-        build/split/asm/handwritten.o(.text);
+        build/src/main.o(.text);
+        build/asm/handwritten.o(.text);
         . = ALIGN(., 16);
         boot_TEXT_END = .;
         boot_TEXT_SIZE = ABSOLUTE(boot_TEXT_END - boot_TEXT_START);
         boot_DATA_START = .;
-        build/split/asm/data/main.data.o(.data);
-        build/split/asm/handwritten.o(.data);
+        build/asm/data/main.data.o(.data);
+        build/asm/handwritten.o(.data);
         . = ALIGN(., 16);
         boot_DATA_END = .;
         boot_DATA_SIZE = ABSOLUTE(boot_DATA_END - boot_DATA_START);
         boot_RODATA_START = .;
-        build/split/src/main.o(.rodata);
-        build/split/asm/handwritten.o(.rodata);
+        build/src/main.o(.rodata);
+        build/asm/handwritten.o(.rodata);
         . = ALIGN(., 16);
         boot_RODATA_END = .;
         boot_RODATA_SIZE = ABSOLUTE(boot_RODATA_END - boot_RODATA_START);
@@ -61,8 +61,8 @@ SECTIONS
     {
         FILL(0x00000000);
         boot_BSS_START = .;
-        build/split/asm/data/main.bss.o(.bss);
-        build/split/asm/handwritten.o(.bss);
+        build/asm/data/main.bss.o(.bss);
+        build/asm/handwritten.o(.bss);
         . = ALIGN(., 16);
         boot_BSS_END = .;
         boot_BSS_SIZE = ABSOLUTE(boot_BSS_END - boot_BSS_START);

diff --git a/test/basic_app/expected/build/test/basic_app/split/src/main.asmproc.d b/test/basic_app/expected/build/test/basic_app/split/src/main.asmproc.d
@@ -0,0 +1,7 @@
+test/basic_app/split/build/test/basic_app/split/src/main.o: \
+    test/basic_app/split/asm/nonmatchings/main/D_80000510.s \
+    test/basic_app/split/asm/nonmatchings/main/func_80000400.s \
+    test/basic_app/split/asm/nonmatchings/main/func_800004A0.s
+test/basic_app/split/asm/nonmatchings/main/D_80000510.s:
+test/basic_app/split/asm/nonmatchings/main/func_80000400.s:
+test/basic_app/split/asm/nonmatchings/main/func_800004A0.s: