Add more tests for otter.test_files (#866)

.coveragerc

@@ -11,9 +11,15 @@ exclude_lines =
     # Don't complain if tests don't hit defensive assertion code:
     raise NotImplementedError
 
+    # Ignore ellipse statements
+    \.\.\.
+
 omit =
     # This file is just calls otter.cli.cli
     otter/__main__.py
 
     # This file is a copy of an external script and is not tested as part of Otter
     otter/plugins/builtin/gmail_notifications/bin/gmail_oauth2.py
+
+    # ignore templates
+    otter/generate/templates/**/*

otter/test_files/abstract_test.py

@@ -98,7 +98,7 @@ def _repr_html_(self):
                 if not tcr.passed and tcr.test_case.failure_message is not None:
                     ret += f"<p><strong><pre style='display: inline;'>{tcr.test_case.name}</pre> message:</strong> {tcr.test_case.failure_message}</p>"
                 ret += f"<p><strong><pre style='display: inline;'>{tcr.test_case.name}</pre> result:</strong></p>"
-                ret += f"<pre>{indent(tcr.message, '    ')}</pre>"
+                ret += f"<pre>{indent(tcr.message or '', '    ')}</pre>"
 
             return ret
 
@@ -145,7 +145,7 @@ def resolve_test_file_points(
             total_points = None
 
         elif total_points is not None and not isinstance(total_points, (int, float)):
-            raise TypeError(f"Test spec points has invalid type: {total_points}")
+            raise TypeError(f"Test spec points has invalid type: {type(total_points)}")
 
         point_values = []
         for test_case in test_cases:
@@ -162,7 +162,7 @@ def resolve_test_file_points(
         pre_specified = sum(p for p in point_values if p is not None)
         if total_points is not None:
             if pre_specified > total_points:
-                raise ValueError(f"More points specified in test cases than allowed for test")
+                raise ValueError("More points specified in test cases than allowed for test")
 
             else:
                 try:
@@ -180,9 +180,6 @@ def resolve_test_file_points(
                 except ZeroDivisionError:
                     per_remaining = 0.0
 
-            elif pre_specified == 0:
-                per_remaining = 1 / len(point_values)
-
             else:
                 # assume all other tests are worth 0 points
                 per_remaining = 0.0
@@ -269,12 +266,12 @@ def summary(self, public_only: bool = False) -> str:
         """
         if (not public_only and self.passed_all) or (public_only and self.passed_all_public):
             ret = f"{self.name} results: All test cases passed!"
-            if (not public_only and self.passed_all) and any(
-                tcr.test_case.success_message is not None for tcr in self.test_case_results
-            ):
-                for tcr in self.test_case_results:
-                    if tcr.test_case.success_message is not None:
-                        ret += f"\n{tcr.test_case.name} message: {tcr.test_case.success_message}"
+            all_tcrs = self.test_case_results
+            if public_only:
+                all_tcrs = [tcr for tcr in self.test_case_results if not tcr.test_case.hidden]
+            for tcr in all_tcrs:
+                if tcr.test_case.success_message is not None:
+                    ret += f"\n{tcr.test_case.name} message: {tcr.test_case.success_message}"
             return ret
 
         tcrs = self.test_case_results
@@ -289,7 +286,7 @@ def summary(self, public_only: bool = False) -> str:
             if not tcr.passed and tcr.test_case.failure_message is not None:
                 smry += f"{tcr.test_case.name} message: {tcr.test_case.failure_message}\n\n"
             smry += f"{tcr.test_case.name} result:\n"
-            smry += f"{indent(tcr.message.strip(), '    ')}\n\n"
+            smry += f"{indent((tcr.message or '').strip(), '    ')}\n\n"
 
             tcr_summaries.append(smry.strip())
 

otter/test_files/exception_test.py

@@ -52,6 +52,17 @@ def __init__(
         self.failure_message = failure_message
         self.test_func = lambda: None
 
+    def __eq__(self, other: Any) -> bool:
+        if not isinstance(other, type(self)):
+            return False
+        return (
+            self.name == other.name
+            and self.points == other.points
+            and self.hidden == other.hidden
+            and self.success_message == other.success_message
+            and self.failure_message == other.failure_message
+        )
+
     def __call__(self, test_func: Callable[..., None]) -> "test_case":
         """
         Wrap a test case function as a decorator.
@@ -175,7 +186,7 @@ def run(self, global_environment: dict[str, Any]):
         Arguments:
             global_environment (``dict[str, Any]``): result of executing a Python notebook/script
         """
-        test_case_results = []
+        self.test_case_results = []
         for tc in self.test_cases:
             test_case = tc.body
             passed, message = True, "✅ Test case passed"
@@ -184,9 +195,9 @@ def run(self, global_environment: dict[str, Any]):
             except Exception as e:
                 passed, message = False, "❌ Test case failed\n" + self._generate_error_message(e)
 
-            test_case_results.append(TestCaseResult(test_case=tc, message=message, passed=passed))
-
-        self.test_case_results = test_case_results
+            self.test_case_results.append(
+                TestCaseResult(test_case=tc, message=message, passed=passed)
+            )
 
     @staticmethod
     def _compile_string(s: str, path: str = "<string>") -> CodeType:

otter/test_files/ok_test.py

@@ -66,6 +66,7 @@ def run(self, global_environment: dict[str, Any]):
         Arguments:
             ``global_environment`` (``dict``): result of executing a Python notebook/script
         """
+        self.test_case_results = []
         for i, test_case in enumerate(self.test_cases):
             passed, result = run_doctest(
                 self.name + " " + str(i), test_case.body, global_environment

test/conftest.py

@@ -9,6 +9,7 @@
 from unittest import mock
 
 from otter import __file__ as OTTER_PATH
+from otter.test_files import TestCase, TestCaseResult, TestFile
 
 from .utils import TestFileManager
 
@@ -17,6 +18,11 @@
 REPO_DIR = os.getcwd()
 REAL_DOCKER_BUILD = docker.build
 
+# prevent pytest from thinking these classes are testing classes
+TestCase.__test__ = False
+TestCaseResult.__test__ = False
+TestFile.__test__ = False
+
 
 def pytest_addoption(parser):
     """

test/test_assign/test_integration.py

@@ -19,10 +19,6 @@
 from ..utils import assert_dirs_equal, TestFileManager, unzip_to_temp
 
 
-# prevent pytest from thinking TestCase is a testing class
-TestCase.__test__ = False
-
-
 FILE_MANAGER = TestFileManager(__file__)