Merge branch 'master' into release

.github/workflows/run-tests.yml

@@ -59,5 +59,6 @@ jobs:
 
       - uses: codecov/codecov-action@v3
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true
           verbose: true

.readthedocs.yml

@@ -14,3 +14,8 @@ formats: all
 
 conda:
   environment: rtd_environment.yml
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "mambaforge-22.9"

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+**v5.3.0 (unreleased):**
+
+* Updated Otter Assign to throw a `ValueError` when invalid Python code is encountered in test cells per [#756](https://github.com/ucbds-infra/otter-grader/issues/756)
+* Fixed an error causing intercell seeding code to be added to cells using cell magic commands which caused syntax errors per [#754](https://github.com/ucbds-infra/otter-grader/issues/754)
+
 **v5.2.3:**
 
 * Fixed the no PDF acknowledgement feature to handle when PDF exports throw an error instead of failing silently

docs/_static/grading-environment-r.yml

@@ -18,6 +18,7 @@ dependencies:
   - r-startup
   - r-rmarkdown
   - r-stringi
+  - r-ottr==1.4.0
   - pip:
       - datascience
       - jupyter_client

docs/_static/r_setup.sh

@@ -30,7 +30,4 @@ mamba env create -f /autograder/source/environment.yml
 mamba run -n otter-env Rscript /autograder/source/requirements.r
 
 # set mamba shell
-mamba init --all
-
-# install ottr
-mamba run -n otter-env Rscript -e 'install.packages("https://cran.r-project.org/package=ottr&version=", dependencies=TRUE, repos=NULL)'
+mamba init --all

docs/debugging.rst

@@ -0,0 +1,87 @@
+Debugging Submissions
+=====================
+
+To help debug submissions that are not passing tests or which are behaving oddly, it can be helpful
+to use Otter's debug mode. The main use for debug mode is to examine errors that are being thrown
+in the submission and which Otter swallows during normal execution. This is a guide for setting up
+an environment where Otter's debug mode is enabled and for using it.
+
+
+Setting Up a Debugging Environment
+----------------------------------
+
+The first step is to set up a debugging environment. This step depends on how you're executing the
+submissions. If you're using containerized local grading (``otter grade``) or Gradescope, follow the
+steps in this section. If you're using non-containerized grading (``otter run``), follow the steps
+in :ref:`debugging_with_otter_run`.
+
+
+Otter Grade Setup
++++++++++++++++++
+
+To create a container for debugging a submission, use the Docker image that was created for your
+assignment when you ran ``otter grade``. The image's tag will use the assignment name (the value of
+the ``-n`` or ``--name`` flag). If you passed ``-n hw01``, the image name will be
+``otter-grade:hw01``. Create a container using this image and run bash:
+
+.. code-block:: console
+
+    docker run -it otter-grade:<assignment name> bash
+
+This should create the container and drop you into a bash prompt inside of it. Now, in a separate
+terminal, copy the submission file into the container you've just created:
+
+.. code-block:: console
+
+    docker cp path/to/subm.ipynb <container id>:/autograder/submission/subm.ipynb
+
+You can get the container ID either from the prompt in the container's bash shell or from the
+``docker container ls`` command.
+
+Now that you've set up the grading environment, continue with the debugging steps
+:ref:`below <debugging_container>`. Once you're finished debugging, remember to stop the container
+and remove it.
+
+
+Gradescope Setup
+++++++++++++++++
+
+Find the submission you would like to debug on Gradescope and use Gradescope's `Debug via SSH
+feature <https://gradescope-autograders.readthedocs.io/en/latest/ssh/>`_ to create a container with
+that submission and SSH into it, then continue with the debugging steps
+:ref:`below <debugging_container>`.
+
+
+.. _debugging_container:
+
+Debugging in a Container
+------------------------
+
+In the container, ``cd`` into the ``/autograder/source`` directory. There should be an
+``otter_config.json`` file in this directory containing Otter's autograder configuration. (If there
+isn't one present, make one.) Edit this file to set ``"debug"`` to ``true`` (this enables debug
+mode).
+
+Now ``cd`` into ``/autograder`` and run
+
+.. code-block:: console
+
+    ./run_autograder
+
+This shell script runs Otter and its results should be printed to the console. If an error is thrown
+while executing the notebook, it will be shown and the autograder will stop. If you're running on
+Gradescope, no changes or autograder runs made here will affect the submission or its grade.
+
+If you need to edit the submission file(s), they are located in the ``/autograder/submission``
+directory.
+
+
+.. _debugging_with_otter_run:
+
+Debugging with Otter Run
+------------------------
+
+Read the :ref:`previous section <debugging_container>` first. Because Otter Run relies on an
+autograder zip file for its configuration intsead of a Docker container, you will need to manually
+edit the ``otter_config.json`` file in your autograder zip file to set ``"debug"`` to ``true``.
+Then, re-zip the zip file's contents and use this new autograder zip file for ``otter run``.

docs/index.rst

@@ -13,6 +13,7 @@ Otter-Grader Documentation
     otter_check/index
     workflow/index
     execution
+    debugging
     plugins/index
     pdfs
     seeding

docs/requirements.txt

@@ -1,5 +1,6 @@
+sphinx>=5.0.0
 sphinx_markdown_tables
-sphinx-book-theme==0.3.3
+sphinx-book-theme==1.1.0
 sphinxcontrib-apidoc
 sphinx-click
 jinja2

docs/resources.rst

@@ -10,3 +10,4 @@ Here are some resources and examples for using Otter-Grader:
   <https://econ140-spring2020.shinyapps.io/otter_quickstart/>`_
 * `Otter-Grader JupyterLab Extension <https://github.com/chrispyles/otter-grader-labextension>`_: A
   JupyterLab extension for authoring Otter Assign-formatted notebooks.
+* `Chris Pyles and Suraj Rampure presenting at Jupytercon 2023 <https://www.youtube.com/watch?v=9_x532_2T2w&list=PL_1BH3ug7n1Ih_Yy2TmM7MZ2zogSLZvzE&index=19>`_  along with a `Jupyterlite demo file <https://chrispyles.io/otter-grader-jupytercon-2023/lab/index.html>`_

docs/workflow/executing_submissions/otter_grade.rst

@@ -22,12 +22,12 @@ re-downloaded when the image for an assignment is built for the first time.
 Assignment Names
 ++++++++++++++++
 
-Whenever you use Otter Grade, you must specify an assignment name. This assignment name is used as
-the tag for the Docker image that Otter creates (so you will have an image called
-``otter-grade:{assignment name}`` for each assignment). These assignment names are required so that
-Otter can make effective user of Docker's image layer cache. This means that if you make changes to
-tests or need to grade an assignment twice, Docker doesn't need to reinstall all of the dependencies
-Otter defines.
+Whenever you use Otter Grade, you must specify an assignment name with the ``-n`` or ``--name``
+flag. This assignment name is used as the tag for the Docker image that Otter creates (so you will
+have an image called ``otter-grade:{assignment name}`` for each assignment). These assignment names
+are required so that Otter can make effective user of Docker's image layer cache. This means that if
+you make changes to tests or need to grade an assignment twice, Docker doesn't need to reinstall all
+of the dependencies Otter defines.
 
 These images can be quite large (~4GB), so Otter provides a way to easily prune all of the Docker
 images it has created:

otter/assign/tests_manager.py

@@ -145,6 +145,7 @@ def read_test(self, cell, question):
             question (``otter.assign.question_config.QuestionConfig``): the question config
         """
         source = get_source(cell)
+        ensure_valid_syntax(source, question)
 
         if source[0].lstrip().startswith("#"):
             hidden = bool(re.search(r"\bhidden\b", source[0], flags=re.IGNORECASE))
@@ -425,3 +426,20 @@ def generate_assignment_summary(self):
         summary += f"Manual:       {manual}\n\n"
         summary += str(pd.DataFrame(rows))
         return summary
+
+
+def ensure_valid_syntax(source: List[str], question: QuestionConfig) -> None:
+    """
+    Check that a cell's source is valid Python syntax.
+
+    Args:
+        source (``list[str]``): the cell source lines
+
+    Raises:
+        ``ValueError``: if invalid syntax is found
+    """
+    source = "\n".join(source)
+    try:
+        ast.parse(source)
+    except SyntaxError as e:
+        raise ValueError(f"A test cell in question {question.name} contains invalid Python syntax:\n{source}")

otter/check/logs.py

@@ -14,17 +14,6 @@
 class EventType(Enum):
     """
     Enum of event types for log entries
-
-    Attributes:
-        AUTH: an auth event
-        BEGIN_CHECK_ALL: beginning of a check-all call
-        BEGIN_EXPORT: beginning of an assignment export
-        CHECK: a check of a single question
-        END_CHECK_ALL: ending of a check-all call
-        END_EXPORT: ending of an assignment export
-        INIT: initialization of an ``otter.Notebook`` object
-        SUBMIT: submission of an assignment
-        TO_PDF: PDF export of a notebook
     """
 
     AUTH = auto()

otter/execute/preprocessor.py

@@ -122,7 +122,7 @@ def add_seeds(self, nb):
 
         cells = nb.cells[1:] if skip_first else nb.cells
         for cell in cells:
-            if cell.cell_type == "code":
+            if cell.cell_type == "code" and not cell.source.startswith("%%"):
                 cell.source = f"{do_seed}\n{cell.source}"
 
     def add_checks(self, nb):