Skip to content

Commit

Permalink
Otter Grade CSV Column Order and Points Possible Tightened up
Browse files Browse the repository at this point in the history
- Key Constants Renamed
- final_grades.csv columns ordered
- added assert for existance of total-points-possible in every row of final_grades.csv
  • Loading branch information
sean-morris committed Jul 10, 2024
1 parent ceb7b03 commit 48f4a9e
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 31 deletions.
7 changes: 4 additions & 3 deletions otter/grade/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from typing import List, Optional, Tuple, Union

from .containers import launch_containers
from .utils import merge_csv, prune_images, KEY_SCORES_DICT_FILE, KEY_SCORES_DICT_TTL_PTS, KEY_SCORES_DICT_PCT_CRT
from .utils import merge_csv, prune_images, SCORES_DICT_FILE_KEY, SCORES_DICT_PERCENT_CORRECT_KEY, SCORES_DICT_TOTAL_POINTS_KEY

from ..run.run_autograder.autograder_config import AutograderConfig
from ..utils import assert_path_exists, loggers
Expand Down Expand Up @@ -129,11 +129,12 @@ def main(
# Merge dataframes
output_df = merge_csv(grade_dfs)
cols = output_df.columns.tolist()
output_df = output_df[[KEY_SCORES_DICT_FILE] + cols[:-3] + [KEY_SCORES_DICT_TTL_PTS, KEY_SCORES_DICT_PCT_CRT]]
question_cols = sorted(c for c in cols if c not in {SCORES_DICT_FILE_KEY, SCORES_DICT_TOTAL_POINTS_KEY, SCORES_DICT_PERCENT_CORRECT_KEY})
output_df = output_df[[SCORES_DICT_FILE_KEY, *question_cols, SCORES_DICT_TOTAL_POINTS_KEY, SCORES_DICT_PERCENT_CORRECT_KEY]]

# write to CSV file
output_df.to_csv(os.path.join(output_dir, "final_grades.csv"), index=False)

# return percentage if a single file was graded
if len(paths) == 1 and os.path.isfile(paths[0]):
return output_df[KEY_SCORES_DICT_PCT_CRT][1]
return output_df[SCORES_DICT_PERCENT_CORRECT_KEY][1]
29 changes: 4 additions & 25 deletions otter/grade/containers.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import json
import os
import pandas as pd
import pathlib
import pkg_resources
import shutil
Expand All @@ -14,7 +13,7 @@
from textwrap import indent
from typing import List, Optional

from .utils import OTTER_DOCKER_IMAGE_NAME, POINTS_POSSIBLE_LABEL, KEY_SCORES_DICT_FILE, KEY_SCORES_DICT_TTL_PTS, KEY_SCORES_DICT_PCT_CRT
from .utils import OTTER_DOCKER_IMAGE_NAME, merge_scores_to_df

from ..run.run_autograder.autograder_config import AutograderConfig
from ..utils import loggers, OTTER_CONFIG_FILENAME
Expand Down Expand Up @@ -85,7 +84,7 @@ def launch_containers(
Grade submissions in parallel Docker containers.
This function runs ``num_containers`` Docker containers in parallel to grade the student
submissions in ``submissions_dir`` using the autograder configuration file at ``ag_zip_path``.
submissions in ``submissions_dir`` using the autograder configuration file at ``ag_zip_path``.
If indicated, it copies the PDFs generated of the submissions out of their containers.
Args:
Expand Down Expand Up @@ -117,28 +116,8 @@ def launch_containers(

# stop execution while containers are running
finished_futures = wait(futures)

# return list of dataframes with points possible as first frame
full_df = []
f = next(iter(finished_futures[0]))
scores = f.result()
scores_dict = scores.to_dict()

pts_poss_dict = {t: [scores_dict[t]["possible"]] for t in scores_dict}
pts_poss_dict[KEY_SCORES_DICT_FILE] = POINTS_POSSIBLE_LABEL
pts_poss_dict[KEY_SCORES_DICT_PCT_CRT] = "NA"
pts_poss_dict[KEY_SCORES_DICT_TTL_PTS] = scores.possible
pts_poss_df = pd.DataFrame(pts_poss_dict)
full_df.append(pts_poss_df)
for f in finished_futures[0]:
scores_dict = f.result().to_dict()
scores_dict = {t: [scores_dict[t]["score"]] for t in scores_dict}
scores_dict[KEY_SCORES_DICT_PCT_CRT] = round(f.result().total / f.result().possible,4)
scores_dict[KEY_SCORES_DICT_TTL_PTS] = f.result().total
scores_dict[KEY_SCORES_DICT_FILE] = f.result().file
df_scores = pd.DataFrame(scores_dict)
full_df.append(df_scores)
return full_df
scores = [f.result() for f in finished_futures[0]]
return merge_scores_to_df(scores)


def grade_submission(
Expand Down
38 changes: 35 additions & 3 deletions otter/grade/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,21 @@
import pandas as pd
import re

from typing import List
from python_on_whales import docker

from ..test_files import GradingResults

OTTER_DOCKER_IMAGE_NAME = "otter-grade"

POINTS_POSSIBLE_LABEL = "points-per-question"

KEY_SCORES_DICT_FILE = "file"
SCORES_DICT_FILE_KEY = "file"

KEY_SCORES_DICT_TTL_PTS = "total_points_earned"
SCORES_DICT_TOTAL_POINTS_KEY = "total_points_earned"

SCORES_DICT_PERCENT_CORRECT_KEY = "percent_correct"

KEY_SCORES_DICT_PCT_CRT = "percent_correct"

def list_files(path):
"""
Expand Down Expand Up @@ -74,3 +77,32 @@ def prune_images(force=False):

else:
print("Prune cancelled.")


def merge_scores_to_df(scores: List[GradingResults]) -> pd.DataFrame:
"""
Convert a list of ``GradingResults`` objects to a scores dataframe, including a row
with the total point values for each question.
Args:
scores (``list[otter.test_files.GradingResults]``): the score objects to merge
Returns:
``pd.DataFrame``: the scores dataframe
"""
full_df = []
pts_poss_dict = {t: [scores[0].to_dict()[t]["possible"]] for t in scores[0].to_dict()}
pts_poss_dict[SCORES_DICT_FILE_KEY] = POINTS_POSSIBLE_LABEL
pts_poss_dict[SCORES_DICT_PERCENT_CORRECT_KEY] = "NA"
pts_poss_dict[SCORES_DICT_TOTAL_POINTS_KEY] = scores[0].possible
pts_poss_df = pd.DataFrame(pts_poss_dict)
full_df.append(pts_poss_df)
for grading_result in scores:
scores_dict = grading_result.to_dict()
scores_dict = {t: [scores_dict[t]["score"]] for t in scores_dict}
scores_dict[SCORES_DICT_PERCENT_CORRECT_KEY] = round(grading_result.total / grading_result.possible, 4)
scores_dict[SCORES_DICT_TOTAL_POINTS_KEY] = grading_result.total
scores_dict[SCORES_DICT_FILE_KEY] = grading_result.file
df_scores = pd.DataFrame(scores_dict)
full_df.append(df_scores)
return full_df
3 changes: 3 additions & 0 deletions test/test_grade/test_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,9 @@ def test_notebooks_with_pdfs(expected_points):
)
assert sorted(dir1_contents) == sorted(dir2_contents), f"'{FILE_MANAGER.get_path('notebooks/')}' and 'test/submission_pdfs' have different contents"

# check that the row with point totals for each question exists
assert any(POINTS_POSSIBLE_LABEL in row for row in df_test.itertuples(index=False))


@mock.patch("otter.grade.launch_containers")
def test_single_notebook_grade(mocked_launch_grade):
Expand Down

0 comments on commit 48f4a9e

Please sign in to comment.