Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Crowdsourced Hints - "0.2 release" #442

Merged
merged 28 commits into from
Aug 27, 2013
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
fc8acb6
Changed voting dialog to only display one tab per answer.
fephsun Jun 28, 2013
6938075
Fixed tests to work with new vote-tabbing method. Fixed some string-…
fephsun Jun 28, 2013
c34a81a
Refactored formula response grader to expose formula-evaluating to ou…
fephsun Jul 10, 2013
b88c6b8
Hinter now works with formula responses. Tests broken.
fephsun Jul 11, 2013
199b632
Crowdsourced hinter now supports formula responses. Tests still broken.
fephsun Jul 11, 2013
a730f91
Fixed numerous 500 errors that result from mal-formatted post requests.
fephsun Jul 15, 2013
6b40c5c
Changed hint voting ui to show all hints on one page.
fephsun Jul 15, 2013
4ee8111
Fixed monkey-patching persistent state bug.
fephsun Jul 16, 2013
8ce53ed
Got rid of answer signatures - these don't work with approximate answ…
fephsun Jul 17, 2013
d9517ea
Fixed tests for removing hash access to hints. Fixed instructor view…
fephsun Jul 18, 2013
e2aea75
Fixed a bug in recording hints shown.
fephsun Jul 18, 2013
544aa55
Added comments to explain confusing field names in crowdsource_hinter.
fephsun Jul 19, 2013
eb5a90a
Added tests for validate_answer in response types.
fephsun Jul 19, 2013
e6db6e9
Fixed bug where user isn't asked to submit a hint unless he's seen a …
fephsun Jul 19, 2013
d671574
Addressed pull request comments.
fephsun Jul 22, 2013
f3c3436
Addressing more PR comments. Fixed a test that broke when new code c…
fephsun Jul 30, 2013
64c91cf
UX improvements to hinting. Interface is now less cluttered.
fephsun Jul 31, 2013
7c93b45
Added wizard / slideshow style hint collection script.
fephsun Jul 31, 2013
35a8e9b
Some fixes on hint ui.
fephsun Aug 1, 2013
2d1c915
Added fancy sliding transitions into hint collection page.
fephsun Aug 1, 2013
4220252
Fixed terribly annoying issues with back button. Back button is now on
fephsun Aug 2, 2013
69fbe77
Fixed a rebase error in responsetypes.
fephsun Aug 15, 2013
e6067d8
Addressed PR comments.
fephsun Aug 19, 2013
68bb45a
Added jasmine tests to crowdsource_hinter. UI is not covered; only e…
fephsun Aug 20, 2013
1f9eafe
Addressed some minor PR comments.
fephsun Aug 21, 2013
74ed2da
Improved docstrings.
fephsun Aug 23, 2013
6e64e99
Fixed a test broken when the mixed modulestore was introduced.
fephsun Aug 26, 2013
444f51d
Fixed some pep/pylint violations.
fephsun Aug 27, 2013
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 94 additions & 44 deletions common/lib/capa/capa/responsetypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -915,7 +915,26 @@ def get_score(self, student_answers):
else:
return CorrectMap(self.answer_id, 'incorrect')

# TODO: add check_hint_condition(self, hxml_set, student_answers)
def compare_answer(self, ans1, ans2):
"""
Outside-facing function that lets us compare two numerical answers,
with this problem's tolerance.
"""
return compare_with_tolerance(
evaluator({}, {}, ans1),
evaluator({}, {}, ans2),
self.tolerance
)

def validate_answer(self, answer):
"""
Returns whether this answer is in a valid form.
"""
try:
evaluator(dict(), dict(), answer)
return True
except (StudentInputError, UndefinedVariable):
return False

def get_answers(self):
return {self.answer_id: self.correct_answer}
Expand Down Expand Up @@ -1778,46 +1797,24 @@ def get_score(self, student_answers):
self.correct_answer, given, self.samples)
return CorrectMap(self.answer_id, correctness)

def check_formula(self, expected, given, samples):
variables = samples.split('@')[0].split(',')
numsamples = int(samples.split('@')[1].split('#')[1])
sranges = zip(*map(lambda x: map(float, x.split(",")),
samples.split('@')[1].split('#')[0].split(':')))

ranges = dict(zip(variables, sranges))
for _ in range(numsamples):
instructor_variables = self.strip_dict(dict(self.context))
student_variables = {}
# ranges give numerical ranges for testing
for var in ranges:
# TODO: allow specified ranges (i.e. integers and complex numbers) for random variables
value = random.uniform(*ranges[var])
instructor_variables[str(var)] = value
student_variables[str(var)] = value
# log.debug('formula: instructor_vars=%s, expected=%s' %
# (instructor_variables,expected))

# Call `evaluator` on the instructor's answer and get a number
instructor_result = evaluator(
instructor_variables, {},
expected, case_sensitive=self.case_sensitive
)
def tupleize_answers(self, answer, var_dict_list):
"""
Takes in an answer and a list of dictionaries mapping variables to values.
Each dictionary represents a test case for the answer.
Returns a tuple of formula evaluation results.
"""
out = []
for var_dict in var_dict_list:
try:
# log.debug('formula: student_vars=%s, given=%s' %
# (student_variables,given))

# Call `evaluator` on the student's answer; look for exceptions
student_result = evaluator(
student_variables,
{},
given,
case_sensitive=self.case_sensitive
)
out.append(evaluator(
var_dict,
dict(),
answer,
case_sensitive=self.case_sensitive,
))
except UndefinedVariable as uv:
log.debug(
'formularesponse: undefined variable in given=%s',
given
)
'formularesponse: undefined variable in formula=%s' % answer)
raise StudentInputError(
"Invalid input: " + uv.message + " not permitted in answer"
)
Expand All @@ -1840,17 +1837,70 @@ def check_formula(self, expected, given, samples):
# If non-factorial related ValueError thrown, handle it the same as any other Exception
log.debug('formularesponse: error {0} in formula'.format(ve))
raise StudentInputError("Invalid input: Could not parse '%s' as a formula" %
cgi.escape(given))
cgi.escape(answer))
except Exception as err:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What errors actually get thrown here?

# traceback.print_exc()
log.debug('formularesponse: error %s in formula', err)
raise StudentInputError("Invalid input: Could not parse '%s' as a formula" %
cgi.escape(given))
cgi.escape(answer))
return out

# No errors in student's response--actually test for correctness
if not compare_with_tolerance(student_result, instructor_result, self.tolerance):
return "incorrect"
return "correct"
def randomize_variables(self, samples):
"""
Returns a list of dictionaries mapping variables to random values in range,
as expected by tupleize_answers.
"""
variables = samples.split('@')[0].split(',')
numsamples = int(samples.split('@')[1].split('#')[1])
sranges = zip(*map(lambda x: map(float, x.split(",")),
samples.split('@')[1].split('#')[0].split(':')))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Probably worth holding on to a version of samples.split('@') since you seem to use it a number of times.

ranges = dict(zip(variables, sranges))

out = []
for i in range(numsamples):
var_dict = {}
# ranges give numerical ranges for testing
for var in ranges:
# TODO: allow specified ranges (i.e. integers and complex numbers) for random variables
value = random.uniform(*ranges[var])
var_dict[str(var)] = value
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code should probably be re factored to reduce complexity since a lot of appending could slow things down a lot.

random_dict = lambda: {str(var): random.uniform(*ranges[var]) for var in ranges}
out = [random_dict() for i in range(numsamples)]

out.append(var_dict)
return out

def check_formula(self, expected, given, samples):
"""
Given an expected answer string, a given (student-produced) answer
string, and a samples string, return whether the given answer is
"correct" or "incorrect".
"""
var_dict_list = self.randomize_variables(samples)
student_result = self.tupleize_answers(given, var_dict_list)
instructor_result = self.tupleize_answers(expected, var_dict_list)

correct = all(compare_with_tolerance(student, instructor, self.tolerance)
for student, instructor in zip(student_result, instructor_result))
if correct:
return "correct"
else:
return "incorrect"

def compare_answer(self, ans1, ans2):
"""
An external interface for comparing whether a and b are equal.
"""
internal_result = self.check_formula(ans1, ans2, self.samples)
return internal_result == "correct"

def validate_answer(self, answer):
"""
Returns whether this answer is in a valid form.
"""
var_dict_list = self.randomize_variables(self.samples)
try:
self.tupleize_answers(answer, var_dict_list)
return True
except StudentInputError:
return False

def strip_dict(self, d):
''' Takes a dict. Returns an identical dict, with all non-word
Expand Down
28 changes: 28 additions & 0 deletions common/lib/capa/capa/tests/test_responsetypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,20 @@ def test_raises_zero_division_err(self):
input_dict = {'1_2_1': '1/0'}
self.assertRaises(StudentInputError, problem.grade_answers, input_dict)

def test_validate_answer(self):
"""
Makes sure that validate_answer works.
"""
sample_dict = {'x': (1, 2)}
problem = self.build_problem(
sample_dict=sample_dict,
num_samples=10,
tolerance="1%",
answer="x"
)
self.assertTrue(problem.responders.values()[0].validate_answer('14*x'))
self.assertFalse(problem.responders.values()[0].validate_answer('3*y+2*x'))


class StringResponseTest(ResponseTest):
from capa.tests.response_xml_factory import StringResponseXMLFactory
Expand Down Expand Up @@ -915,6 +929,20 @@ def evaluator_side_effect(_, __, math_string):
with self.assertRaisesRegexp(StudentInputError, msg_regex):
problem.grade_answers({'1_2_1': 'foobar'})

def test_compare_answer(self):
"""Tests the answer compare function."""
problem = self.build_problem(answer="42")
responder = problem.responders.values()[0]
self.assertTrue(responder.compare_answer('48', '8*6'))
self.assertFalse(responder.compare_answer('48', '9*5'))

def test_validate_answer(self):
"""Tests the answer validation function."""
problem = self.build_problem(answer="42")
responder = problem.responders.values()[0]
self.assertTrue(responder.validate_answer('23.5'))
self.assertFalse(responder.validate_answer('fish'))


class CustomResponseTest(ResponseTest):
from capa.tests.response_xml_factory import CustomResponseXMLFactory
Expand Down
Loading