forked from datacamp/pythonwhat
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_spec.py
More file actions
254 lines (216 loc) · 9.88 KB
/
test_spec.py
File metadata and controls
254 lines (216 loc) · 9.88 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
import unittest
import helper
class TestFChain(unittest.TestCase):
def setUp(self):
self.data = {
"DC_PEC": '',
"DC_SOLUTION": '''
[[ii+1 for ii in range(aa)] for aa in range(2)]
[[ii*2 for ii in range(bb)] for bb in range(1,3)]
'''
}
self.data["DC_CODE"] = self.data["DC_SOLUTION"]
def test_F(self):
self.data["DC_SCT"] = '''
list_comp = F().check_list_comp(0).check_body().set_context(ii=2).has_equal_value('unequal')
Ex().check_list_comp(0).check_body().set_context(aa=2).multi(list_comp)
'''
sct_payload = helper.run(self.data)
self.assertTrue(sct_payload['correct'])
def test_check_to_F(self):
self.data["DC_SCT"] = '''
list_comp = check_list_comp(0).check_body().set_context(ii=2).has_equal_value('unequal')
Ex().check_list_comp(0).check_body().set_context(aa=2).multi(list_comp)
'''
def test_check_to_F_nested(self):
self.data["DC_SCT"] = '''
# funky, but we're testing nested check functions!
multi_test = multi(check_list_comp(0).check_body().set_context(aa=2).has_equal_value('badbody'))
Ex().multi(multi_test)
'''
sct_payload = helper.run(self.data)
self.assertTrue(sct_payload['correct'])
def test_F_reused(self):
self.data["DC_SCT"] = '''
list_comp = F().check_list_comp(0).check_body().set_context(ii=2).has_equal_value('unequal')
Ex().check_list_comp(0).check_body().set_context(aa=2).multi(list_comp)
Ex().check_list_comp(1).check_body().set_context(bb=4).multi(list_comp)
'''
sct_payload = helper.run(self.data)
self.assertTrue(sct_payload['correct'])
def test_F_extends(self):
self.data["DC_SCT"] = '''
list_comp = F().check_list_comp(0)
body_check = list_comp.check_body().set_context(aa=2).has_equal_value('unequal')
Ex().extend(body_check)\
.check_list_comp(0).check_body().set_context(ii=2).has_equal_value('unequal')
'''
sct_payload = helper.run(self.data)
self.assertTrue(sct_payload['correct'])
def test_F_assign_getattr(self):
self.data["DC_SCT"] = '''
eq_test = F().check_list_comp(0).check_body().has_equal_value
Ex().multi(eq_test('unequal'))
'''
class TestSpecInterop(unittest.TestCase):
def setUp(self):
self.data = {
"DC_PEC": '',
"DC_SOLUTION": '''[aa+1 for aa in range(2)]'''
}
self.data["DC_CODE"] = self.data["DC_SOLUTION"]
self.FAIL_CODE = '''[aa for aa in range(2)]'''
def test_spec1_in_multi_pass(self):
self.data["DC_SCT"] = '''
te = test_expression_result(extra_env={'aa':2}, incorrect_msg='unequal')
Ex().multi(test_list_comp(body=te)) # spec 1 inside multi
Ex().check_list_comp(0).check_body().multi(te) # half of each spec
Ex().check_list_comp(0).check_body().set_context(aa=2).has_equal_value('unequal') # full spec 2
test_list_comp(body=te) # full spec 1
'''
sct_payload = helper.run(self.data)
self.assertTrue(sct_payload['correct'])
def test_spec1_in_multi_fail(self):
# TODO: this test fails because spec1 tests are run after spec2 tests,
# even if they come first in the SCT script, due to building the tree
# for spec1 tests but not spec2 (which are run immediately)
self.data["DC_CODE"] = '''for aa in range(3): aa'''
self.data["DC_SCT"] = '''
test_list_comp(body=test_expression_result(expr_code = 'aa', incorrect_msg='unequal'))
Ex().check_list_comp(0).check_body().multi(test_expression_result(incorrect_msg='unequal'))
Ex().check_list_comp(0).check_body().has_equal_value('unequal')
'''
sct_payload = helper.run(self.data)
self.assertFalse(sct_payload['correct'])
def test_spec_run_order(self):
self.data["DC_CODE"] = '''[aa for aa in range(2)]'''
self.data["DC_SCT"] = '''
Ex().test_list_comp(body=test_expression_result(extra_env={'aa': 2}, incorrect_msg = 'spec1'))
Ex().check_list_comp(0).check_body().set_context(aa=2).has_equal_value('spec2')
'''
sct_payload = helper.run(self.data)
self.assertFalse(sct_payload['correct'])
self.assertIn('spec1', sct_payload['message'])
class TestMulti(unittest.TestCase):
def setUp(self):
self.data = {
"DC_PEC": '',
"DC_SOLUTION": '''[aa+1 for aa in range(2)]'''
}
self.data["DC_CODE"] = self.data["DC_SOLUTION"]
self.FAIL_CODE = '''[aa for aa in range(2)]'''
def test_nested_multi(self):
self.data["DC_SCT"] = '''
test_body = F().check_body().set_context(aa=2).has_equal_value('wrong')
Ex().check_list_comp(0).multi(F().multi(test_body))
'''
sct_payload = helper.run(self.data)
self.assertTrue(sct_payload['correct'])
def test_multi_splits_node_and_check(self):
self.data["DC_SCT"] = '''
test_body = F().check_list_comp(0).check_body().set_context(aa=2).has_equal_value('wrong')
Ex().check_list_comp(0).multi(F().check_body().set_context(aa=2).has_equal_value('wrong'))
'''
sct_payload = helper.run(self.data)
self.assertTrue(sct_payload['correct'])
def test_multi_generator(self):
self.data["DC_SCT"] = """
Ex().check_list_comp(0).check_body()\
.multi(set_context(aa=i).has_equal_value('wrong') for i in range(2))
"""
sct_payload = helper.run(self.data)
self.assertTrue(sct_payload['correct'])
class TestTestNot(unittest.TestCase):
def setUp(self):
self.data = {
"DC_SOLUTION": "x = 1",
"DC_CODE": "x = 1"
}
def test_pass(self):
self.data["DC_SCT"] = """Ex().test_not(check_list_comp(0), msg="no")"""
sct_payload = helper.run(self.data)
self.assertTrue(sct_payload['correct'])
def test_fail(self):
# obviously this would be a terrible sct...
self.data["DC_SCT"] = """Ex().test_not(test_object('x'), msg="no")"""
sct_payload = helper.run(self.data)
self.assertFalse(sct_payload['correct'])
class TestTestFail(unittest.TestCase):
def setUp(self):
self.data = {
"DC_SOLUTION": "", "DC_CODE": ""
}
def test_fail(self):
self.data["DC_SCT"] = """Ex().fail()"""
sct_payload = helper.run(self.data)
self.assertFalse(sct_payload['correct'])
class TestOverride(unittest.TestCase):
"""
This class is used to test overriding w/ correct and incorrect code. Tests are
run for entire nodes (e.g. an if block) and their parts (e.g. body of if block)
"""
def do_exercise(self, code, base_check, parts, override=None, part_name = None, part_index = "", passes=True):
"""High level function used to generate tests"""
if part_name:
if not override: override = parts[part_name]
sct = base_check + '.check_{}({}).override("""{}""").has_equal_ast()'\
.format(part_name, part_index, override)
else:
# whole code (e.g. if expression, or for loop)
if not override: override = code.format(**parts)
sct = base_check + '.override("""{}""").has_equal_ast()'.format(override)
data = {
"DC_SOLUTION": code.format(**parts),
"DC_CODE": code.format(**parts),
"DC_SCT": sct
}
sct_payload = helper.run(data)
self.assertTrue(sct_payload['correct']) if passes else self.assertFalse(sct_payload['correct'])
# used to generate tests
EXPRESSIONS = {
'if_exp': "{body} if {test} else {orelse}",
'list_comp': "[{body} for i in {iter}]",
'dict_comp': "{{ {key}: {value} for i in {iter} }}",
'for_loop': "for i in {iter}: {body}",
'while': "while {test}: {body}",
'try_except': "try: {body}\nexcept: pass\nelse: {orelse}",
'lambda_function': "lambda a={args}: {body}",
'function_def': ["'sum'", "def sum(a={args}): {body}"],
'function': ["'sum', 0", "sum({args})"]
}
PARTS = {'body': "1", "test": "False", 'orelse': "2", 'iter': "range(3)",
'key': "3", 'value': "4", 'args': "(1,2,3)"}
import re
def gen_exercise(*args, **kwargs):
return lambda self: TestOverride.do_exercise(self, *args, **kwargs)
for k, code in TestOverride.EXPRESSIONS.items():
# base SCT, w/ special indexing if function checks
if isinstance(code, list): indx, code = code
else: indx = '0'
base_check = "Ex().check_{}({})".format(k, indx)
# pass overall test ----
pf = gen_exercise(code, base_check, TestOverride.PARTS)
setattr(TestOverride, 'test_{}_pass'.format(k), pf)
# fail overall test ----
pf = gen_exercise(code, base_check, TestOverride.PARTS, override="'WRONG ANSWER'", passes=False)
setattr(TestOverride, 'test_{}_fail'.format(k), pf)
# test individual pieces --------------------------------------------------
for part in re.findall("\{([^{]*?)\}", code): # find all str.format vars, e.g. {body}
part_index = "" if part != 'args' else 0
# pass individual piece ----
test_name = 'test_{}_{}_pass'.format(k, part)
pf = gen_exercise(code, base_check, TestOverride.PARTS, part_name=part, part_index=part_index)
setattr(TestOverride, test_name, pf)
# fail individual piece ----
test_name = 'test_{}_{}_fail'.format(k, part)
bad_code = code.format(**{part: "[]", **TestOverride.PARTS})
pf = gen_exercise(code, base_check, TestOverride.PARTS, part_name=part, part_index=part_index, override=bad_code, passes=False)
setattr(TestOverride, test_name, pf)
#class TestSetContext(unittest.TestCase):
# def setUp(self):
# self.data = {
# "DC_PEC": "",
# "DC_SOLUTION": "[(i,j) for i,j in enumerate(range(10))]"
# }
if __name__ == "__main__":
unittest.main()