From 7605c0a974241adc9491ba8ce5aa1390dc050966 Mon Sep 17 00:00:00 2001 From: Greg Hamerly Date: Fri, 6 Jan 2023 16:02:25 -0600 Subject: [PATCH 1/3] if there are no sub-results, do not call a grader Instead, return the default verdict (AC) and score (0). Empty groups should only occur when using -d to filter out some data; we may want to warn about real empty groups somewhere else. --- problemtools/verifyproblem.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 02bb4a9b..8200a8a4 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -1320,6 +1320,10 @@ def grade(self, sub_results, testcasegroup, shadow_result=False): verdict = 'AC' score = 0 + if not sub_results: + self.info('No results on %s, so no graders ran' % (testcasegroup,)) + return (verdict, score) + grader_flags = testcasegroup.config['grader_flags'].split() self.debug('Grading %d results:\n%s' % (len(sub_results), grader_input)) self.debug('Grader flags: %s' % grader_flags) From fa6ed3f70756bd443be0d6f12b5bc1c6b6a0d46e Mon Sep 17 00:00:00 2001 From: Greg Hamerly Date: Fri, 6 Jan 2023 17:00:10 -0600 Subject: [PATCH 2/3] default_grader: return score with JE The specification says that the verdict must come with a score. Even though a score does not make sense for a judge error verdict, this sticks with the specification, and prevents the caller (verifyproblem) from crashing. --- support/default_grader/default_grader | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support/default_grader/default_grader b/support/default_grader/default_grader index 3da8247a..b0335cf7 100755 --- a/support/default_grader/default_grader +++ b/support/default_grader/default_grader @@ -68,4 +68,4 @@ try: score = aggregate_scores(scores) print('%s %f' % (verdict, score)) except: - print('JE') + print('JE 0') From 1ac5fc8495746a3a7a65f762e59c596c843ea9fa Mon Sep 17 00:00:00 2001 From: Greg Hamerly Date: Sat, 7 Jan 2023 13:51:32 -0600 Subject: [PATCH 3/3] report an error if there are any empty test case groups --- problemtools/verifyproblem.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 8200a8a4..ce6c0481 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -497,6 +497,9 @@ def check(self, args): if not f[:-4] + '.in' in infiles: self.error("No matching input file for answer '%s'" % f) + if not self.get_subgroups() and not self.get_testcases(): + self.error('Test case group is empty') + # Check whether a <= b according to a natural sorting where numeric components # are compactified, so that e.g. "a" < "a1" < "a2" < "a10" = "a010" < "a10a". def natural_sort_le(a, b):