diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/tester.py | 404 |
1 files changed, 404 insertions, 0 deletions
diff --git a/tools/tester.py b/tools/tester.py new file mode 100755 index 0000000..7f48400 --- /dev/null +++ b/tools/tester.py @@ -0,0 +1,404 @@ +#!/usr/bin/env python3 + +import argparse +import glob +import os +import sys +import subprocess +import shutil + +class Configuration: + def __init__(self, xic, testdir, spim, workdir, registers_description, plugin): + self.xic = xic + self.testdir = testdir + self.spim = spim + self.workdir = workdir + self.registers_description = registers_description + self.plugin = plugin + + def printself(self): + print('Configuration:') + print(' - xic: ', self.xic) + print(' - testdir: ', self.testdir) + print(' - spim: ', self.spim) + print(' - workdir: ', self.workdir) + if self.registers_description is not None: + print(' - regdescr:', self.registers_description) + if self.plugin: + print(' - plugin:', self.plugin) + +class TestOutput: + class Status: + COMPILER_FAILURE = 0 + COMPILER_SUCCES = 1 + SPIM_FAILURE = 2 + SPIM_SUCCESS = 3 + + def __init__(self): + self.status = None + self.compiler_stdout = None + self.compiler_stderr = None + self.compiler_ok = None + self.spim_stdout = None + self.spim_stderr = None + self.spim_ok = None + + +class TestInstrumentation: + def __init__(self, test): + self.test = test + self.instrumented = False + self.expected_output = [] + self.should_parse = True + self.stop_after = None + self.should_typecheck = True + self.typechecking_errors = [] + self.selftest = None + self.env = {} + + self.parse() + self.validate() + + def content(self): + # get content of all comments started with //@ + lines = open(self.test).readlines() + lines = [ line.strip() for line in lines ] + lines = [ line for line in lines if line.startswith("//@") ] + lines = [ line[3:] for line in lines ] + return lines + + def parse(self): + content = self.content() + self.instrumented = "PRACOWNIA" in content + if not self.instrumented: + raise Exception('Test instrumentation is missing: %s' % self.test) + + for line in content: + if line.startswith("out "): + self.expected_output.append(line[4:]) + elif line == "should_not_parse": + self.should_parse = False + elif line == "should_not_typecheck": + self.should_typecheck = False + elif line.startswith("tcError "): + self.typechecking_errors.append(line[len("tcError "):]) + elif line.startswith("stop_after"): + self.stop_after = line[len("stop_after "):] + elif line.startswith("env"): + keyvalue = line[len("env "):] + keyvalue = keyvalue.split('=') + self.env[keyvalue[0]] = keyvalue[1] + elif line.startswith('selftest'): + arg = line[len('selftest'):].strip() + if arg == 'pass': + self.selftest = True + elif arg == 'fail': + self.selftest = False + else: + raise Exception("invalid @selftest directive") + + elif line == "PRACOWNIA": + pass + else: + raise Exception("invalid test instrumentation: unknown directive: " + line) + + def validate(self): + if not self.instrumented: + return + + if not self.should_parse: + if len(self.expected_output) > 0: + raise Exception("test %s marked as @should_not_parse, but expected runtime output is specified (@out)" % self.test) + if len(self.typechecking_errors) > 0: + raise Exception("test %s marked as @should_not_parse, but expected typechecking errors are specified (@tcError)" % self.test) + if not self.should_typecheck: + raise Exception("test %s marked as @should_not_parse, but expected typechecking failure is marked (@should_not_typecheck)" % self.test) + + if not self.should_typecheck: + if len(self.expected_output) > 0: + raise Exception("test %s expects typechecking failure, but expected runtime output is specified (@out)" % self.test) + + if len(self.typechecking_errors): + if len(self.expected_output) > 0: + raise Exception("test %s expects typechecking errors, but expected runtime output is specified (@out)" % self.test) + self.should_typecheck = False + +class Test: + def __init__(self, test, instrumentation): + self.test = test + self.instrumentation = instrumentation + + def expecting_parsing_error(self): + return not self.instrumentation.should_parse + + def expecting_typechecking_error(self): + return not self.instrumentation.should_typecheck + + def expecting_compilation_failure(self): + return self.expecting_parsing_error() or self.expecting_typechecking_error() + + def expecting_runtime_output(self): + return not self.expecting_compilation_failure() and not self.instrumentation.stop_after + +class ExpectationMatcher: + def __init__(self, test , output): + self.test = test + self.output = output + + def __match_output(self, stdout, expected): + actual = list(reversed(stdout)) + expected = list(reversed(expected)) + + for i in range(0, len(expected)): + if len(actual) <= i: + # nie sprawdzam tego przed petla bo chcialbym aby najpierw zmatchowal te linijki + # co sie rzeczywiscie na stdout pojawily + return False, "program output is too short, it contains %u lines, while expected out has %u" % (len(actual), len(expected)) + + expected_line = expected[i] + actual_line = actual[i] + if expected_line != actual_line: + explanation = "mismatch on line (counting from bottom): %u\nexpected: %s\nactual: %s" % (i + 1, expected_line, actual_line) + return False, explanation + return True, "" + + def __real_match(self): + # print('self.test.expecting_compilation_failure()', self.test.expecting_compilation_failure()) + # print('self.test.instrumentation.should_typecheck', self.test.instrumentation.should_typecheck) + # print('self.output.compiler_ok', self.output.compiler_ok) + if self.test.expecting_compilation_failure(): + xic_stderr = self.output.compiler_stderr.decode('utf8') + xic_stderr = xic_stderr.splitlines() + if len(xic_stderr) == 0: + xic_last_line_stderr = None + else: + xic_last_line_stderr = xic_stderr[-1].strip() + if self.output.compiler_ok: + return False, "expected compiler failure" + if self.test.expecting_parsing_error(): + if xic_last_line_stderr == "Failed: parser": + return True, "" + return False, "expected parsing error" + if self.test.expecting_typechecking_error(): + if xic_last_line_stderr == "Failed: typechecker": + return True, "" + return False, "expected typchecking error" + else: + if not self.output.compiler_ok: + return False, "program should be compiled, but compiler failed" + + if self.test.instrumentation.stop_after: + return True, "" + + if len(self.output.spim_stderr) > 0: + return False, "spim's stderr is not empty, execute program manually" + + + if self.test.expecting_runtime_output(): + if not self.output.spim_ok: + return False, "spim was not executed properly" + + if len(self.test.instrumentation.expected_output) > 0: + spim_stdout = self.output.spim_stdout.decode('utf8') + spim_stdout = spim_stdout.splitlines() + spim_stdout = [ line.strip() for line in spim_stdout ] + return self.__match_output(spim_stdout,self.test.instrumentation.expected_output) + + return True, "" + + return None, "cannot match test expectations" + + def match(self): + x_result, x_explanation = self.__real_match() + + if self.test.instrumentation.selftest == None: + result, explanation = x_result, x_explanation + else: + result = x_result == self.test.instrumentation.selftest + if result: + explanation = "" + else: + explanation = "selftest failed: expected %s, got: %s, %s" % (self.test.instrumentation.selftest, x_result, x_explanation) + + return result, explanation + +class TestRawExecutor: + + def __init__(self, conf, test, env, run_spim, stop_point): + self.conf = conf + self.test = test + self.env = env + self.output_file = os.path.join(conf.workdir, 'main.s') + self.test_output = TestOutput() + self.run_spim = run_spim + self.stop_point = stop_point + + def execute(self): + self.prepare_env() + ok, stdout, stderr = self.compile_program() + self.test_output.compiler_stdout = stdout + self.test_output.compiler_stderr = stderr + self.test_output.compiler_ok = ok + if not ok or not self.run_spim or self.stop_point: + self.clean_env() + return self.test_output + + ok, stdout, stderr = self.execute_program() + self.test_output.spim_stdout = stdout + self.test_output.spim_stderr = stderr + self.test_output.spim_ok = ok + self.clean_env() + + return self.test_output + + def compile_program(self): + xs = [self.conf.xic, '-o', self.output_file] + if self.stop_point: + xs.append('--stop-after') + xs.append(self.stop_point) + if self.conf.registers_description is not None: + xs.append('--registers-description') + xs.append(self.conf.registers_description) + if self.conf.plugin is not None: + xs.append('--plugin') + xs.append(self.conf.plugin) + + xs.append('--xi-log') + xs.append(os.path.join(self.conf.workdir, 'xilog')) + xs.append(self.test) + env = dict(self.env) + return self.__call(xs, env) + + def execute_program(self): + return self.__call([self.conf.spim, '-file', self.output_file]) + + def prepare_env(self): + shutil.rmtree(self.conf.workdir, ignore_errors=True) + os.makedirs(self.conf.workdir) + + def clean_env(self): + shutil.rmtree(self.conf.workdir, ignore_errors=True) + + def __call(self, xs, extenv={}): + env = os.environ + for k in extenv: + env[k] = extenv[k] + + try: + p = subprocess.Popen(xs, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + stdin, stdout = p.communicate(timeout=5) + status = p.returncode == 0 + return (status, stdin, stdout) + except subprocess.TimeoutExpired: + return (False, [], []) + except Exception: + # potem cos tu doklepie aby wykonywarka testow mogla sie kapnac, ze to nie test wykryl blad + # a cos innego + return (False, [], []) + +class TestExecutor: + def __init__(self, test, conf): + self.test = test + self.conf = conf + + def execute(self): + try: + run_spim = self.test.expecting_runtime_output() + stop_point = None + if not run_spim: + if self.test.expecting_parsing_error(): + stop_point = "parser" + elif self.test.expecting_typechecking_error(): + stop_point = "typechecker" + elif self.test.instrumentation.stop_after: + stop_point = self.test.instrumentation.stop_after + + rawExecutor = TestRawExecutor(self.conf, self.test.test, self.test.instrumentation.env, run_spim, stop_point) + test_output = rawExecutor.execute() + matcher = ExpectationMatcher(self.test, test_output) + return matcher.match() + except Exception as e: + raise e + return None, "internal error: " + str(e) + + +class TestRepository: + def __init__(self, testdirs): + self.tests = [] + self.collect_tests(testdirs) + + def collect_tests(self, testdirs): + testfiles = [] + for testdir in testdirs: + for path, _, files in os.walk(testdir): + for file in files: + if file.endswith(".xi"): + testfiles.append(os.path.join(path, file)) + testfiles = list(sorted(testfiles)) + + for testfile in testfiles: + instrumentation = TestInstrumentation(testfile) + test = Test(testfile, instrumentation) + self.tests.append(test) + + def gen(self): + for t in self.tests: + yield t + +class Application: + def __init__(self): + args = self.create_argparse().parse_args() + self.conf = Configuration(xic=args.xic, + testdir=args.testdir, + spim=args.spim, + workdir=args.workdir, + registers_description=args.registers_description, + plugin=args.plugin) + + + def create_argparse(self): + parser = argparse.ArgumentParser(description='Xi tester') + parser.add_argument('--xic', help='path to xi binary', default='./_build/install/default/bin/xi', type=str) + parser.add_argument('--spim', help='path to spim binary', default='spim', type=str) + parser.add_argument('--testdir', help='path to test directory', default='./tests', type=str) + parser.add_argument('--workdir', help='working directory', default='workdir', type=str) + parser.add_argument('--registers-description', help='xi --registers-description', default=None, type=str) + parser.add_argument('--plugin', help='xi --plugin', type=str) + return parser + + def run(self): + print('Xi tester') + self.conf.printself() + self.test_repository = TestRepository([self.conf.testdir]) + passed_tests = [] + failed_tests = [] + inconclusive_tests = [] + for test in self.test_repository.gen(): + print('==> running test', test.test) + executor = TestExecutor(test, self.conf) + result, explanation = executor.execute() + + if result == None: + inconclusive_tests.append(test) + status = "inconclusive: " + explanation + elif result: + passed_tests.append(test) + status = "pass" + elif not result: + failed_tests.append(test) + status = "fail: " + explanation + + print('--- result:', status) + + total = len(passed_tests) + len(failed_tests) + len(inconclusive_tests) + + print('===================') + print('Total: ', total) + print('Passed:', len(passed_tests)) + print('Inconc:', len(inconclusive_tests)) + print('Failed:', len(failed_tests)) + for test in failed_tests: + print(' -', test.test) + + +Application().run() |