source: tester/tester.py @ 903

Last change on this file since 903 was 903, checked in by Maciej Komosinski, 6 months ago

Tester no longer exits on popular exceptions like 'file not found'

  • Property svn:eol-style set to native
File size: 10.9 KB
Line 
1import os, os.path, sys, platform, re, copy
2import traceback # for custom printing of exception trace/stack
3import errno  # for delete_file_if_present()
4import argparse
5from subprocess import Popen, PIPE
6from time import sleep
7import telnetlib
8
9import comparison  # our source
10import globals  # our source
11
12
13def test(args, test_name, input, output_net, output_msg, exe_prog, exeargs):
14        print(test_name, end=" ")
15        command = prepare_exe_with_name(exe_prog)
16        command += exeargs
17        if len(output_net) > 0:
18                command += globals.EXENETMODE
19        if args.valgrind:
20                command = globals.EXEVALGRINDMODE + command
21
22        p = Popen(command, stdout=PIPE, stderr=PIPE, stdin=PIPE)
23
24        if len(output_net) > 0:
25                sleep(10 if args.valgrind else 1)  # time for the server to warm up
26                tn = telnetlib.Telnet("localhost", 9009)
27                tn.write(bytes(input, "UTF-8"))
28                sleep(2)  # time for the server to respond...
29                # if we had a command in the frams server protocol to close the connection gracefully, then we could use read_all() instead of the trick with sleep()+read_very_eager()+close()
30                stdnet = tn.read_very_eager().decode().split("\n")  # the server uses "\n" as the end-of-line character on each platform
31                tn.close()  # after this, the server is supposed to close by itself (the -N option)
32                input = ""
33        # under Windows, p.stderr.read() and p.stdout.read() block while the process works, under linux it may be different
34        # http://stackoverflow.com/questions/3076542/how-can-i-read-all-availably-data-from-subprocess-popen-stdout-non-blocking?rq=1
35        # http://stackoverflow.com/questions/375427/non-blocking-read-on-a-subprocess-pipe-in-python
36        # p.terminate() #this was required when the server did not have the -N option
37        # stderrdata=p.stderr.read() #fortunately it is possible to reclaim (a part of?) stream contents after the process is killed... under Windows this is the ending of the stream
38
39        (stdoutdata, stderrdata) = p.communicate(bytes(input, "UTF-8"))  # the server process ends...
40        stdoutdata = stdoutdata.decode()  # bytes to str
41        stderrdata = stderrdata.decode()  # bytes to str
42        # p.stdin.write(we) #this is not recommended because buffers can overflow and the process will hang up (and indeed it does under Windows) - so communicate() is recommended
43        # stdout = p.stdout.read()
44        # p.terminate()
45
46        # print repr(input)
47        # print repr(stdoutdata)
48
49        stdout = stdoutdata.split(os.linesep)
50        # print stdout
51        stderr = stderrdata.split(os.linesep)
52        ok = check(stdnet if len(output_net) > 0 else stdout, output_list if len(output_list) > 0 else output_net, output_msg)
53
54        if p.returncode != 0 and p.returncode is not None:
55                print("  ", p.returncode, "<- returned code")
56
57        if len(stderrdata) > 0:
58                print("   (stderr has %d lines)" % len(stderr))
59                # valgrind examples:
60                # ==2176== ERROR SUMMARY: 597 errors from 50 contexts (suppressed: 35 from 8)
61                # ==3488== ERROR SUMMARY: 0 errors from 0 contexts (suppressed: 35 from 8)
62                if (not args.valgrind) or ("ERROR SUMMARY:" in stderrdata and " 0 errors" not in stderrdata) or (args.always_show_stderr):
63                        print(stderrdata)
64
65        if not ok and args.stop:
66                sys.exit("First test failure, stopping early.")
67        return ok
68
69
70def compare(jest, goal, was_compared_to):
71        p = comparison.Comparison(jest, goal)
72        if p.ok:
73                print("\r", globals.ANSI_SETGREEN + " ok" + globals.ANSI_RESET)
74        else:
75                print("\r", globals.ANSI_SETRED + " FAIL\7" + globals.ANSI_RESET)
76                print(p.result)
77                f = open(os.path.join(globals.THISDIR, '_last_failed' + was_compared_to + '.output'), 'w')  # files are easier to compare than stdout
78                print('\n'.join(jest), end="", file=f)
79                f = open(os.path.join(globals.THISDIR, '_last_failed' + was_compared_to + '.goal'), 'w')  # files are easier to compare than stdout
80                print('\n'.join(goal), end="", file=f)
81        return p.ok
82
83
84def remove_prefix(text, prefix):
85        return text[len(prefix):] if text.startswith(prefix) else text
86
87
88def check(stdout, output_net, output_msg):
89        actual_out_msg = []
90        if len(output_net) > 0:  # in case of the server, there is no filtering
91                for line in stdout:
92                        actual_out_msg.append(line)
93                return compare(actual_out_msg, output_net, '')
94        else:
95                FROMSCRIPT = "Script.Message: "
96                beginnings = tuple(["[" + v + "] " for v in ("INFO", "WARN", "ERROR", "CRITICAL")])  # there is also "DEBUG"
97                header_begin = 'VMNeuronManager.autoload: Neuro classes added: '  # header section printed when the simulator is created
98                header_end = "UserScripts.autoload: "  # ending of the header section
99                now_in_header = False
100                for line in stdout:
101                        if now_in_header:
102                                if header_end in line:  # "in" because multithreaded simulators prefix their messages with their numerical id, e.g. #12/...
103                                        now_in_header = False
104                                continue
105                        else:
106                                if header_begin in line:  # as above
107                                        now_in_header = True
108                                        continue
109                                line = remove_prefix(line, beginnings[0])  # cut out [INFO], other prefixes we want to leave as they are
110                                line = remove_prefix(line, FROMSCRIPT)  # cut out FROMSCRIPT
111                                actual_out_msg.append(line)
112                if actual_out_msg[-1] == '':  # empty line at the end which is not present in our "goal" contents
113                        actual_out_msg.pop()
114                return compare(actual_out_msg, output_msg, '')
115
116
117def delete_file_if_present(filename):
118        print('"%s" (%s)' % (filename, "the file was present" if os.path.exists(filename) else "this file did not exist"))
119        try:
120                os.remove(filename)
121        except OSError as e:
122                if e.errno != errno.ENOENT:  # errno.ENOENT = no such file or directory
123                        raise  # re-raise exception if a different error occurred
124
125
126def reset_values():
127        global input_text, output_net, output_msg, test_name, ini, output_list, exeargs, exe_prog
128        input_text = ""
129        output_list = []
130        ini = ""
131        output_net, output_msg = [], []
132        exeargs = []
133        test_name = "no-name test"
134
135
136def is_test_active():
137        global test_name
138        if name_template == "":
139                return True
140        if re.match(name_template, test_name):
141                return True
142        return False
143
144
145def prepare_exe_with_name(name):
146        if name in globals.EXENAMES:
147                exename = copy.copy(globals.EXENAMES[name])  # without copy, the following modifications would change values in the EXENAMES table
148        else:
149                exename = [name]
150        for rule in globals.EXERULES:
151                exename[0] = re.sub(rule[0], rule[1], exename[0])
152        return exename
153
154
155def print_exception():
156        print("\n"+("-"*60),'begin exception')
157        traceback.print_exc()
158        print("-"*60,'end exception')
159
160
161def main():
162        global input_text, name_template, test_name, exe_prog, exeargs
163        name_template = ""
164        exeargs = []
165
166        parser = argparse.ArgumentParser()
167        parser.add_argument("-val", "--valgrind", help="Use valgrind", action="store_true")
168        parser.add_argument("-c", "--nocolor", help="Don't use color output", action="store_true")
169        parser.add_argument("-f", "--file", help="File name with tests", required=True)
170        parser.add_argument("-tp", "--tests-path", help="tests directory, files containing test definitions, inputs and outputs are relative to this directory, default is '" + globals.THISDIR + "'")
171        parser.add_argument("-fp", "--files-path", help="files directory, files tested by OUTFILECOMPARE are referenced relative to this directory, default is '" + globals.FILESDIR + "'")
172        parser.add_argument("-wp", "--working-path", help="working directory, test executables are launched after chdir to this directory, default is '" + globals.EXEDIR + "'")
173        parser.add_argument("-n", "--name", help="Test name (regexp)")  # e.g. '^((?!python).)*$' = these tests which don't have the word "python" in their name
174        parser.add_argument("-s", "--stop", help="Stops on first difference", action="store_true")
175        parser.add_argument("-ds", "--diffslashes", help="Discriminate between slashes (consider / and \\ different)", action="store_true")
176        parser.add_argument("-err", "--always-show-stderr", help="Always print stderr (by default it is hidden if 0 errors in valgrind)", action="store_true")
177        parser.add_argument("-e", "--exe", help="Regexp 'search=replace' rule(s) transforming executable name(s) into paths (eg. '(.*)=path/to/\\1.exe')", action='append')  # in the example, double backslash is just for printing
178        parser.add_argument("-p", "--platform", help="Override platform identifier (referencing platform specific files " + globals.SPEC_INSERTPLATFORMDEPENDENTFILE + "), default:sys.platform (win32,linux2)")
179        args = parser.parse_args()
180        if args.valgrind:
181                print("Using valgrind...")
182        if args.diffslashes:
183                globals.DIFFSLASHES = args.diffslashes
184        if args.file:
185                main_test_filename = args.file
186        if args.tests_path:
187                globals.THISDIR = args.tests_path
188        if args.files_path:
189                globals.FILESDIR = args.files_path
190        if args.working_path:
191                globals.EXEDIR = args.working_path
192        if args.name:
193                name_template = args.name
194        if args.exe:
195                for e in args.exe:
196                        search, replace = e.split('=', 1)
197                        globals.EXERULES.append((search, replace))
198        if args.platform:
199                globals.PLATFORM = args.platform
200
201        os.chdir(globals.EXEDIR)
202
203        globals.init_colors(args)
204
205        fin = open(os.path.join(globals.THISDIR, args.file))
206        reset_values()
207        exe_prog = "default"  # no longer in reset_values (exe: persists across tests)
208        outfile = []
209        tests_failed = 0
210        tests_total = 0
211        for line in fin:
212                line = globals.stripEOL(line)
213                if len(line) == 0 or line.startswith("#"):
214                        continue
215                line = line.split(":", 1)
216                # print line
217                command = line[0]
218                if command == "TESTNAME":
219                        reset_values()
220                        test_name = line[1]
221                elif command == "arg":
222                        exeargs.append(line[1])
223                elif command == "exe":
224                        exe_prog = line[1]
225                elif command == "in":
226                        input_text += line[1] + "\n"
227                elif command == "out-net":
228                        output_net.append(line[1])
229                elif command == "out-file":
230                        outfile.append(line[1])
231                elif command == "out-mesg":
232                        output_msg.append(line[1])
233                elif command == "out":
234                        output_list.append(line[1])
235                elif command == "DELETEFILENOW":
236                        if is_test_active():
237                                print("\t ", command, end=" ")
238                                delete_file_if_present(os.path.join(globals.FILESDIR, line[1]))
239                elif command == "OUTFILECLEAR":
240                        outfile = []
241                elif command == "OUTFILECOMPARE":
242                        if is_test_active():
243                                print("\t", command, '"%s"' % line[1], end=" ")
244                                try:
245                                        contents = []
246                                        with open(os.path.join(globals.FILESDIR, line[1]), 'r') as main_test_filename:
247                                                for line in main_test_filename:
248                                                        contents.append(globals.stripEOL(line))
249                                        ok = compare(contents, outfile, '_file')
250                                except Exception as e: # could also 'raise' for some types of exceptions if we wanted
251                                        print_exception()
252                                        ok = 0
253                                tests_failed += int(not ok)
254                                tests_total += 1
255                elif command == "RUNTEST":
256                        if is_test_active():
257                                print("\t", command, end=" ")
258                                try:
259                                        ok = test(args, test_name, input_text, output_net, output_msg, exe_prog, exeargs)
260                                except Exception as e: # could also 'raise' for some types of exceptions if we wanted
261                                        print_exception()
262                                        ok = 0
263                                tests_failed += int(not ok)
264                                tests_total += 1
265                else:
266                        raise Exception("Don't know what to do with this line in test file: ", line)
267
268        return (tests_failed, tests_total)
269
270
271if __name__ == "__main__":
272        tests_failed, tests_total = main()
273        print("%d / %d failed tests" % (tests_failed, tests_total))
274        sys.exit(tests_failed)  # return the number of failed tests as exit code ("error level") to shell
Note: See TracBrowser for help on using the repository browser.