evaluate/testme.py at master · marcoffee/evaluate · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
#!/usr/bin/env python3

import io
import os
import sys
import time
import queue
import shutil
import tempfile
import subprocess
import threading
import multiprocessing as mp

import config
import watch
import generate
import evaluate
import progress


def evaluate_thread (dirname, tid):
    real_stdout = sys.stdout
    real_stderr = sys.stderr

    sys.stdout = open(os.path.join(dirname, "{}.out".format(tid)), "w")
    sys.stderr = sys.stdout

    evaluate.main([])

    sys.stdout.close()
    sys.stderr = real_stderr
    sys.stdout = real_stdout

def progress_thread ():
    progress.main([ "-refresh", "1" ])

def watch_function ():
    prog = progress.main([ "-report", "-no-print" ])

    tail = subprocess.run(
        [ "tail", "-n", "20", config.files.log ],
        stdout = subprocess.PIPE, stderr = subprocess.STDOUT,
        universal_newlines = True
    ).stdout

    return "{}\n\n{}".format(prog, tail)

def watch_stop (wqueue):
    def func ():
        value = False

        try:
            value = wqueue.get(False)
        except queue.Empty:
            pass

        return value

    return func

def watch_thread (wqueue):
    watch.watch(watch_function, run_until=watch_stop(wqueue))

def main ():
    dirname = tempfile.mkdtemp(prefix="test_", dir=".")
    config.set_task_path(dirname)

    config.defaults = [
        ( "param1"   , "a" ),
        ( "-param2"  , 1 ),
        ( "-param3"  , 1.0 ),
        ( "-param4"  , "a 1 1.0" ),
        ( "-param5"  , config.ENABLE ),
        ( "-param6"  , "fixed" ),
        ( "-param7"  , "1" )
    ]

    config.tests = [(
        ( "param1"   , [ "a", "b", "c", "d", "e" ] ),
        ( "-param2"  , [ 1, 2, 3, 4, 5 ] ),
        ( "-param3"  , [ 1.0, 1.1, 1.2, 1.3, 1.4 ] ),
        ( "-param4"  , [ "1 2 3", "a b c", "1 a bla" ] ),
        ( "-param5"  , [ config.ENABLE, config.DISABLE ] ),
        ( "-param7"  , [ "1", "2", "3" ] ),
    ), (
        ( "param1"   , [ "e", "f", "g", "h" ] ),
        ( "-param2"  , [ 1, 2, 3, 4, 5 ] ),
        ( "-param3"  , [ 1.0, 1.5, 2.0, 2.5, 3.0 ] ),
        ( "-param4"  , [ "1 2 3", "a b c", "1 a bla" ] ),
        ( "-param5"  , [ config.ENABLE, config.DISABLE ] ),
        ( "-param7"  , [ "1", "2", "4" ] ),
    )]

    eval_path = os.path.join(dirname, "evals")
    os.makedirs(eval_path, exist_ok=True)

    workers = []

    generate.main([])

    prog = mp.Process(target=progress_thread)
    prog.start()

    wqueue = queue.Queue()

    wat = threading.Thread(target=watch_thread, args=( wqueue, ))
    wat.start()

    for i in range(100):
        wrk = mp.Process(target=evaluate_thread, args=( eval_path, i ))
        wrk.start()
        workers.append(wrk)

    try:
        prog.join()

        for wrk in workers:
            wrk.join()

        time.sleep(1)

        wqueue.put(True)
        wat.join()

    except KeyboardInterrupt:
        print()
        pass

    finally:
        prog.terminate()

        for wrk in workers:
            wrk.terminate()

        wqueue.put(True)
        wat.join()

        print("test saved to", dirname)

if __name__ == "__main__":
    main()