source: framspy/evolalg/tests/test_diferent_settings.py @ 1291

Last change on this file since 1291 was 1291, checked in by Maciej Komosinski, 6 weeks ago

Cosmetic

File size: 9.6 KB
Line 
1import argparse
2import sys
3import numpy as np
4
5from FramsticksLib import FramsticksLib
6
7from ..frams_base.experiment_frams_niching import ExperimentFramsNiching
8from ..frams_base.experiment_frams_islands import ExperimentFramsIslands
9from ..numerical_example.numerical_example import ExperimentNumerical
10from ..numerical_example.numerical_islands_example import ExperimentNumericalIslands
11
12from ..utils import ensureDir
13
14
15GENERATIONS = 10
16
17SETTINGS_TO_TEST_NUMERIC = {
18    'hof_size': [0, 10],
19    'popsize': [8],
20    'archive': [8],
21    'pmut': [0.7],
22    'pxov': [0.2],
23    'tournament': [5],
24    'initialgenotype':[np.array([100, 100, 100, 100]), [-100,-100]]
25}
26
27SETTINGS_TO_TEST_NUMERIC_ISLAND = {
28    'hof_size': [0, 10],
29    'popsize': [8],
30    'archive': [8],
31    'pmut': [0.7],
32    'pxov': [0.2],
33    'tournament': [5],
34    'migration_interval': [1,5],
35    'number_of_populations':[1,5],
36    'initialgenotype':[np.array([100, 100, 100, 100]), [-100,-100]]
37}
38
39SETTINGS_TO_TEST_FRAMS_NICHING = {
40    'opt': ['velocity', 'vertpos'],
41    'max_numparts': [None],
42    'max_numjoints': [20],
43    'max_numneurons': [20],
44    'max_numconnections': [None],
45    'max_numgenochars': [20],
46    'hof_size': [0, 10],
47    'normalize': ['none', 'max', 'sum'],
48    'dissim': [-2, -1, 1, 2],
49    'fit': ['niching', 'novelty', 'nsga2', 'nslc', 'raw'],
50    'genformat': ['1'],
51    'popsize': [8],
52    'archive': [8],
53    'initialgenotype': [None],
54    'pmut': [0.7],
55    'pxov': [0.2],
56    'tournament': [5]
57}
58
59SETTINGS_TO_TEST_FRAMS_ISLANDS = {
60    'opt': ['velocity', 'vertpos'],
61    'max_numparts': [None],
62    'max_numjoints': [20],
63    'max_numneurons': [20],
64    'max_numconnections': [None],
65    'max_numgenochars': [20],
66    'hof_size': [0, 10],
67    'migration_interval': [1,5],
68    'number_of_populations':[1,5],
69    'genformat': ['1'],
70    'popsize': [8],
71    'initialgenotype': [None],
72    'pmut': [0.7],
73    'pxov': [0.2],
74    'tournament': [5]
75}
76
77def test_run_experiment_numerical(params):
78    # multiple criteria not supported here. If needed, use FramsticksEvolution.py
79
80    experiment = ExperimentNumerical(
81                                    hof_size=params['hof_size'],
82                                    popsize=params['popsize'],
83                                    save_only_best=True,)
84
85    experiment.evolve(hof_savefile=None,
86                      generations=GENERATIONS,
87                      initialgenotype=params['initialgenotype'],
88                      pmut=params['pmut'],
89                      pxov=params['pxov'],
90                      tournament_size=params['tournament'])
91
92
93def test_run_experiment_numerical_islands(params):
94    # multiple criteria not supported here. If needed, use FramsticksEvolution.py
95
96    experiment = ExperimentNumericalIslands(hof_size=params['hof_size'],
97                                    popsize=params['popsize'],
98                                    save_only_best=True,
99                                    migration_interval=params['migration_interval'],
100                                    number_of_populations=params['number_of_populations'])
101
102
103    experiment.evolve(hof_savefile=None,
104                      generations=GENERATIONS,
105                      initialgenotype=params['initialgenotype'],
106                      pmut=params['pmut'],
107                      pxov=params['pxov'],
108                      tournament_size=params['tournament'])
109
110def test_run_experiment_frams_niching(params):
111    # multiple criteria not supported here. If needed, use FramsticksEvolution.py
112    opt_criteria = params['opt'].split(",")
113    framsLib = FramsticksLib(
114        parsed_args.path, parsed_args.lib, parsed_args.sim)
115    constrains = {"max_numparts": params['max_numparts'],
116                  "max_numjoints": params['max_numjoints'],
117                  "max_numneurons": params['max_numneurons'],
118                  "max_numconnections": params['max_numconnections'],
119                  "max_numgenochars": params['max_numgenochars'],
120                  }
121
122    experiment = ExperimentFramsNiching(frams_lib=framsLib,
123                                        optimization_criteria=opt_criteria,
124                                        hof_size=params['hof_size'],
125                                        constraints=constrains,
126                                        normalize=params['normalize'],
127                                        dissim=params['dissim'],
128                                        fit=params['fit'],
129                                        genformat=params['genformat'],
130                                        popsize=params['popsize'],
131                                        archive_size=params['archive'],
132                                        save_only_best=True,
133                                        knn_niching=5,
134                                        knn_nslc=5)
135
136    experiment.evolve(hof_savefile=None,
137                      generations=GENERATIONS,
138                      initialgenotype=params['initialgenotype'],
139                      pmut=params['pmut'],
140                      pxov=params['pxov'],
141                      tournament_size=params['tournament'])
142
143def test_run_experiment_frams_island(params):
144    # multiple criteria not supported here. If needed, use FramsticksEvolution.py
145    opt_criteria = params['opt'].split(",")
146    framsLib = FramsticksLib(
147        parsed_args.path, parsed_args.lib, parsed_args.sim)
148    constrains = {"max_numparts": params['max_numparts'],
149                  "max_numjoints": params['max_numjoints'],
150                  "max_numneurons": params['max_numneurons'],
151                  "max_numconnections": params['max_numconnections'],
152                  "max_numgenochars": params['max_numgenochars'],
153                  }
154
155    experiment = ExperimentFramsIslands(frams_lib=framsLib,
156                                        optimization_criteria=opt_criteria,
157                                        hof_size=params['hof_size'],
158                                        constraints=constrains,
159                                        genformat=params['genformat'],
160                                        popsize=params['popsize'],
161                                        migration_interval=params['migration_interval'],
162                                        number_of_populations=params['number_of_populations'],
163                                        save_only_best=True)
164
165    experiment.evolve(hof_savefile=None,
166                      generations=GENERATIONS,
167                      initialgenotype=params['initialgenotype'],
168                      pmut=params['pmut'],
169                      pxov=params['pxov'],
170                      tournament_size=params['tournament'])
171
172
173def parseArguments():
174    parser = argparse.ArgumentParser(
175        description='Run this program with "python -u %s" if you want to disable buffering of its output.' % sys.argv[0])
176    parser.add_argument('-path', type=ensureDir, required=True,
177                        help='Path to Framsticks CLI without trailing slash.')
178    parser.add_argument('-lib', required=False,
179                        help='Library name. If not given, "frams-objects.dll" or "frams-objects.so" is assumed depending on the platform.')
180    parser.add_argument('-sim', required=False, default="eval-allcriteria.sim",
181                        help="The name of the .sim file with settings for evaluation, mutation, crossover, and similarity estimation. If not given, \"eval-allcriteria.sim\" is assumed by default. Must be compatible with the \"standard-eval\" expdef. If you want to provide more files, separate them with a semicolon ';'.")
182
183    return parser.parse_args()
184
185
186def get_params_sets(settings):
187    params_sets = []
188    for k in settings.keys():
189        temp_param_set = []
190        for value in settings[k]:
191            if params_sets:
192                for exsiting_set in params_sets:
193                    copy_of_set = exsiting_set.copy()
194                    copy_of_set[k] = value
195                    temp_param_set.append(copy_of_set)
196            else:
197                temp_param_set.append({k: value})
198        params_sets = temp_param_set
199    return params_sets
200
201
202def cover_to_test(params, run_exp):
203    try:
204        run_exp(params)
205        return [1, None]
206    except Exception as ex:
207        return [0, f"Experiment {run_exp.__name__} with params: {params} failed with the stack: {ex}"]
208
209
210def run_tests():
211    result = []
212    print("TESTING NUMERICAL")
213    params_sets = get_params_sets(SETTINGS_TO_TEST_NUMERIC)
214    print(f"Starting executing {len(params_sets)} experiments")
215    result.extend([cover_to_test(params,test_run_experiment_numerical) for params in params_sets])
216
217    print("TESTING NUMERICAL ISLANDS")
218    params_sets = get_params_sets(SETTINGS_TO_TEST_NUMERIC_ISLAND)
219    print(f"Starting executing {len(params_sets)} experiments")
220    result.extend([cover_to_test(params,test_run_experiment_numerical_islands) for params in params_sets])
221
222    print("TESTING FRAMS NICHING")
223    params_sets = get_params_sets(SETTINGS_TO_TEST_FRAMS_NICHING)
224    print(f"Starting executing {len(params_sets)} experiments")
225    result.extend([cover_to_test(params, test_run_experiment_frams_niching) for params in params_sets])
226
227    print("TESTING FRAMS ISLANDS")
228    params_sets = get_params_sets(SETTINGS_TO_TEST_FRAMS_ISLANDS)
229    print(f"Starting executing {len(params_sets)} experiments")
230    result.extend([cover_to_test(params,test_run_experiment_frams_island) for params in params_sets])
231
232    Successful = sum([1 for r in result  if r[0] == 1])
233    Failed = sum([1 for r in result if r[0] == 0])
234    print(f"{Successful} out of {len(result)} passed")
235    if Successful < Failed:
236        print("Experiments that failed :{Failed}")
237        [print(r[1]) for r in result if r[1]]
238
239
240if __name__ == "__main__":
241    parsed_args = parseArguments()
242    run_tests()
Note: See TracBrowser for help on using the repository browser.