source: framspy/FramsticksLib.py @ 1200

Last change on this file since 1200 was 1200, checked in by Maciej Komosinski, 16 months ago

Make errors during evaluation of creatures Python exceptions so they are more noticeable and consequential

File size: 13.7 KB
Line 
1from typing import List  # to be able to specify a type hint of list(something)
2import json
3import sys, os
4import argparse
5import numpy as np
6import frams
7
8
9class FramsticksLib:
10        """Communicates directly with Framsticks library (.dll or .so or .dylib).
11        You can perform basic operations like mutation, crossover, and evaluation of genotypes.
12        This way you can perform evolution controlled by python as well as access and manipulate genotypes.
13        You can even design and use in evolution your own genetic representation implemented entirely in python,
14        or access and control the simulation and simulated creatures step by step.
15
16        Should you want to modify or extend this class, first see and test the examples in frams-test.py.
17
18        You need to provide one or two parameters when you run this class: the path to Framsticks where .dll/.so/.dylib resides
19        and, optionally, the name of the Framsticks dll/so/dylib (if it is non-standard). See::
20                FramsticksLib.py -h"""
21
22        PRINT_FRAMSTICKS_OUTPUT: bool = False  # set to True for debugging
23        DETERMINISTIC: bool = False  # set to True to have the same results in each run
24
25        GENOTYPE_INVALID = "/*invalid*/"  # this is how genotype invalidity is represented in Framsticks
26        EVALUATION_SETTINGS_FILE = [  # all files MUST be compatible with the standard-eval expdef. The order they are loaded in is important!
27                "eval-allcriteria.sim",  # a good trade-off in performance sampling period ("perfperiod") for vertpos and velocity
28                # "deterministic.sim",  # turns off random noise (added for robustness) so that each evaluation yields identical performance values (causes "overfitting")
29                # "sample-period-2.sim", # short performance sampling period so performance (e.g. vertical position) is sampled more often
30                # "sample-period-longest.sim",  # increased performance sampling period so distance and velocity are measured rectilinearly
31        ]
32
33
34        # This function is not needed because in Python, "For efficiency reasons, each module is only imported once per interpreter session."
35        # @staticmethod
36        # def getFramsModuleInstance():
37        #       """If some other party needs access to the frams module to directly access or modify Framsticks objects,
38        #       use this function to avoid importing the "frams" module multiple times and avoid potentially initializing
39        #       it many times."""
40        #       return frams
41
42        def __init__(self, frams_path, frams_lib_name, sim_settings_files):
43                if frams_lib_name is None:
44                        frams.init(frams_path)  # could add support for setting alternative directories using -D and -d
45                else:
46                        frams.init(frams_path, "-L" + frams_lib_name)  # could add support for setting alternative directories using -D and -d
47
48                print('Available objects:', dir(frams))
49                print()
50
51                print('Performing a basic test 1/2... ', end='')
52                simplest = self.getSimplest("1")
53                assert simplest == "X" and type(simplest) is str
54                print('OK.')
55                print('Performing a basic test 2/2... ', end='')
56                assert self.isValid(["X[0:0],", "X[0:0]", "X[1:0]"]) == [False, True, False]
57                print('OK.')
58                if not self.DETERMINISTIC:
59                        frams.Math.randomize()
60                frams.Simulator.expdef = "standard-eval"  # this expdef (or fully compatible) must be used by EVALUATION_SETTINGS_FILE
61                if sim_settings_files is not None:
62                        self.EVALUATION_SETTINGS_FILE = sim_settings_files # overwrite defaults
63                print('Using settings:', self.EVALUATION_SETTINGS_FILE)
64                assert isinstance(self.EVALUATION_SETTINGS_FILE, list)  # ensure settings file(s) are provided as a list
65               
66                for simfile in self.EVALUATION_SETTINGS_FILE:
67                        ec = frams.MessageCatcher.new()  # catch potential errors, warnings, messages - just to detect if there are ERRORs
68                        ec.store = 2; # store all, because they are caught by MessageCatcher and will not appear on console (which we want)
69                        frams.Simulator.ximport(simfile, 4 + 8 + 16)
70                        ec.close()
71                        print(ec.messages) # output all caught messages
72                        if ec.error_count._value() > 0:
73                                raise ValueError("Problem while importing file '%s'" % simfile) # make missing files or incorrect paths fatal because error messages are easy to overlook in output, and these errors would not prevent Framsticks simulator from performing genetic operations, starting and running in evaluate()
74
75
76        def getSimplest(self, genetic_format) -> str:
77                return frams.GenMan.getSimplest(genetic_format).genotype._string()
78
79
80        def evaluate(self, genotype_list: List[str]):
81                """
82                Returns:
83                        List of dictionaries containing the performance of genotypes evaluated using self.EVALUATION_SETTINGS_FILE.
84                        Note that for whatever reason (e.g. incorrect genotype), the dictionaries you will get may be empty or
85                        partially empty and may not have the fields you expected, so handle such cases properly.
86                """
87                assert isinstance(genotype_list, list)  # because in python, str has similar capabilities as list and here it would pretend to work too, so to avoid any ambiguity
88
89                if not self.PRINT_FRAMSTICKS_OUTPUT:
90                        ec = frams.MessageCatcher.new()  # mute potential errors, warnings, messages
91                        ec.store = 2; # store all, because they are caught by MessageCatcher and will not appear on console
92
93                frams.GenePools[0].clear()
94                for g in genotype_list:
95                        frams.GenePools[0].add(g)
96                frams.ExpProperties.evalsavefile = ""  # no need to store results in a file - we will get evaluations directly from Genotype's "data" field
97                frams.Simulator.init()
98                frams.Simulator.start()
99
100                # step = frams.Simulator.step  # cache reference to avoid repeated lookup in the loop (just for performance)
101                # while frams.Simulator.running._int():  # standard-eval.expdef sets running to 0 when the evaluation is complete
102                #       step()
103                frams.Simulator.eval("while(Simulator.running) Simulator.step();")  # fastest
104                # Timing for evaluating a single simple creature 100x:
105                # - python step without caching: 2.2s
106                # - python step with caching   : 1.6s
107                # - pure FramScript and eval() : 0.4s
108
109                if not self.PRINT_FRAMSTICKS_OUTPUT:
110                        ec.close()
111                        if ec.error_count._value() > 0:
112                                print(ec.messages) # if errors occurred, output all caught messages for debugging
113                                raise RuntimeError("[ERROR] %d error(s) and %d warning(s) while evaluating %d genotype(s)" % (ec.error_count._value(), ec.warning_count._value()-ec.error_count._value(), len(genotype_list))) # make errors fatal; by default they stop the simulation anyway so let's not use potentially incorrect or partial results and fix the cause first.
114
115                results = []
116                for g in frams.GenePools[0]:
117                        serialized_dict = frams.String.serialize(g.data[frams.ExpProperties.evalsavedata._value()])
118                        evaluations = json.loads(serialized_dict._string())  # Framsticks native ExtValue's get converted to native python types such as int, float, list, str.
119                        # now, for consistency with FramsticksCLI.py, add "num" and "name" keys that are missing because we got data directly from Genotype, not from the file produced by standard-eval.expdef's function printStats(). What we do below is what printStats() does.
120                        result = {"num": g.num._value(), "name": g.name._value(), "evaluations": evaluations}
121                        results.append(result)
122
123                return results
124
125
126        def mutate(self, genotype_list: List[str]) -> List[str]:
127                """
128                Returns:
129                        The genotype(s) of the mutated source genotype(s). self.GENOTYPE_INVALID for genotypes whose mutation failed (for example because the source genotype was invalid).
130                """
131                assert isinstance(genotype_list, list)  # because in python, str has similar capabilities as list and here it would pretend to work too, so to avoid any ambiguity
132
133                mutated = []
134                for g in genotype_list:
135                        mutated.append(frams.GenMan.mutate(frams.Geno.newFromString(g)).genotype._string())
136                assert len(genotype_list) == len(mutated), "Submitted %d genotypes, received %d validity values" % (len(genotype_list), len(mutated))
137                return mutated
138
139
140        def crossOver(self, genotype_parent1: str, genotype_parent2: str) -> str:
141                """
142                Returns:
143                        The genotype of the offspring. self.GENOTYPE_INVALID if the crossing over failed.
144                """
145                return frams.GenMan.crossOver(frams.Geno.newFromString(genotype_parent1), frams.Geno.newFromString(genotype_parent2)).genotype._string()
146
147
148        def dissimilarity(self, genotype_list: List[str], method: int) -> np.ndarray:
149                """
150                        :param method: -1 = genetic Levenshtein distance; 0, 1, 2 = phenetic dissimilarity (SimilMeasureGreedy, SimilMeasureHungarian, SimilMeasureDistribution)
151                        :return: A square array with dissimilarities of each pair of genotypes.
152                """
153                assert isinstance(genotype_list, list)  # because in python, str has similar capabilities as list and here it would pretend to work too, so to avoid any ambiguity
154
155                # if you want to override what EVALUATION_SETTINGS_FILE sets, you can do it below:
156                # frams.SimilMeasureHungarian.simil_partgeom = 1
157                # frams.SimilMeasureHungarian.simil_weightedMDS = 1
158
159                n = len(genotype_list)
160                square_matrix = np.zeros((n, n))
161
162                if method in (0, 1, 2):  # Framsticks phenetic dissimilarity methods
163                        frams.SimilMeasure.simil_type = method
164                        genos = []  # prepare an array of Geno objects so that we don't need to convert raw strings to Geno objects all the time in loops
165                        for g in genotype_list:
166                                genos.append(frams.Geno.newFromString(g))
167                        frams_evaluateDistance = frams.SimilMeasure.evaluateDistance  # cache function reference for better performance in loops
168                        for i in range(n):
169                                for j in range(n):  # maybe calculate only one triangle if you really need a 2x speedup
170                                        square_matrix[i][j] = frams_evaluateDistance(genos[i], genos[j])._double()
171                elif method == -1:
172                        import Levenshtein
173                        for i in range(n):
174                                for j in range(n):  # maybe calculate only one triangle if you really need a 2x speedup
175                                        square_matrix[i][j] = Levenshtein.distance(genotype_list[i], genotype_list[j])
176                else:
177                        raise Exception("Don't know what to do with dissimilarity method = %d" % method)
178
179                for i in range(n):
180                        assert square_matrix[i][i] == 0, "Not a correct dissimilarity matrix, diagonal expected to be 0"
181                non_symmetric_diff = square_matrix - square_matrix.T
182                non_symmetric_count = np.count_nonzero(non_symmetric_diff)
183                if non_symmetric_count > 0:
184                        non_symmetric_diff_abs = np.abs(non_symmetric_diff)
185                        max_pos1d = np.argmax(non_symmetric_diff_abs)  # location of the largest discrepancy
186                        max_pos2d_XY = np.unravel_index(max_pos1d, non_symmetric_diff_abs.shape)  # 2D coordinates of the largest discrepancy
187                        max_pos2d_YX = max_pos2d_XY[1], max_pos2d_XY[0]  # 2D coordinates of the largest discrepancy mirror
188                        worst_guy_XY = square_matrix[max_pos2d_XY]  # this distance and the other below (its mirror) are most different
189                        worst_guy_YX = square_matrix[max_pos2d_YX]
190                        print("[WARN] Dissimilarity matrix: expecting symmetry, but %g out of %d pairs were asymmetrical, max difference was %g (%g %%)" %
191                              (non_symmetric_count / 2,
192                               n * (n - 1) / 2,
193                               non_symmetric_diff_abs[max_pos2d_XY],
194                               non_symmetric_diff_abs[max_pos2d_XY] * 100 / ((worst_guy_XY + worst_guy_YX) / 2)))  # max diff is not necessarily max %
195                return square_matrix
196
197
198        def isValid(self, genotype_list: List[str]) -> List[bool]:
199                assert isinstance(genotype_list, list)  # because in python, str has similar capabilities as list and here it would pretend to work too, so to avoid any ambiguity
200                valid = []
201                for g in genotype_list:
202                        valid.append(frams.Geno.newFromString(g).is_valid._int() == 1)
203                assert len(genotype_list) == len(valid), "Tested %d genotypes, received %d validity values" % (len(genotype_list), len(valid))
204                return valid
205
206
207def parseArguments():
208        parser = argparse.ArgumentParser(description='Run this program with "python -u %s" if you want to disable buffering of its output.' % sys.argv[0])
209        parser.add_argument('-path', type=ensureDir, required=True, help='Path to the Framsticks library (.dll or .so or .dylib) without trailing slash.')
210        parser.add_argument('-lib', required=False, help='Library name. If not given, "frams-objects.dll" (or .so or .dylib) is assumed depending on the platform.')
211        parser.add_argument('-simsettings', required=False, help='The name of the .sim file with settings for evaluation, mutation, crossover, and similarity estimation. If not given, "eval-allcriteria.sim" is assumed by default. Must be compatible with the "standard-eval" expdef.')
212        parser.add_argument('-genformat', required=False, help='Genetic format for the demo run, for example 4, 9, or S. If not given, f1 is assumed.')
213        return parser.parse_args()
214
215
216def ensureDir(string):
217        if os.path.isdir(string):
218                return string
219        else:
220                raise NotADirectoryError(string)
221
222
223if __name__ == "__main__":
224        # A demo run.
225
226        # TODO ideas:
227        # - check_validity with three levels (invalid, corrected, valid)
228        # - a pool of binaries running simultaneously, balance load - in particular evaluation
229
230        parsed_args = parseArguments()
231        framsLib = FramsticksLib(parsed_args.path, parsed_args.lib, parsed_args.simsettings)
232
233        print("Sending a direct command to Framsticks library that calculates \"4\"+2 yields", frams.Simulator.eval("return \"4\"+2;"))
234
235        simplest = framsLib.getSimplest('1' if parsed_args.genformat is None else parsed_args.genformat)
236        print("\tSimplest genotype:", simplest)
237        parent1 = framsLib.mutate([simplest])[0]
238        parent2 = parent1
239        MUTATE_COUNT = 10
240        for x in range(MUTATE_COUNT):  # example of a chain of 10 mutations
241                parent2 = framsLib.mutate([parent2])[0]
242        print("\tParent1 (mutated simplest):", parent1)
243        print("\tParent2 (Parent1 mutated %d times):" % MUTATE_COUNT, parent2)
244        offspring = framsLib.crossOver(parent1, parent2)
245        print("\tCrossover (Offspring):", offspring)
246        print('\tDissimilarity of Parent1 and Offspring:', framsLib.dissimilarity([parent1, offspring], 1)[0, 1])
247        print('\tPerformance of Offspring:', framsLib.evaluate([offspring]))
248        print('\tValidity of Parent1, Parent 2, and Offspring:', framsLib.isValid([parent1, parent2, offspring]))
Note: See TracBrowser for help on using the repository browser.