NEAT Implemenation
The first step in our project was to create a basic NEAT implementation. NEAT stands for NeuroEvolution of Augmenting Topologies, which is a form of evolutionary machine learning developed by researchers at MIT. The result is a fast but structured machine learning algorithm.
Our Code:
NEAT Model Class:
1[NEAT]
2fitness_criterion = max
3fitness_threshold = 100000
4pop_size = 10
5reset_on_extinction = True
6
7[DefaultGenome]
8# node activation options
9activation_default = sigmoid
10activation_mutate_rate = 0.05
11activation_options = sigmoid gauss
12#abs clamped cube exp gauss hat identity inv log relu sigmoid sin softplus square tanh
13
14# node aggregation options
15aggregation_default = random
16aggregation_mutate_rate = 0.05
17aggregation_options = sum product min max mean median maxabs
18
19# node bias options
20bias_init_mean = 0.05
21bias_init_stdev = 1.0
22bias_max_value = 30.0
23bias_min_value = -30.0
24bias_mutate_power = 0.5
25bias_mutate_rate = 0.7
26bias_replace_rate = 0.1
27
28# genome compatibility options
29compatibility_disjoint_coefficient = 1.0
30compatibility_weight_coefficient = 0.5
31
32# connection add/remove rates
33conn_add_prob = 0.5
34conn_delete_prob = 0.1
35
36# connection enable options
37enabled_default = True
38enabled_mutate_rate = 0.2
39
40feed_forward = False
41#initial_connection = unconnected
42initial_connection = partial_nodirect 0.5
43
44# node add/remove rates
45node_add_prob = 0.5
46node_delete_prob = 0.5
47
48# network parameters
49num_hidden = 0
50num_inputs = 1120
51num_outputs = 12
52
53# node response options
54response_init_mean = 1.0
55response_init_stdev = 0.05
56response_max_value = 30.0
57response_min_value = -30.0
58response_mutate_power = 0.1
59response_mutate_rate = 0.75
60response_replace_rate = 0.1
61
62# connection weight options
63weight_init_mean = 0.1
64weight_init_stdev = 1.0
65weight_max_value = 30
66weight_min_value = -30
67weight_mutate_power = 0.5
68weight_mutate_rate = 0.8
69weight_replace_rate = 0.1
70
71[DefaultSpeciesSet]
72compatibility_threshold = 2.5
73
74[DefaultStagnation]
75species_fitness_func = max
76max_stagnation = 50
77species_elitism = 0
78
79[DefaultReproduction]
80elitism = 1
81survival_threshold = 0.3NEAT Agent Class:
1import sys
2import os
3
4script_dir = os.path.dirname(__file__)
5
6sys.path.append(script_dir + '/../agents')
7sys.path.append(script_dir + '/../interface')
8sys.path.append(script_dir + '/../learning')
9
10from agent_base import *
11from action_space import *
12from train_neat import *
13
14class NeatAgent(AgentBase):
15
16 def load(self, filename):
17 None
18 # TODO:
19
20 def save(self, filename):
21 None
22 # TODO:
23
24 def train(self):
25 train_neat()
26
27 def decide(self, obs, info) -> list:
28 buttons = ActionSpace.move_right()
29
30 return buttons
31
32 # Returns name of agent as a string
33 def name(self) -> str:
34 return "NeatAgent"
35
36 def to_string(self) -> str:
37 return self.name()NEAT Traing Class:
1##---------------Sources-------------------------##
2# Neat NN Implementation: https://gitlab.com/lucasrthompson/Sonic-Bot-In-OpenAI-and-NEAT
3# DeepQ Image Processing for GymRetro: https://github.com/deepanshut041/Reinforcement-Learning
4# Helper Functions for Gym Retro: https://github.com/moversti/sonicNEAT
5##-----------------------------------------------##
6
7import retro
8import numpy as np
9import cv2
10import neat
11import pickle
12import sys
13import os
14
15from configparser import Interpolation
16from inspect import getsourcefile
17
18from vision.greyImageViewer import GreyImageViewer
19from vision.controllerViewer import ControllerViewer
20
21script_dir = os.path.dirname(__file__)
22sys.path.append(script_dir + '/../agents')
23sys.path.append(script_dir + '/../interface')
24
25# Trains a NEAT NN.
26def train_neat():
27 env = retro.make(game="SonicTheHedgehog-Genesis", state="GreenHillZone.Act1", scenario="contest", record='.')
28 imgarray = []
29 xpos_end = 0
30
31 SEE_NETWORK_INPUT=True
32
33 resume = True
34 restore_file = "neat-checkpoint-32"
35
36 viewer = GreyImageViewer()
37 controllerViewer = ControllerViewer()
38
39 def eval_genomes(genomes, config):
40 for genome_id, genome in genomes:
41 ob = env.reset()
42 ac = env.action_space.sample()
43
44 inx, iny, inc = env.observation_space.shape
45
46 inx = int(inx / 8)
47 iny = int(iny / 8)
48
49 net = neat.nn.recurrent.RecurrentNetwork.create(genome, config)
50
51 current_max_fitness = 0
52 fitness_current = 0
53 frame = 0
54 counter = 0
55 xpos = 0
56
57 done = False
58
59 while not done:
60
61 env.render()
62 frame += 1
63 ob = cv2.resize(ob, (inx, iny))
64 #ob = cv2.blur(ob, (3,3))
65 ob = cv2.cvtColor(ob, cv2.COLOR_BGR2GRAY)
66
67 if SEE_NETWORK_INPUT:
68 img = ob.copy()
69 dst = (img.shape[0] * 8, img.shape[1] * 8)
70 img = cv2.resize(img, dst, interpolation=cv2.INTER_NEAREST)
71 img = np.flipud(img)
72 viewer.imshow(img)
73
74 ob = np.reshape(ob, (inx, iny))
75
76 imgarray = np.ndarray.flatten(ob)
77
78 nnOutput = net.activate(imgarray)
79 ac = env.action_to_array(nnOutput)
80 # print(ac)
81 controllerViewer.actionshow(ac)
82 ob, rew, done, info = env.step(nnOutput)
83
84 xpos = info['x']
85
86 if xpos >= 60000:
87 fitness_current += 10000000
88 done = True
89
90 fitness_current += rew
91
92 if fitness_current > current_max_fitness:
93 current_max_fitness = fitness_current
94 counter = 0
95 else:
96 counter += 1
97
98 if done or counter == 250:
99 done = True
100 print(genome_id, fitness_current)
101
102 genome.fitness = fitness_current
103
104 # Get directory of current script. This directory is also the one contain 'config-feedforward'
105 config_dir = os.path.dirname(getsourcefile(lambda:0))
106
107 config = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
108 neat.DefaultSpeciesSet, neat.DefaultStagnation,
109 'source/models/neat-feedforward')
110 if resume == True:
111 p = neat.Checkpointer.restore_checkpoint(restore_file)
112 else:
113 p = neat.Population(config)
114
115 p.add_reporter(neat.StdOutReporter(True))
116 stats = neat.StatisticsReporter()
117 p.add_reporter(stats)
118 p.add_reporter(neat.Checkpointer(1, filename_prefix='neat-checkpoint-'))
119
120 winner = p.run(eval_genomes, 1)
121
122 with open('winnerTEST.pkl', 'wb') as output:
123 pickle.dump(winner, output, 1)
124
125if __name__ == "__main__":
126 train_neat()