diff --git a/code_pipeline/model_executor.py b/code_pipeline/model_executor.py new file mode 100644 index 0000000..00bae54 --- /dev/null +++ b/code_pipeline/model_executor.py @@ -0,0 +1,264 @@ +from code_pipeline.executors import AbstractTestExecutor + +import time +import traceback + +from models.config import ModelsConfig +from self_driving.beamng_brewer import BeamNGBrewer +from self_driving.beamng_tig_maps import maps, LevelsFolder +from self_driving.beamng_waypoint import BeamNGWaypoint +from self_driving.simulation_data import SimulationDataRecord, SimulationData +from self_driving.simulation_data_collector import SimulationDataCollector +from self_driving.utils import get_node_coords, points_distance +from self_driving.vehicle_state_reader import VehicleStateReader +from beamngpy.sensors import Camera +from abc import abstractmethod + +from shapely.geometry import Point + +import logging as log +import os.path +import cv2 +import numpy + +DRIVER_CAMERA_NAME = 'driver_view_camera' + + +def preprocess_image(image, resize=None): + image_array = numpy.asarray(image) + # removes sky and front of car + image_array = image_array[80:-1, :, :] + image_array = cv2.resize(image_array, resize, cv2.INTER_AREA) + image_array = cv2.cvtColor(image_array, cv2.COLOR_RGB2YUV) + return image_array + + +class ModelExecutor(AbstractTestExecutor): + + def __init__(self, result_folder, time_budget, map_size, + oob_tolerance=0.95, max_speed=70, + beamng_home=None, beamng_user=None, road_visualizer=None): + super(ModelExecutor, self).__init__(result_folder, time_budget, map_size) + self.test_time_budget = 250000 + self.MIN_SPEED = 5.0 + self.MAX_SPEED = max_speed + + self.oob_tolerance = oob_tolerance + self.maxspeed = max_speed + self.brewer: BeamNGBrewer = None + self.beamng_home = beamng_home + self.beamng_user = beamng_user + self.config = ModelsConfig() + + if self.beamng_user is not None and not os.path.exists(os.path.join(self.beamng_user, "research.key")): + log.warning("%s is missing but is required to use BeamNG.research", ) + + if self.config.model_path is None or not os.path.exists(self.config.model_path): + log.warning("Required model not available") + + # Runtime Monitor about relative movement of the car + self.last_observation = None + # Not sure how to set this... How far can a car move in 250 ms at 5Km/h + self.min_delta_position = 1.0 + self.road_visualizer = road_visualizer + + @abstractmethod + def load_model(self): + print('implement custom load operation') + + @abstractmethod + def predict(self, image): + print('implement custom predict operation') + + def _execute(self, the_test): + # Ensure we do not execute anything longer than the time budget + super()._execute(the_test) + + # TODO Show name of the test? + log.info("Executing test %s", the_test.id) + + # TODO Not sure why we need to repeat this 2 times... + counter = 2 + + attempt = 0 + sim = None + condition = True + while condition: + attempt += 1 + if attempt == counter: + test_outcome = "ERROR" + description = 'Exhausted attempts' + break + if attempt > 1: + self._close() + if attempt > 2: + time.sleep(5) + + sim = self._run_simulation(the_test) + + if sim.info.success: + if sim.exception_str: + test_outcome = "FAIL" + description = sim.exception_str + else: + test_outcome = "PASS" + description = 'Successful test' + condition = False + + execution_data = sim.states + + # TODO: report all test outcomes + return test_outcome, description, execution_data + + def _is_the_car_moving(self, last_state): + """ Check if the car moved in the past 10 seconds """ + + # Has the position changed + if self.last_observation is None: + self.last_observation = last_state + return True + + # If the car moved since the last observation, we store the last state and move one + if Point(self.last_observation.pos[0], self.last_observation.pos[1]).distance( + Point(last_state.pos[0], last_state.pos[1])) > self.min_delta_position: + self.last_observation = last_state + return True + else: + # How much time has passed since the last observation? + if last_state.timer - self.last_observation.timer > 10.0: + return False + else: + return True + + def _run_simulation(self, the_test) -> SimulationData: + if not self.brewer: + self.brewer = BeamNGBrewer(beamng_home=self.beamng_home, beamng_user=self.beamng_user) + self.vehicle = self.brewer.setup_vehicle() + + # For the execution we need the interpolated points + nodes = the_test.interpolated_points + + brewer = self.brewer + brewer.setup_road_nodes(nodes) + beamng = brewer.beamng + waypoint_goal = BeamNGWaypoint('waypoint_goal', get_node_coords(nodes[-1])) + + # TODO Make sure that maps points to the right folder ! + if self.beamng_user is not None: + beamng_levels = LevelsFolder(os.path.join(self.beamng_user, 'levels')) + maps.beamng_levels = beamng_levels + maps.beamng_map = maps.beamng_levels.get_map('tig') + + maps.install_map_if_needed() + maps.beamng_map.generated().write_items(brewer.decal_road.to_json() + '\n' + waypoint_goal.to_json()) + + # camera settings taken from https://github.com/testingautomated-usi/DeepHyperion/blob/main/DeepHyperion-BNG + # /udacity_integration/beamng_car_cameras.py + cam_pos = (-0.3, 1.7, 1.0) + cam_dir = (0, 1, 0) + cam_fov = 120 + cam_res = (self.config.IMAGE_WIDTH, self.config.IMAGE_HEIGHT) + camera = (DRIVER_CAMERA_NAME, Camera(cam_pos, cam_dir, cam_fov, cam_res, colour=True, depth=True, + annotation=True)) + additional_sensors = [camera] + vehicle_state_reader = VehicleStateReader(self.vehicle, beamng, additional_sensors=additional_sensors) + brewer.vehicle_start_pose = brewer.road_points.vehicle_start_pose() + + steps = brewer.params.beamng_steps + simulation_id = time.strftime('%Y-%m-%d--%H-%M-%S', time.localtime()) + name = 'model_executor/{}/sim_{}'.format(self.config.model_name, simulation_id) + sim_data_collector = SimulationDataCollector(self.vehicle, beamng, brewer.decal_road, brewer.params, + vehicle_state_reader=vehicle_state_reader, + simulation_name=name) + + sim_data_collector.oob_monitor.tolerance = self.oob_tolerance + + sim_data_collector.get_simulation_data().start() + try: + brewer.bring_up() + + while True: + # get camera image and preprocess it + img = self.get_driver_camera_image() + + # predict steering angle + steering_angle = float(self.predict(img)) + + # show driver view with predicted steering angle + img_cv = cv2.cvtColor(numpy.asarray(img), cv2.COLOR_RGB2BGR) + # draw the same text a bit thicker to create a black outline around the text + cv2.putText(img_cv, f"{steering_angle:.9f}", (2, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 2, + cv2.LINE_AA) + cv2.putText(img_cv, f"{steering_angle:.9f}", (2, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1, + cv2.LINE_AA) + cv2.imshow('Predicted Steering Angle', img_cv) + cv2.waitKey(1) + + sim_data_collector.collect_current_data(oob_bb=False) + last_state: SimulationDataRecord = sim_data_collector.states[-1] + + if last_state.vel_kmh > self.MAX_SPEED: + self.speed_limit = self.MIN_SPEED # slow down + else: + self.speed_limit = self.MAX_SPEED + + # Based on the predicted steering angle the controller sets the speed: + # for sharper turns will slow down more than for straight segments + throttle = 1.0 - steering_angle ** 2 - (last_state.vel_kmh / self.speed_limit) ** 2 + + # pass controls to vehicle + self.vehicle.control(steering=steering_angle, throttle=throttle) + + # Target point reached + if points_distance(last_state.pos, waypoint_goal.position) < 8.0: + break + + assert self._is_the_car_moving(last_state), "Car is not moving fast enough " + str( + sim_data_collector.name) + + assert not last_state.is_oob, "Car drove out of the lane " + str(sim_data_collector.name) + + beamng.step(steps) + + sim_data_collector.get_simulation_data().end(success=True) + self.total_elapsed_time = self.get_elapsed_time() + except AssertionError as aex: + sim_data_collector.save() + # An assertion that trigger is still a successful test execution, otherwise it will count as ERROR + sim_data_collector.get_simulation_data().end(success=True, exception=aex) + traceback.print_exception(type(aex), aex, aex.__traceback__) + except Exception as ex: + sim_data_collector.save() + sim_data_collector.get_simulation_data().end(success=False, exception=ex) + traceback.print_exception(type(ex), ex, ex.__traceback__) + finally: + sim_data_collector.save() + try: + sim_data_collector.take_car_picture_if_needed() + except: + pass + + self.end_iteration() + + return sim_data_collector.simulation_data + + def get_driver_camera_image(self): + sensors = self.brewer.beamng.poll_sensors(self.vehicle) + cam = sensors[DRIVER_CAMERA_NAME] + img = cam['colour'].convert('RGB') + return img + + def end_iteration(self): + try: + if self.brewer: + self.brewer.beamng.stop_scenario() + except Exception as ex: + traceback.print_exception(type(ex), ex, ex.__traceback__) + + def _close(self): + if self.brewer: + try: + self.brewer.beamng.close() + except Exception as ex: + traceback.print_exception(type(ex), ex, ex.__traceback__) + self.brewer = None diff --git a/models/GUIDELINES.md b/models/GUIDELINES.md new file mode 100644 index 0000000..3b3eacd --- /dev/null +++ b/models/GUIDELINES.md @@ -0,0 +1,29 @@ +# How to use model executors + +To execute a model please add a custom model executor that needs to implement a `load_model()` and `predict(image)` +function from `model_executor.py`. When running the tool-competition-pipeline use `model` as `executor`! + +## Add configs +Add all config settings to `models/config`, if needed this can also be extended and used in a `model_executor`. + +## Example +Please check the usage of `models/dave2`! + +``` +python competition.py + --visualize-tests + --time-budget + 60 + --executor + model + --beamng-home + BeamNG.research + --map-size + 200 + --module-name + sample_test_generators.one_test_generator + --class-name + OneTestGenerator + --speed-limit + 30 +``` \ No newline at end of file diff --git a/models/autumn/autumn_executor.py b/models/autumn/autumn_executor.py new file mode 100644 index 0000000..663392a --- /dev/null +++ b/models/autumn/autumn_executor.py @@ -0,0 +1,95 @@ +# +# Code taken from https://github.com/udacity/self-driving-car/blob/master/steering-models/evaluation/autumn.py + +import scipy.misc +import cv2 +import numpy as np +import tensorflow as tf + +from keras.models import * +from keras.layers import * + +from code_pipeline.model_executor import ModelExecutor + + +class AutumnModelExecutor(ModelExecutor): + + def __init__(self, result_folder, time_budget, map_size, + oob_tolerance=0.95, max_speed=70, + beamng_home=None, beamng_user=None, road_visualizer=None, model_path=None): + super(AutumnModelExecutor, self).__init__(result_folder, time_budget, map_size, oob_tolerance, max_speed, + beamng_home, beamng_user, road_visualizer, model_path) + self.model_path = model_path + self.model = None + self.cnn = None + self.fc3 = None + self.y = None + self.x = None + self.keep_prob = None + self.prev_image = None + self.last = [] + self.steps = [] + self.load_model() + + def load_model(self): + # cnn_graph, lstm_json, cnn_weights, lstm_weights + sess = tf.InteractiveSession() + saver = tf.train.import_meta_graph("./model-step-2000-val-0.150803.ckpt.meta") + saver.restore(sess, "./model-step-2000-val-0.150803.ckpt") + self.cnn = tf.get_default_graph() + + # TODO does this change have any effects? + # self.fc3 = self.cnn.get_tensor_by_name("fc3/mul:0") + self.fc3 = self.cnn.get_tensor_by_name("fc3:0") + self.y = self.cnn.get_tensor_by_name("y:0") + self.x = self.cnn.get_tensor_by_name("x:0") + self.keep_prob = self.cnn.get_tensor_by_name("keep_prob:0") + + # with open(self.model_path, 'r') as f: + # json_string = f.read() + # self.model = model_from_json(json_string) + # self.model.load_weights( + + def process(self, img): + img = np.asarray(img) + prev_image = self.prev_image if self.prev_image is not None else img + self.prev_image = img + prev = cv2.cvtColor(prev_image, cv2.COLOR_RGB2GRAY) + next = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) + + flow = cv2.calcOpticalFlowFarneback(prev, next, None, 0.5, 3, 15, 3, 5, 1.2, 0) + + self.last.append(flow) + + if len(self.last) > 4: + self.last.pop(0) + + weights = [1, 1, 2, 2] + last = list(self.last) + for x in range(len(last)): + last[x] = last[x] * weights[x] + + avg_flow = sum(self.last) / len(self.last) + mag, ang = cv2.cartToPolar(avg_flow[..., 0], avg_flow[..., 1]) + + hsv = np.zeros_like(prev_image) + hsv[..., 1] = 255 + hsv[..., 0] = ang * 180 / np.pi / 2 + hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) + rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB) + return rgb + + def predict(self, img): + processed_img = self.process(img) + # cv2.imshow("Flow", img) + # cv2.waitKey(1) + image = scipy.misc.imresize(processed_img[-400:], [66, 200]) / 255.0 + cv2.imshow("after scipy", image) + cv2.waitKey(1) + cnn_output = self.fc3.eval(feed_dict={self.x: [image], self.keep_prob: 1.0}) + self.steps.append(cnn_output) + if len(self.steps) > 100: + self.steps.pop(0) + output = self.y.eval(feed_dict={self.x: [image], self.keep_prob: 1.0}) + angle = output[0][0] + return angle diff --git a/models/autumn/cnn_model.py b/models/autumn/cnn_model.py new file mode 100644 index 0000000..8114594 --- /dev/null +++ b/models/autumn/cnn_model.py @@ -0,0 +1,95 @@ +# taken from https://github.com/udacity/self-driving-car/blob/master/steering-models/community-models/autumn/autumn/cnn_model.py + +import tensorflow as tf + + +def weight_variable(shape): + initializer = tf.contrib.layers.xavier_initializer_conv2d() + initial = initializer(shape=shape) + return tf.Variable(initial) + + +def bias_variable(shape): + initial = tf.constant(0.1, shape=shape) + return tf.Variable(initial) + + +def conv2d(x, W, stride): + return tf.nn.conv2d(x, W, strides=[1, stride, stride, 1], padding='VALID') + + +class ConvModel(object): + ''' Implements the ConvNet model from the NVIDIA paper ''' + + def __init__(self, dropout_prob=0.2, batch_norm=False, whitening=False, is_training=True): + x = tf.placeholder(tf.float32, shape=[None, 66, 200, 3], name='x') + y_ = tf.placeholder(tf.float32, shape=[None, 1]) + keep_prob = tf.placeholder(tf.float32, name='keep_prob') + x_image = x + + self.W_conv1 = weight_variable([5, 5, 3, 24]) + self.b_conv1 = bias_variable([24]) + self.h_conv1 = tf.nn.relu(conv2d(x_image, self.W_conv1, 2) + self.b_conv1) + if batch_norm: + self.h_conv1 = tf.contrib.layers.batch_norm(self.h_conv1, is_training=is_training, trainable=True) + + self.W_conv2 = weight_variable([5, 5, 24, 36]) + self.b_conv2 = bias_variable([36]) + self.h_conv2 = tf.nn.relu(conv2d(self.h_conv1, self.W_conv2, 2) + self.b_conv2) + + self.W_conv3 = weight_variable([5, 5, 36, 48]) + self.b_conv3 = bias_variable([48]) + self.h_conv3 = tf.nn.relu(conv2d(self.h_conv2, self.W_conv3, 2) + self.b_conv3) + if batch_norm: + self.h_conv3 = tf.contrib.layers.batch_norm(self.h_conv3, is_training=is_training, trainable=True) + + self.W_conv4 = weight_variable([3, 3, 48, 64]) + self.b_conv4 = bias_variable([64]) + self.h_conv4 = tf.nn.relu(conv2d(self.h_conv3, self.W_conv4, 1) + self.b_conv4) + + self.W_conv5 = weight_variable([3, 3, 64, 64]) + self.b_conv5 = bias_variable([64]) + self.h_conv5 = tf.nn.relu(conv2d(self.h_conv4, self.W_conv5, 1) + self.b_conv5) + if batch_norm: + self.h_conv5 = tf.contrib.layers.batch_norm(self.h_conv5, is_training=is_training, trainable=True) + + self.W_fc1 = weight_variable([1152, 1164]) + self.b_fc1 = bias_variable([1164]) + + self.h_conv5_flat = tf.reshape(self.h_conv5, [-1, 1152]) + self.h_fc1 = tf.nn.relu(tf.matmul(self.h_conv5_flat, self.W_fc1) + self.b_fc1) + if batch_norm: + self.h_fc1 = tf.contrib.layers.batch_norm(self.h_fc1, is_training=is_training, trainable=True) + self.h_fc1_drop = tf.nn.dropout(self.h_fc1, keep_prob) + + self.W_fc2 = weight_variable([1164, 100]) + self.b_fc2 = bias_variable([100]) + self.h_fc2 = tf.nn.relu(tf.matmul(self.h_fc1_drop, self.W_fc2) + self.b_fc2, name='fc2') + if batch_norm: + self.h_fc2 = tf.contrib.layers.batch_norm(self.h_fc2, is_training=is_training, trainable=True) + self.h_fc2_drop = tf.nn.dropout(self.h_fc2, keep_prob) + + self.W_fc3 = weight_variable([100, 50]) + self.b_fc3 = bias_variable([50]) + self.h_fc3 = tf.nn.relu(tf.matmul(self.h_fc2_drop, self.W_fc3) + self.b_fc3, name='fc3') + if batch_norm: + self.h_fc3 = tf.contrib.layers.batch_norm(self.h_fc3, is_training=is_training, trainable=True) + self.h_fc3_drop = tf.nn.dropout(self.h_fc3, keep_prob) + + self.W_fc4 = weight_variable([50, 10]) + self.b_fc4 = bias_variable([10]) + self.h_fc4 = tf.nn.relu(tf.matmul(self.h_fc3_drop, self.W_fc4) + self.b_fc4, name='fc4') + if batch_norm: + self.h_fc4 = tf.contrib.layers.batch_norm(self.h_fc4, is_training=is_training, trainable=True) + self.h_fc4_drop = tf.nn.dropout(self.h_fc4, keep_prob) + + self.W_fc5 = weight_variable([10, 1]) + self.b_fc5 = bias_variable([1]) + y = tf.multiply(tf.atan(tf.matmul(self.h_fc4_drop, self.W_fc5) + self.b_fc5), 2, name='y') + + self.x = x + self.y_ = y_ + self.y = y + self.keep_prob = keep_prob + self.fc2 = self.h_fc2 + self.fc3 = self.h_fc3 diff --git a/models/autumn/data_reader.py b/models/autumn/data_reader.py new file mode 100644 index 0000000..3e59ba9 --- /dev/null +++ b/models/autumn/data_reader.py @@ -0,0 +1,177 @@ +import scipy.misc +import random +import csv +import cv2 +import numpy as np +import os +import pandas as pd +from sklearn.model_selection import train_test_split + +DATA_DIR = '' +FILE_EXT = '.jpg' + + +class DataReader(object): + def __init__(self, data_dir=DATA_DIR, file_ext=FILE_EXT, sequential=False): + self.train_batch_pointer = 0 + self.val_batch_pointer = 0 + self.data_dir = data_dir + self.load() + + # load data as it was loaded from DeepHyperion + # https://github.com/testingautomated-usi/DeepHyperion/blob/f7f696ba95124125dfe967ea4890d944a9958d77/DeepHyperion-BNG/udacity_integration/train-from-recordings.py#L17 + def load(self): + """ + Load training data and split it into training and validation set + """ + tracks = [self.data_dir] + + x = np.empty([0, 3]) + y = np.array([]) + for track in tracks: + drive = os.listdir(track) + for drive_style in drive: + try: + csv_name = 'driving_log.csv' + csv_folder = os.path.join(track, drive_style) + csv_path = os.path.join(csv_folder, csv_name) + + def fix_path(serie): + return serie.apply(lambda d: os.path.join(csv_folder, d)) + + data_df = pd.read_csv(csv_path) + pictures = data_df[['center', 'left', 'right']] + pictures_fixpath = pictures.apply(fix_path) + csv_x = pictures_fixpath.values + + csv_y = data_df['steering'].values + x = np.concatenate((x, csv_x), axis=0) + y = np.concatenate((y, csv_y), axis=0) + except FileNotFoundError: + print("Unable to read file %s" % csv_path) + exit() + + try: + X_train, X_valid, y_train, y_valid = train_test_split(x, y, test_size=0.2, random_state=0) + except TypeError: + print("Missing header to csv files") + exit() + + print("Train dataset: " + str(len(X_train)) + " elements") + print("Test dataset: " + str(len(X_valid)) + " elements") + self.x_train = X_train + self.x_valid = X_valid + self.y_train = y_train + self.y_valid = y_valid + self.num_train_images = len(self.x_train) + self.num_val_images = len(self.x_valid) + self.prev_image = None + self.last = [] + + # total = 0 + # count01 = count005 = count002 = count0 = 0 + + # with open('interpolated_center.csv') as f: + # reader = csv.DictReader(f) + # for row in reader: + # angle = float(row['steering_angle']) + # if angle > 0.1 or angle < -0.1 and random.random() > 0.2: + # xs.append(DATA_DIR + 'training/center/flow_7_cart/' + row['frame_id'] + FILE_EXT) + # ys.append(row['steering_angle']) + # count01 += 1 + # elif (angle > 0.05 or angle < -0.5) and random.random() > 0.2: + # xs.append(DATA_DIR + 'training/center/flow_7_cart/' + row['frame_id'] + FILE_EXT) + # ys.append(row['steering_angle']) + # count005 += 1 + # elif (angle > 0.02 or angle < -0.02) and random.random() > 0.7: + # xs.append(DATA_DIR + 'training/center/flow_7_cart/' + row['frame_id'] + FILE_EXT) + # ys.append(row['steering_angle']) + # count002 += 1 + # elif random.random() > 0.8: + # xs.append(DATA_DIR + 'training/center/flow_7_cart/' + row['frame_id'] + FILE_EXT) + # ys.append(row['steering_angle']) + # count0 += 1 + # total += 1 + # + # with open('train_center.csv') as f: + # reader = csv.DictReader(f) + # for row in reader: + # angle = float(row['steering_angle']) + # xs.append(DATA_DIR + 'Ch2_Train/center/flow_7_local/' + row['frame_id'] + FILE_EXT) + # ys.append(row['steering_angle']) + # total += 1 + # + # print('> 0.1 or < -0.1: ' + str(count01)) + # print('> 0.05 or < -0.05: ' + str(count005)) + # print('> 0.02 or < -0.02: ' + str(count002)) + # print('~0: ' + str(count0)) + # print('Total data: ' + str(total)) + + # self.num_images = len(xs) + # + # c = list(zip(xs, ys)) + # random.shuffle(c) + # xs, ys = zip(*c) + # + # self.train_xs = xs[:int(len(xs) * 0.8)] + # self.train_ys = ys[:int(len(xs) * 0.8)] + # + # self.val_xs = xs[-int(len(xs) * 0.2):] + # self.val_ys = ys[-int(len(xs) * 0.2):] + + def load_train_batch(self, batch_size): + x_out = [] + y_out = [] + self.prev_image = None + self.last = [] + for i in range(0, batch_size): + # autumn only uses the center images so only use index [0] as image path + im_path = self.x_train[(self.train_batch_pointer + i) % self.num_train_images][0] + image = scipy.misc.imread(im_path) + image = self.process(image) + x_out.append(scipy.misc.imresize(image[-400:], [66, 200]) / 255.0) + y_out.append([self.y_train[(self.train_batch_pointer + i) % self.num_train_images]]) + self.train_batch_pointer += batch_size + return x_out, y_out + + def load_val_batch(self, batch_size): + x_out = [] + y_out = [] + self.prev_image = None + self.last = [] + for i in range(0, batch_size): + # autumn only uses the center images so only use index [0] as image path + im_path = self.x_valid[(self.val_batch_pointer + i) % self.num_val_images][0] + image = scipy.misc.imread(im_path) + image = self.process(image) + x_out.append(scipy.misc.imresize(image[-400:], [66, 200]) / 255.0) + y_out.append([self.y_valid[(self.val_batch_pointer + i) % self.num_val_images]]) + self.val_batch_pointer += batch_size + return x_out, y_out + + def process(self, img): + img = np.asarray(img) + prev_image = self.prev_image if self.prev_image is not None else img + self.prev_image = img + prev = cv2.cvtColor(prev_image, cv2.COLOR_RGB2GRAY) + next = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) + + flow = cv2.calcOpticalFlowFarneback(prev, next, None, 0.5, 3, 15, 3, 5, 1.2, 0) + + self.last.append(flow) + + # weights = [1, 1, 2, 2] + # last = list(self.last) + # for x in range(len(last)): + # last[x] = last[x] * weights[x] + + avg_flow = sum(self.last) / len(self.last) + mag, ang = cv2.cartToPolar(avg_flow[..., 0], avg_flow[..., 1]) + + hsv = np.zeros_like(prev_image) + hsv[..., 1] = 255 + hsv[..., 0] = ang * 180 / np.pi / 2 + hsv[..., 2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX) + # rgb = cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB) + bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR) + return bgr diff --git a/models/autumn/train_autumn.py b/models/autumn/train_autumn.py new file mode 100644 index 0000000..be6fbdb --- /dev/null +++ b/models/autumn/train_autumn.py @@ -0,0 +1,99 @@ +import os +import tensorflow as tf +from models.autumn.cnn_model import ConvModel +from models.autumn.data_reader import DataReader +# from models.autumn.batch_generator import Generator +import argparse +import numpy as np +from keras.callbacks import ModelCheckpoint +from keras.optimizers import Adam + +np.random.seed(0) + +# train_cnn autumn from +# https://github.com/udacity/self-driving-car/blob/master/steering-models/community-models/autumn/train-cnn.py + +BATCH_SIZE = 100 +DATA_DIR = './training_recordings' +LOGDIR = './train_model' +CHECKPOINT_EVERY = 100 +# NUM_STEPS = int(1e5) +NUM_STEPS = 1000 +CKPT_FILE = 'model.ckpt' +LEARNING_RATE = 1e-3 +KEEP_PROB = 0.8 +L2_REG = 0 +EPSILON = 0.001 +MOMENTUM = 0.9 + + +def get_arguments(): + parser = argparse.ArgumentParser(description='ConvNet training') + parser.add_argument('--batch_size', type=int, default=BATCH_SIZE, + help='Number of images in batch.') + parser.add_argument('--data_dir', '--data', type=str, default=DATA_DIR, + help='The directory containing the training data.') + parser.add_argument('--store_metadata', type=bool, default=False, + help='Storing debug information for TensorBoard.') + parser.add_argument('--logdir', type=str, default=LOGDIR, + help='Directory for log files.') + parser.add_argument('--restore_from', type=str, default=None, + help='Checkpoint file to restore model weights from.') + parser.add_argument('--checkpoint_every', type=int, default=CHECKPOINT_EVERY, + help='How many steps to save each checkpoint after') + parser.add_argument('--num_steps', type=int, default=NUM_STEPS, + help='Number of training steps.') + parser.add_argument('--learning_rate', type=float, default=LEARNING_RATE, + help='Learning rate for training.') + parser.add_argument('--keep_prob', type=float, default=KEEP_PROB, + help='Dropout keep probability.') + parser.add_argument('--l2_reg', type=float, + default=L2_REG) + return parser.parse_args() + + +def main(): + args = get_arguments() + sess = tf.compat.v1.Session() + + model = ConvModel() + train_vars = tf.trainable_variables() + loss = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(model.y_, model.y)))) + tf.add_n( + [tf.nn.l2_loss(v) for v in train_vars]) * args.l2_reg + train_step = tf.train.AdamOptimizer(args.learning_rate).minimize(loss) + + sess.run(tf.initialize_all_variables()) + saver = tf.train.Saver() + + start_step = 0 + min_loss = 1.0 + # train_model(model, args, *data) + data_reader = DataReader(data_dir=args.data_dir) + + for i in range(start_step, start_step + args.num_steps): + xs, ys = data_reader.load_train_batch(args.batch_size) + train_step.run(feed_dict={model.x: xs, model.y_: ys, model.keep_prob: args.keep_prob}, session=sess) + train_error = loss.eval(feed_dict={model.x: xs, model.y_: ys, model.keep_prob: 1.0}, session=sess) + print("Step %d, train loss %g" % (i, train_error)) + + if i % 10 == 0: + xs, ys = data_reader.load_val_batch(args.batch_size) + val_error = loss.eval(feed_dict={model.x: xs, model.y_: ys, model.keep_prob: 1.0}, session=sess) + print("Step %d, val loss %g" % (i, val_error)) + if i > 0 and i % args.checkpoint_every == 0: + if not os.path.exists(args.logdir): + os.makedirs(args.logdir) + checkpoint_path = os.path.join(args.logdir, "model-step-%d-val-%g.ckpt" % (i, val_error)) + filename = saver.save(sess, checkpoint_path) + print("Model saved in file: %s" % filename) + elif val_error < min_loss: + min_loss = val_error + if not os.path.exists(args.logdir): + os.makedirs(args.logdir) + checkpoint_path = os.path.join(args.logdir, "model-step-%d-val-%g.ckpt" % (i, val_error)) + filename = saver.save(sess, checkpoint_path) + print("Model saved in file: %s" % filename) + + +if __name__ == '__main__': + main() diff --git a/models/config.py b/models/config.py new file mode 100644 index 0000000..3ec9fdb --- /dev/null +++ b/models/config.py @@ -0,0 +1,7 @@ +class ModelsConfig(): + # example inputs for dave2 + model_path = "./self-driving-car-146-2020.h5" + model_name = "Dave2" + IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS = 160, 320, 3 + executor_class_name = "Dave2ModelExecutor" + module_name = "models.dave2.dave2_executor" diff --git a/competition.py b/models/dave2/competition.py similarity index 91% rename from competition.py rename to models/dave2/competition.py index 928e83e..589ac1c 100644 --- a/competition.py +++ b/models/dave2/competition.py @@ -15,6 +15,7 @@ from code_pipeline.visualization import RoadTestVisualizer from code_pipeline.tests_generation import TestGenerationStatistic from code_pipeline.test_generation_utils import register_exit_fun +from models.config import ModelsConfig from code_pipeline.tests_evaluation import OOBAnalyzer @@ -54,7 +55,8 @@ def validate_map_size(ctx, param, value): The size of the map is defined by its edge. The edge can be any (integer) value between 100 and 1000 """ if int(value) < 100 or int(value) > 1000: - raise click.UsageError('The provided value for ' + str(param) + ' is invalid. Choose an integer between 100 and 1000') + raise click.UsageError( + 'The provided value for ' + str(param) + ' is invalid. Choose an integer between 100 and 1000') else: return int(value) @@ -92,7 +94,7 @@ def create_summary(result_folder, raw_data): summary_file = os.path.join(result_folder, "generation_stats.csv") csv_content = raw_data.as_csv() with open(summary_file, 'w') as output_file: - output_file.write( csv_content) + output_file.write(csv_content) log.info("Test Statistics Report available: %s", summary_file) log.info("Creating OOB Report") @@ -120,9 +122,6 @@ def post_process(ctx, result_folder, the_executor): create_summary(result_folder, the_executor.get_stats()) - - - def create_post_processing_hook(ctx, result_folder, executor): """ Uses HighOrder functions to setup the post processing hooks that will be trigger ONLY AND ONLY IF the @@ -143,7 +142,6 @@ def _f(): def setup_logging(log_to, debug): - def log_exception(extype, value, trace): log.exception('Uncaught exception:', exc_info=(extype, value, trace)) @@ -157,7 +155,7 @@ def log_exception(extype, value, trace): if log_to is not None: file_handler = log.FileHandler(log_to, 'a', 'utf-8') - log_handlers.append( file_handler ) + log_handlers.append(file_handler) start_msg += " ".join(["writing to file: ", str(log_to)]) log_level = log.DEBUG if debug else log.INFO @@ -169,9 +167,8 @@ def log_exception(extype, value, trace): log.info(start_msg) - @click.command() -@click.option('--executor', type=click.Choice(['mock', 'beamng'], case_sensitive=False), default="mock", +@click.option('--executor', type=click.Choice(['mock', 'beamng', 'model'], case_sensitive=False), default="mock", show_default='Mock Executor (meant for debugging)', help="The name of the executor to use. Currently we have 'mock' or 'beamng'.") @click.option('--beamng-home', required=False, default=None, type=click.Path(exists=True), @@ -196,7 +193,7 @@ def log_exception(extype, value, trace): @click.option('--speed-limit', type=int, default=70, callback=validate_speed_limit, show_default='70 Km/h', help="The max speed of the ego-vehicle" - "Expressed in Kilometers per hours") + "Expressed in Kilometers per hours") @click.option('--module-name', required=True, type=str, help="Name of the module where your test generator is located.") @click.option('--module-path', required=False, type=click.Path(exists=True), @@ -225,7 +222,6 @@ def generate(ctx, executor, beamng_home, beamng_user, # TODO Refactor by adding a create summary command and forwarding the output of this run to that command - # Setup logging setup_logging(log_to, debug) @@ -233,6 +229,11 @@ def generate(ctx, executor, beamng_home, beamng_user, module = importlib.import_module(module_name, module_path) the_class = getattr(module, class_name) + # Setup model executor by dynamically loading it + model_config = ModelsConfig() + model_module = importlib.import_module(model_config.module_name) + the_model_class = getattr(model_module, model_config.executor_class_name) + road_visualizer = None # Setup visualization if visualize_tests: @@ -272,6 +273,11 @@ def generate(ctx, executor, beamng_home, beamng_user, oob_tolerance=oob_tolerance, max_speed=speed_limit, beamng_home=beamng_home, beamng_user=beamng_user, road_visualizer=road_visualizer) + elif executor == "model": + the_executor = the_model_class(result_folder, time_budget, map_size, + oob_tolerance=oob_tolerance, max_speed=speed_limit, + beamng_home=beamng_home, beamng_user=beamng_user, + road_visualizer=road_visualizer) # Register the shutdown hook for post processing results register_exit_fun(create_post_processing_hook(ctx, result_folder, the_executor)) diff --git a/models/dave2/dave2_executor.py b/models/dave2/dave2_executor.py new file mode 100644 index 0000000..29d4157 --- /dev/null +++ b/models/dave2/dave2_executor.py @@ -0,0 +1,73 @@ +from code_pipeline.model_executor import ModelExecutor +import tensorflow as tf +import numpy as np +import cv2 + +from models.config import ModelsConfig + +IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS = 160, 320, 3 +INPUT_SHAPE = (IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS) + + +def preprocess(image): + """ + Combine all preprocess functions into one + """ + image = crop(image) + image = resize(image) + image = rgb2yuv(image) + return image + + +def crop(image): + """ + Crop the image (removing the sky at the top and the car front at the bottom) + """ + return image[80:-1, :, :] # remove the sky and the car front + + +def resize(image): + """ + Resize the image to the input shape used by the network model + """ + return cv2.resize(image, (IMAGE_WIDTH, IMAGE_HEIGHT), cv2.INTER_AREA) + + +def rgb2yuv(image): + """ + Convert the image from RGB to YUV (This is what the NVIDIA model does) + """ + return cv2.cvtColor(image, cv2.COLOR_RGB2YUV) + + +class Dave2ModelExecutor(ModelExecutor): + + def __init__(self, result_folder, time_budget, map_size, + oob_tolerance=0.95, max_speed=70, + beamng_home=None, beamng_user=None, road_visualizer=None): + super(Dave2ModelExecutor, self).__init__(result_folder, time_budget, map_size, oob_tolerance, max_speed, + beamng_home, beamng_user, road_visualizer) + self.config = ModelsConfig() + self.model_path = self.config.model_path + self.model = None + self.load_model() + + def load_model(self): + self.model: tf.keras.Model = tf.keras.models.load_model(self.model_path) + # self.model: tf.keras.Model = tf.keras.models.model_from_json(model_path) + self.model.compile() + # weights_file = model_path.replace('json', 'hdf5') + # self.model.load_weights(weights_file) + + # Check model architecture + self.model.summary() + + def predict(self, image): + image = np.asarray(image) + + image = preprocess(image) + image = np.array([image]) + + predicted_values = self.model.predict(image, batch_size=1) + steering_angle = float(predicted_values) + return steering_angle diff --git a/models/dave2/self-driving-car-146-2020.h5 b/models/dave2/self-driving-car-146-2020.h5 new file mode 100644 index 0000000..ec25c87 Binary files /dev/null and b/models/dave2/self-driving-car-146-2020.h5 differ diff --git a/models/komanda/komanda_executor.py b/models/komanda/komanda_executor.py new file mode 100644 index 0000000..6a0650f --- /dev/null +++ b/models/komanda/komanda_executor.py @@ -0,0 +1,67 @@ +# +# Code taken from https://github.com/udacity/self-driving-car/blob/master/steering-models/evaluation/komanda.py + +""" +Udacity self-driving car challenge 2 +Team komanda steering model +Author: Ilya Edrenkin, ilya.edrenkin@gmail.com +""" + +from collections import deque + +import numpy as np + +from code_pipeline.model_executor import ModelExecutor +import tensorflow as tf + + +class KomandaModelExecutor(ModelExecutor): + + def __init__(self, result_folder, time_budget, map_size, + oob_tolerance=0.95, max_speed=70, + beamng_home=None, beamng_user=None, road_visualizer=None, model_path=None): + super(KomandaModelExecutor, self).__init__(result_folder, time_budget, map_size, oob_tolerance, max_speed, + beamng_home, beamng_user, road_visualizer, model_path) + self.model_path = model_path + self.model = None + self.session = None + self.graph = tf.Graph() + self.LEFT_CONTEXT = 5 # TODO remove hardcode; store it in the graph + self.input_images = deque() # will be of size self.LEFT_CONTEXT + 1 + self.internal_state = [] # will hold controller_{final -> initial}_state_{0,1,2} + + self.load_model() + + def load_model(self): + with self.graph.as_default(): + saver = tf.train.import_meta_graph(self.model_path) + # ckpt = tf.train.latest_checkpoint(checkpoint_dir) + self.session = tf.Session(graph=self.graph) + # saver.restore(self.session, ckpt) + + # TODO controller state names should be stored in the graph + self.input_tensors = map(self.graph.get_tensor_by_name, + ["input_images:0", "controller_initial_state_0:0", "controller_initial_state_1:0", + "controller_initial_state_2:0"]) + self.output_tensors = map(self.graph.get_tensor_by_name, + ["output_steering:0", "controller_final_state_0:0", "controller_final_state_1:0", + "controller_final_state_2:0"]) + + # Check model architecture + # self.model.summary() + + def predict(self, img): + if len(self.input_images) == 0: + self.input_images += [img] * (self.LEFT_CONTEXT + 1) + else: + self.input_images.popleft() + self.input_images.append(img) + input_images_tensor = np.stack(self.input_images) + if not self.internal_state: + key_list = list(self.input_tensors) + feed_dict = {key_list[0]: input_images_tensor} + else: + feed_dict = dict(zip(self.input_tensors, [input_images_tensor] + self.internal_state)) + steering, c0, c1, c2 = self.session.run(self.output_tensors, feed_dict=feed_dict) + self.internal_state = [c0, c1, c2] + return steering[0][0] diff --git a/models/komanda/train_komanda.py b/models/komanda/train_komanda.py new file mode 100644 index 0000000..684f82c --- /dev/null +++ b/models/komanda/train_komanda.py @@ -0,0 +1,366 @@ +import tensorflow as tf +import numpy as np +import os + +slim = tf.contrib.slim + +# define some constants + +# RNNs are typically trained using (truncated) backprop through time. SEQ_LEN here is the length of BPTT. +# Batch size specifies the number of sequence fragments used in a sigle optimization step. +# (Actually we can use variable SEQ_LEN and BATCH_SIZE, they are set to constants only for simplicity). +# LEFT_CONTEXT is the number of extra frames from the past that we append to the left of our input sequence. +# We need to do it because 3D convolution with "VALID" padding "eats" frames from the left, decreasing the sequence length. +# One should be careful here to maintain the model's causality. +SEQ_LEN = 10 +BATCH_SIZE = 4 +LEFT_CONTEXT = 5 + +# These are the input image parameters. +HEIGHT = 480 +WIDTH = 640 +CHANNELS = 3 # RGB + +# The parameters of the LSTM that keeps the model state. +RNN_SIZE = 32 +RNN_PROJ = 32 + +# Our training data follows the "interpolated.csv" format from Ross Wightman's scripts. +CSV_HEADER = "index,timestamp,width,height,frame_id,filename,angle,torque,speed,lat,long,alt".split(",") +OUTPUTS = CSV_HEADER[-6:-3] # angle,torque,speed +OUTPUT_DIM = len(OUTPUTS) # predict all features: steering angle, torque and vehicle speed + +gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=1.0) + +checkpoint_dir = os.getcwd() + "/v3" + +global_train_step = 0 +global_valid_step = 0 + +KEEP_PROB_TRAIN = 0.25 + + +class BatchGenerator(object): + def __init__(self, sequence, seq_len, batch_size): + self.sequence = sequence + self.seq_len = seq_len + self.batch_size = batch_size + chunk_size = 1 + (len(sequence) - 1) / batch_size + self.indices = [(i * chunk_size) % len(sequence) for i in range(batch_size)] + + def next(self): + while True: + output = [] + for i in range(self.batch_size): + idx = self.indices[i] + left_pad = self.sequence[idx - LEFT_CONTEXT:idx] + if len(left_pad) < LEFT_CONTEXT: + left_pad = [self.sequence[0]] * (LEFT_CONTEXT - len(left_pad)) + left_pad + assert len(left_pad) == LEFT_CONTEXT + leftover = len(self.sequence) - idx + if leftover >= self.seq_len: + result = self.sequence[idx:idx + self.seq_len] + else: + result = self.sequence[idx:] + self.sequence[:self.seq_len - leftover] + assert len(result) == self.seq_len + self.indices[i] = (idx + self.seq_len) % len(self.sequence) + images, targets = zip(*result) + images_left_pad, _ = zip(*left_pad) + output.append((np.stack(images_left_pad + images), np.stack(targets))) + output = zip(*output) + output[0] = np.stack(output[0]) # batch_size x (LEFT_CONTEXT + seq_len) + output[1] = np.stack(output[1]) # batch_size x seq_len x OUTPUT_DIM + return output + + +def read_csv(filename): + with open(filename, 'r') as f: + lines = [ln.strip().split(",")[-7:-3] for ln in f.readlines()] + lines = map(lambda x: (x[0], np.float32(x[1:])), lines) # imagefile, outputs + return lines + + +def process_csv(filename, val=5): + sum_f = np.float128([0.0] * OUTPUT_DIM) + sum_sq_f = np.float128([0.0] * OUTPUT_DIM) + lines = read_csv(filename) + # leave val% for validation + train_seq = [] + valid_seq = [] + cnt = 0 + for ln in lines: + if cnt < SEQ_LEN * BATCH_SIZE * (100 - val): + train_seq.append(ln) + sum_f += ln[1] + sum_sq_f += ln[1] * ln[1] + else: + valid_seq.append(ln) + cnt += 1 + cnt %= SEQ_LEN * BATCH_SIZE * 100 + mean = sum_f / len(train_seq) + var = sum_sq_f / len(train_seq) - mean * mean + std = np.sqrt(var) + print(len(train_seq), len(valid_seq)) + print(mean, std) # we will need these statistics to normalize the outputs (and ground truth inputs) + return (train_seq, valid_seq), (mean, std) + + +layer_norm = lambda x: tf.contrib.layers.layer_norm(inputs=x, center=True, scale=True, activation_fn=None, + trainable=True) + + +def get_optimizer(loss, lrate): + optimizer = tf.train.AdamOptimizer(learning_rate=lrate) + gradvars = optimizer.compute_gradients(loss) + gradients, v = zip(*gradvars) + print([x.name for x in v]) + gradients, _ = tf.clip_by_global_norm(gradients, 15.0) + return optimizer.apply_gradients(zip(gradients, v)) + + +def apply_vision_simple(image, keep_prob, batch_size, seq_len, scope=None, reuse=None): + video = tf.reshape(image, shape=[batch_size, LEFT_CONTEXT + seq_len, HEIGHT, WIDTH, CHANNELS]) + with tf.variable_scope(scope, 'Vision', [image], reuse=reuse): + net = slim.convolution(video, num_outputs=64, kernel_size=[3, 12, 12], stride=[1, 6, 6], padding="VALID") + net = tf.nn.dropout(x=net, keep_prob=keep_prob) + aux1 = slim.fully_connected(tf.reshape(net[:, -seq_len:, :, :, :], [batch_size, seq_len, -1]), 128, + activation_fn=None) + + net = slim.convolution(net, num_outputs=64, kernel_size=[2, 5, 5], stride=[1, 2, 2], padding="VALID") + net = tf.nn.dropout(x=net, keep_prob=keep_prob) + aux2 = slim.fully_connected(tf.reshape(net[:, -seq_len:, :, :, :], [batch_size, seq_len, -1]), 128, + activation_fn=None) + + net = slim.convolution(net, num_outputs=64, kernel_size=[2, 5, 5], stride=[1, 1, 1], padding="VALID") + net = tf.nn.dropout(x=net, keep_prob=keep_prob) + aux3 = slim.fully_connected(tf.reshape(net[:, -seq_len:, :, :, :], [batch_size, seq_len, -1]), 128, + activation_fn=None) + + net = slim.convolution(net, num_outputs=64, kernel_size=[2, 5, 5], stride=[1, 1, 1], padding="VALID") + net = tf.nn.dropout(x=net, keep_prob=keep_prob) + # at this point the tensor 'net' is of shape batch_size x seq_len x ... + aux4 = slim.fully_connected(tf.reshape(net, [batch_size, seq_len, -1]), 128, activation_fn=None) + + net = slim.fully_connected(tf.reshape(net, [batch_size, seq_len, -1]), 1024, activation_fn=tf.nn.relu) + net = tf.nn.dropout(x=net, keep_prob=keep_prob) + net = slim.fully_connected(net, 512, activation_fn=tf.nn.relu) + net = tf.nn.dropout(x=net, keep_prob=keep_prob) + net = slim.fully_connected(net, 256, activation_fn=tf.nn.relu) + net = tf.nn.dropout(x=net, keep_prob=keep_prob) + net = slim.fully_connected(net, 128, activation_fn=None) + return layer_norm(tf.nn.elu(net + aux1 + aux2 + aux3 + aux4)) # aux[1-4] are residual connections (shortcuts) + + +class SamplingRNNCell(tf.nn.rnn_cell.RNNCell): + """Simple sampling RNN cell.""" + + def __init__(self, num_outputs, use_ground_truth, internal_cell): + """ + if use_ground_truth then don't sample + """ + self._num_outputs = num_outputs + self._use_ground_truth = use_ground_truth # boolean + self._internal_cell = internal_cell # may be LSTM or GRU or anything + + @property + def state_size(self): + return self._num_outputs, self._internal_cell.state_size # previous output and bottleneck state + + @property + def output_size(self): + return self._num_outputs # steering angle, torque, vehicle speed + + def __call__(self, inputs, state, scope=None): + (visual_feats, current_ground_truth) = inputs + prev_output, prev_state_internal = state + context = tf.concat(1, [prev_output, visual_feats]) + # here the internal cell (e.g. LSTM) is called + new_output_internal, new_state_internal = self.internal_cell(context, prev_state_internal) + new_output = tf.contrib.layers.fully_connected( + inputs=tf.concat(1, [new_output_internal, prev_output, visual_feats]), + num_outputs=self._num_outputs, + activation_fn=None, + scope="OutputProjection") + # if self._use_ground_truth == True, we pass the ground truth as the state; otherwise, we use the model's predictions + return new_output, (current_ground_truth if self._use_ground_truth else new_output, new_state_internal) + + +graph = tf.Graph() + +with graph.as_default(): + # inputs + learning_rate = tf.placeholder_with_default(input=1e-4, shape=()) + keep_prob = tf.placeholder_with_default(input=1.0, shape=()) + aux_cost_weight = tf.placeholder_with_default(input=0.1, shape=()) + + inputs = tf.placeholder(shape=(BATCH_SIZE, LEFT_CONTEXT + SEQ_LEN), + dtype=tf.string) # pathes to png files from the central camera + targets = tf.placeholder(shape=(BATCH_SIZE, SEQ_LEN, OUTPUT_DIM), + dtype=tf.float32) # seq_len x batch_size x OUTPUT_DIM + targets_normalized = (targets - mean) / std + + input_images = tf.pack([tf.image.decode_png(tf.read_file(x)) + for x in tf.unpack(tf.reshape(inputs, shape=[(LEFT_CONTEXT + SEQ_LEN) * BATCH_SIZE]))]) + input_images = -1.0 + 2.0 * tf.cast(input_images, tf.float32) / 255.0 + input_images.set_shape([(LEFT_CONTEXT + SEQ_LEN) * BATCH_SIZE, HEIGHT, WIDTH, CHANNELS]) + visual_conditions_reshaped = apply_vision_simple(image=input_images, keep_prob=keep_prob, + batch_size=BATCH_SIZE, seq_len=SEQ_LEN) + visual_conditions = tf.reshape(visual_conditions_reshaped, [BATCH_SIZE, SEQ_LEN, -1]) + visual_conditions = tf.nn.dropout(x=visual_conditions, keep_prob=keep_prob) + + rnn_inputs_with_ground_truth = (visual_conditions, targets_normalized) + rnn_inputs_autoregressive = ( + visual_conditions, tf.zeros(shape=(BATCH_SIZE, SEQ_LEN, OUTPUT_DIM), dtype=tf.float32)) + + internal_cell = tf.nn.rnn_cell.LSTMCell(num_units=RNN_SIZE, num_proj=RNN_PROJ) + cell_with_ground_truth = SamplingRNNCell(num_outputs=OUTPUT_DIM, use_ground_truth=True, + internal_cell=internal_cell) + cell_autoregressive = SamplingRNNCell(num_outputs=OUTPUT_DIM, use_ground_truth=False, + internal_cell=internal_cell) + + def get_initial_state(complex_state_tuple_sizes): + flat_sizes = tf.nn.rnn_cell.nest.flatten(complex_state_tuple_sizes) + init_state_flat = [tf.tile( + multiples=[BATCH_SIZE, 1], + input=tf.get_variable("controller_initial_state_%d" % i, initializer=tf.zeros_initializer, + shape=([1, s]), dtype=tf.float32)) + for i, s in enumerate(flat_sizes)] + init_state = tf.nn.rnn_cell.nest.pack_sequence_as(complex_state_tuple_sizes, init_state_flat) + return init_state + + + def deep_copy_initial_state(complex_state_tuple): + flat_state = tf.nn.rnn_cell.nest.flatten(complex_state_tuple) + flat_copy = [tf.identity(s) for s in flat_state] + deep_copy = tf.nn.rnn_cell.nest.pack_sequence_as(complex_state_tuple, flat_copy) + return deep_copy + + + controller_initial_state_variables = get_initial_state(cell_autoregressive.state_size) + controller_initial_state_autoregressive = deep_copy_initial_state(controller_initial_state_variables) + controller_initial_state_gt = deep_copy_initial_state(controller_initial_state_variables) + + with tf.variable_scope("predictor"): + out_gt, controller_final_state_gt = tf.nn.dynamic_rnn(cell=cell_with_ground_truth, + inputs=rnn_inputs_with_ground_truth, + sequence_length=[SEQ_LEN] * BATCH_SIZE, + initial_state=controller_initial_state_gt, + dtype=tf.float32, + swap_memory=True, time_major=False) + with tf.variable_scope("predictor", reuse=True): + out_autoregressive, controller_final_state_autoregressive = tf.nn.dynamic_rnn(cell=cell_autoregressive, + inputs=rnn_inputs_autoregressive, + sequence_length=[ + SEQ_LEN] * BATCH_SIZE, + initial_state=controller_initial_state_autoregressive, + dtype=tf.float32, + swap_memory=True, + time_major=False) + + mse_gt = tf.reduce_mean(tf.squared_difference(out_gt, targets_normalized)) + mse_autoregressive = tf.reduce_mean(tf.squared_difference(out_autoregressive, targets_normalized)) + mse_autoregressive_steering = tf.reduce_mean( + tf.squared_difference(out_autoregressive[:, :, 0], targets_normalized[:, :, 0])) + steering_predictions = (out_autoregressive[:, :, 0] * std[0]) + mean[0] + + total_loss = mse_autoregressive_steering + aux_cost_weight * (mse_gt + mse_autoregressive) + + optimizer = get_optimizer(total_loss, learning_rate) + + tf.scalar_summary("MAIN TRAIN METRIC: rmse_autoregressive_steering", tf.sqrt(mse_autoregressive_steering)) + tf.scalar_summary("rmse_gt", tf.sqrt(mse_gt)) + tf.scalar_summary("rmse_autoregressive", tf.sqrt(mse_autoregressive)) + + summaries = tf.merge_all_summaries() + train_writer = tf.train.SummaryWriter('v3/train_summary', graph=graph) + valid_writer = tf.train.SummaryWriter('v3/valid_summary', graph=graph) + saver = tf.train.Saver(write_version=tf.train.SaverDef.V2) + + +def do_epoch(session, sequences, mode): + global global_train_step, global_valid_step + test_predictions = {} + valid_predictions = {} + batch_generator = BatchGenerator(sequence=sequences, seq_len=SEQ_LEN, batch_size=BATCH_SIZE) + total_num_steps = 1 + (batch_generator.indices[1] - 1) / SEQ_LEN + controller_final_state_gt_cur, controller_final_state_autoregressive_cur = None, None + acc_loss = np.float128(0.0) + for step in range(total_num_steps): + feed_inputs, feed_targets = batch_generator.next() + feed_dict = {inputs: feed_inputs, targets: feed_targets} + if controller_final_state_autoregressive_cur is not None: + feed_dict.update({controller_initial_state_autoregressive: controller_final_state_autoregressive_cur}) + if controller_final_state_gt_cur is not None: + feed_dict.update({controller_final_state_gt: controller_final_state_gt_cur}) + if mode == "train": + feed_dict.update({keep_prob: KEEP_PROB_TRAIN}) + summary, _, loss, controller_final_state_gt_cur, controller_final_state_autoregressive_cur = \ + session.run([summaries, optimizer, mse_autoregressive_steering, controller_final_state_gt, + controller_final_state_autoregressive], + feed_dict=feed_dict) + train_writer.add_summary(summary, global_train_step) + global_train_step += 1 + elif mode == "valid": + model_predictions, summary, loss, controller_final_state_autoregressive_cur = \ + session.run([steering_predictions, summaries, mse_autoregressive_steering, + controller_final_state_autoregressive], + feed_dict=feed_dict) + valid_writer.add_summary(summary, global_valid_step) + global_valid_step += 1 + feed_inputs = feed_inputs[:, LEFT_CONTEXT:].flatten() + steering_targets = feed_targets[:, :, 0].flatten() + model_predictions = model_predictions.flatten() + stats = np.stack([steering_targets, model_predictions, (steering_targets - model_predictions) ** 2]) + for i, img in enumerate(feed_inputs): + valid_predictions[img] = stats[:, i] + elif mode == "test": + model_predictions, controller_final_state_autoregressive_cur = \ + session.run([steering_predictions, controller_final_state_autoregressive], + feed_dict=feed_dict) + feed_inputs = feed_inputs[:, LEFT_CONTEXT:].flatten() + model_predictions = model_predictions.flatten() + for i, img in enumerate(feed_inputs): + test_predictions[img] = model_predictions[i] + if mode != "test": + acc_loss += loss + print() + '\r', step + 1, "/", total_num_steps, np.sqrt(acc_loss / (step + 1)), + print() + return (np.sqrt(acc_loss / total_num_steps), valid_predictions) if mode != "test" else (None, test_predictions) + + +NUM_EPOCHS = 100 + +best_validation_score = None +with tf.Session(graph=graph, config=tf.ConfigProto(gpu_options=gpu_options)) as session: + session.run(tf.initialize_all_variables()) + print('Initialized') + ckpt = tf.train.latest_checkpoint(checkpoint_dir) + if ckpt: + print("Restoring from", ckpt) + saver.restore(sess=session, save_path=ckpt) + for epoch in range(NUM_EPOCHS): + print("Starting epoch %d" % epoch) + print("Validation:") + valid_score, valid_predictions = do_epoch(session=session, sequences=valid_seq, mode="valid") + if best_validation_score is None: + best_validation_score = valid_score + if valid_score < best_validation_score: + saver.save(session, 'v3/checkpoint-sdc-ch2') + best_validation_score = valid_score + print('\r', "SAVED at epoch %d" % epoch, ) + with open("v3/valid-predictions-epoch%d" % epoch, "w") as out: + result = np.float128(0.0) + for img, stats in valid_predictions.items(): + print >> out, img, stats + result += stats[-1] + print("Validation unnormalized RMSE:", np.sqrt(result / len(valid_predictions))) + with open("v3/test-predictions-epoch%d" % epoch, "w") as out: + _, test_predictions = do_epoch(session=session, sequences=test_seq, mode="test") + print("frame_id,steering_angle", file=out) + for img, pred in test_predictions.items(): + img = img.replace("challenge_2/Test-final/center/", "") + print("%s,%f" % (img, pred), file=out) + if epoch != NUM_EPOCHS - 1: + print("Training") + do_epoch(session=session, sequences=train_seq, mode="train") diff --git a/models/rambo/config.py b/models/rambo/config.py new file mode 100644 index 0000000..cddd934 --- /dev/null +++ b/models/rambo/config.py @@ -0,0 +1,25 @@ +class DataConfig(object): + data_path = "./models/rambo/data" + data_name = "hsv_gray_diff_ch4" + img_height = 160 + img_width = 320 + num_channels = 2 + + +class TrainConfig(DataConfig): + model_name = "comma_prelu" + batch_size = 32 + num_epoch = 10 + val_part = 33 + X_train_mean_path = "./data/X_train_gray_diff2_mean.npy" + + +class TestConfig(TrainConfig): + model_path = "./datamodels/weights_hsv_gray_diff_ch4_comma_prelu-03-0.04265.hdf5" + angle_train_mean = -0.004179079 + + +class VisualizeConfig(object): + pred_path = "./submissions/final.csv" + true_path = "./data/CH2_final_evaluation.csv" + img_path = "./phase2_test/center/*.jpg" \ No newline at end of file diff --git a/models/rambo/datamodels/weights_hsv_gray_diff_ch4_comma_prelu-03-0.04265.hdf5 b/models/rambo/datamodels/weights_hsv_gray_diff_ch4_comma_prelu-03-0.04265.hdf5 new file mode 100644 index 0000000..6ab430c Binary files /dev/null and b/models/rambo/datamodels/weights_hsv_gray_diff_ch4_comma_prelu-03-0.04265.hdf5 differ diff --git a/models/rambo/preprocess_test_data.py b/models/rambo/preprocess_test_data.py new file mode 100644 index 0000000..0a740b9 --- /dev/null +++ b/models/rambo/preprocess_test_data.py @@ -0,0 +1,96 @@ +from keras.preprocessing.image import load_img, img_to_array +import pandas as pd +import numpy as np +from skimage.exposure import rescale_intensity +from matplotlib.colors import rgb_to_hsv +from models.rambo.config import TestConfig, DataConfig +import glob + + +def make_grayscale_diff_data(num_channels=2): + num_rows = len(filenames) + + X = np.zeros((num_rows - num_channels, row, col, num_channels), dtype=np.uint8) + for i in range(num_channels, num_rows): + if i % 1000 == 0: + print("Processed " + str(i) + " images...") + for j in range(num_channels): + path0 = filenames[i - j - 1] + path1 = filenames[i - j] + img0 = load_img(path0, grayscale=True, target_size=(row, col)) + img1 = load_img(path1, grayscale=True, target_size=(row, col)) + img0 = img_to_array(img0) + img1 = img_to_array(img1) + img = img1 - img0 + img = rescale_intensity(img, in_range=(-255, 255), out_range=(0, 255)) + img = np.array(img, dtype=np.uint8) + + X[i - num_channels, :, :, j] = img[:, :, 0] + return X + + +def make_grayscale_diff_tx_data(num_channels=2): + num_rows = len(filenames) + + X = np.zeros((num_rows - num_channels, row, col, num_channels), dtype=np.uint8) + for i in range(num_channels, num_rows): + if i % 1000 == 0: + print("Processed " + str(i) + " images...") + path1 = filenames[i] + img1 = load_img(path1, grayscale=True, target_size=(row, col)) + img1 = img_to_array(img1) + for j in range(1, num_channels + 1): + path0 = filenames[i - j] + img0 = load_img(path0, grayscale=True, target_size=(row, col)) + img0 = img_to_array(img0) + + img = img1 - img0 + img = rescale_intensity(img, in_range=(-255, 255), out_range=(0, 255)) + img = np.array(img, dtype=np.uint8) + + X[i - num_channels, :, :, j - 1] = img[:, :, 0] + return X + + +def make_hsv_grayscale_diff_data(num_channels=2): + num_rows = len(filenames) + + X = np.zeros((num_rows - num_channels, row, col, num_channels), dtype=np.uint8) + for i in range(num_channels, num_rows): + if i % 1000 == 0: + print("Processed " + str(i) + " images...") + for j in range(num_channels): + path0 = filenames[i - j - 1] + path1 = filenames[i - j] + img0 = load_img(path0, target_size=(row, col)) + img1 = load_img(path1, target_size=(row, col)) + img0 = img_to_array(img0) + img1 = img_to_array(img1) + img0 = rgb_to_hsv(img0) + img1 = rgb_to_hsv(img1) + img = img1[:, :, 2] - img0[:, :, 2] + img = rescale_intensity(img, in_range=(-255, 255), out_range=(0, 255)) + img = np.array(img, dtype=np.uint8) + + X[i - num_channels, :, :, j] = img + return X + + +if __name__ == "__main__": + config = DataConfig() + data_path = config.data_path + row, col = config.height, config.width + + print("Pre-processing phase 1 data...") + filenames = glob.glob("{}/Challenge 2/Test/center/*.png".format(data_path)) + filenames = sorted(filenames) + out_name = "{}/X_test_round1_hsv_gray_diff_ch4".format(data_path) + X_test = make_hsv_grayscale_diff_data(4) + np.save(out_name, X_test) + + print("Pre-processing phase 2 data...") + filenames = glob.glob(DATA_PATH + "{}/Ch2_001/center/*.jpg".format(data_path)) + filenames = sorted(filenames) + out_name = DATA_PATH + "{}/X_test_round2_hsv_gray_diff_ch4".format(data_path) + X_test = make_hsv_grayscale_diff_data(4) + np.save(out_name, X_test) diff --git a/models/rambo/preprocess_train_data.py b/models/rambo/preprocess_train_data.py new file mode 100644 index 0000000..752f7a6 --- /dev/null +++ b/models/rambo/preprocess_train_data.py @@ -0,0 +1,189 @@ +from keras.preprocessing.image import load_img, img_to_array +import pandas as pd +import numpy as np +from skimage.exposure import rescale_intensity +from matplotlib.colors import rgb_to_hsv +from sklearn.model_selection import train_test_split +import os + +from models.rambo.config import DataConfig + + +def make_hsv_data(path): + df = pd.read_csv(path) + num_rows = df.shape[0] + + X = np.zeros((num_rows, row, col, 3), dtype=np.uint8) + for i in range(num_rows): + if i % 1000 == 0: + print("Processed " + str(i) + " images...") + + path = df['fullpath'].iloc[i] + img = load_img(data_path + path, target_size=(row, col)) + img = img_to_array(img) + img = rgb_to_hsv(img) + img = np.array(img, dtype=np.uint8) + + X[i] = img + + return X, np.array(df["angle"]) + + +def make_color_data(path): + df = pd.read_csv(path) + num_rows = df.shape[0] + + X = np.zeros((num_rows, row, col, 3), dtype=np.uint8) + for i in range(num_rows): + if i % 1000 == 0: + print("Processed " + str(i) + " images...") + + path = df['fullpath'].iloc[i] + img = load_img(data_path + path, target_size=(row, col)) + img = img_to_array(img) + img = np.array(img, dtype=np.uint8) + + X[i] = img + + return X, np.array(df["angle"]) + + +def make_grayscale_diff_data(path, num_channels=2): + df = pd.read_csv(path) + num_rows = df.shape[0] + + X = np.zeros((num_rows - num_channels, row, col, num_channels), dtype=np.uint8) + for i in range(num_channels, num_rows): + if i % 1000 == 0: + print("Processed " + str(i) + " images...") + for j in range(num_channels): + path0 = df['fullpath'].iloc[i - j - 1] + path1 = df['fullpath'].iloc[i - j] + img0 = load_img(data_path + path0, grayscale=True, target_size=(row, col)) + img1 = load_img(data_path + path1, grayscale=True, target_size=(row, col)) + img0 = img_to_array(img0) + img1 = img_to_array(img1) + img = img1 - img0 + img = rescale_intensity(img, in_range=(-255, 255), out_range=(0, 255)) + img = np.array(img, dtype=np.uint8) + + X[i - num_channels, :, :, j] = img[:, :, 0] + return X, np.array(df["angle"].iloc[num_channels:]) + + +def make_grayscale_diff_tx_data(path, num_channels=2): + df = pd.read_csv(path) + num_rows = df.shape[0] + + X = np.zeros((num_rows - num_channels, row, col, num_channels), dtype=np.uint8) + for i in range(num_channels, num_rows): + if i % 1000 == 0: + print("Processed " + str(i) + " images...") + path1 = df['fullpath'].iloc[i] + img1 = load_img(data_path + path1, grayscale=True, target_size=(row, col)) + img1 = img_to_array(img1) + for j in range(1, num_channels + 1): + path0 = df['fullpath'].iloc[i - j] + img0 = load_img(data_path + path0, grayscale=True, target_size=(row, col)) + img0 = img_to_array(img0) + + img = img1 - img0 + img = rescale_intensity(img, in_range=(-255, 255), out_range=(0, 255)) + img = np.array(img, dtype=np.uint8) + + X[i - num_channels, :, :, j - 1] = img[:, :, 0] + return X, np.array(df["angle"].iloc[num_channels:]) + + +def make_hsv_grayscale_diff_data(path, image_paths, angles, num_channels=2): + num_rows = min(len(image_paths), len(angles)) + + X = np.zeros((num_rows - num_channels, row, col, num_channels), dtype=np.uint8) + # for i in range(num_channels, num_rows): + for i in range(num_channels, 500): + if i % 1000 == 0: + print("Processed " + str(i) + " images...") + for j in range(num_channels): + path0 = image_paths[i - j - 2][0] + path1 = image_paths[i - j - 1][0] + img0 = load_img(path0, target_size=(row, col)) + img1 = load_img(path1, target_size=(row, col)) + img0.show() + img0 = img_to_array(img0) + img1 = img_to_array(img1) + img0 = rgb_to_hsv(img0) + img1 = rgb_to_hsv(img1) + img = img1[:, :, 2] - img0[:, :, 2] + img = rescale_intensity(img, in_range=(-255, 255), out_range=(0, 255)) + img = np.array(img, dtype=np.uint8) + + X[i - num_channels, :, :, j] = img + return X, np.array(angles[num_channels:]) + + +def load_data(): + """ + Load training data and split it into training and validation set + """ + tracks = [DATA_DIR] + + x = np.empty([0, 3]) + y = np.array([]) + for track in tracks: + drive = os.listdir(track) + for drive_style in drive: + try: + csv_name = 'driving_log.csv' + csv_folder = os.path.join(track, drive_style) + csv_path = os.path.join(csv_folder, csv_name) + + def fix_path(serie): + return serie.apply(lambda d: os.path.join(csv_folder, d)) + + data_df = pd.read_csv(csv_path) + pictures = data_df[['center', 'left', 'right']] + pictures_fixpath = pictures.apply(fix_path) + csv_x = pictures_fixpath.values + + csv_y = data_df['steering'].values + x = np.concatenate((x, csv_x), axis=0) + y = np.concatenate((y, csv_y), axis=0) + except FileNotFoundError: + print("Unable to read file %s" % csv_path) + exit() + + try: + X_train, X_valid, y_train, y_valid = train_test_split(x, y, test_size=0.2, random_state=0) + except TypeError: + print("Missing header to csv files") + exit() + + # print("Train dataset: " + str(len(X_train)) + " elements") + # print("Test dataset: " + str(len(X_valid)) + " elements") + return X_train, X_valid, y_train, y_valid + + +DATA_DIR = './training_recordings' + +if __name__ == "__main__": + config = DataConfig() + data_path = config.data_path + row, col = config.img_height, config.img_width + + X_train, X_valid, y_train, y_valid = load_data() + + print("Pre-processing phase 1 data...") + X_train_gray_diff, y_train_gray_diff = make_hsv_grayscale_diff_data("data/train_round1.txt", X_train, y_train, 2) + np.save(data_path + "/X_train_round1_hsv_gray_diff_ch4", X_train_gray_diff) + np.save(data_path + "/y_train_round1_hsv_gray_diff_ch4", y_train_gray_diff) + + X_val_gray_diff, y_val_gray_diff = make_hsv_grayscale_diff_data("data/val_round1.txt", X_valid, y_valid, 2) + np.save("{}/X_train_round1_hsv_gray_diff_ch4".format(data_path), X_val_gray_diff) + np.save("{}/y_train_round1_hsv_gray_diff_ch4".format(data_path), y_val_gray_diff) + + print("Pre-processing phase 2 data...") + for i in range(1, 6): + # X_train, y_train = make_hsv_grayscale_diff_data("data/train_round2_part" + str(i) + ".txt", X_train, y_train, + # 2) + np.save("{}/X_train_round2_hsv_gray_diff_ch4_part{}".format(data_path, i), X_train_gray_diff) + np.save("{}/y_train_round2_hsv_gray_diff_ch4_part{}".format(data_path, i), y_train_gray_diff) diff --git a/models/rambo/rambo_executor.py b/models/rambo/rambo_executor.py new file mode 100644 index 0000000..5de88f6 --- /dev/null +++ b/models/rambo/rambo_executor.py @@ -0,0 +1,62 @@ +from keras.preprocessing.image import img_to_array, load_img +from skimage.exposure import rescale_intensity +from matplotlib.colors import rgb_to_hsv +from PIL import Image +import cv2 +import numpy as np +from keras.models import load_model + +from models.rambo.config import TestConfig, DataConfig, TrainConfig +from code_pipeline.model_executor import ModelExecutor + + +class RamboModelExecutor(ModelExecutor): + + def __init__(self, result_folder, time_budget, map_size, + oob_tolerance=0.95, max_speed=70, + beamng_home=None, beamng_user=None, road_visualizer=None, model_path=None): + super(RamboModelExecutor, self).__init__(result_folder, time_budget, map_size, oob_tolerance, max_speed, + beamng_home, beamng_user, road_visualizer, model_path) + self.prev_image = None + self.last_images = [] + self.test_config = TestConfig() + self.data_config = DataConfig() + self.train_config = TrainConfig() + self.load_model() + self.X_train_mean = np.load(self.train_config.X_train_mean_path) + + def process(self, img): + return self.make_hsv_grayscale_diff_data(img) + + def make_hsv_grayscale_diff_data(self, image, num_channels=2): + X = np.zeros((2, self.data_config.img_height, self.data_config.img_width, num_channels), dtype=np.uint8) + prev_image = self.prev_image if self.prev_image is not None else image + self.prev_image = image + for j in range(num_channels): + # img0 = load_img(prev_image, target_size=(self.data_config.img_height, self.data_config.img_width)) + # img1 = load_img(image, target_size=(self.data_config.img_height, self.data_config.img_width)) + img0 = prev_image + img1 = image + img0 = img_to_array(img0) + img1 = img_to_array(img1) + img0 = rgb_to_hsv(img0) + img1 = rgb_to_hsv(img1) + img = img1[:, :, 2] - img0[:, :, 2] + img = rescale_intensity(img, in_range=(-255, 255), out_range=(0, 255)) + img = np.array(img, dtype=np.uint8) + + X[j, :, :, j] = img + X = X.astype("float32") + X -= self.X_train_mean + X /= 255.0 + return X + + def predict(self, img): + pil_img = cv2.cvtColor(np.asarray(img), cv2.COLOR_BGR2RGB) + pil_img = Image.fromarray(pil_img) + processed = self.process(pil_img) + prediction = self.model.predict(processed) + return prediction[0][0] + + def load_model(self): + self.model = load_model(self.test_config.model_path) diff --git a/models/rambo/train_rambo.py b/models/rambo/train_rambo.py new file mode 100644 index 0000000..b631cbd --- /dev/null +++ b/models/rambo/train_rambo.py @@ -0,0 +1,523 @@ +from keras.models import Sequential +from keras.layers import Dense, Dropout, Flatten, Lambda, ELU, Activation +from keras.layers.advanced_activations import LeakyReLU, PReLU +from keras.layers.convolutional import Convolution2D +from keras.layers.normalization import BatchNormalization +from keras.preprocessing.image import load_img, img_to_array +from keras.callbacks import ModelCheckpoint +import pandas as pd +import numpy as np +from sklearn.model_selection import train_test_split + +from models.rambo.config import TrainConfig +import os + + +def create_comma_model_relu(): + model = Sequential() + + model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode="same", input_shape=(row, col, ch))) + model.add(Activation('relu')) + model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Flatten()) + model.add(Activation('relu')) + model.add(Dense(512)) + model.add(Activation('relu')) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_comma_model_lrelu(): + model = Sequential() + + model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode="same", input_shape=(row, col, ch))) + model.add(LeakyReLU()) + model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(LeakyReLU()) + model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Flatten()) + # model.add(Dropout(.5)) + model.add(LeakyReLU()) + model.add(Dense(512)) + # model.add(Dropout(.5)) + model.add(LeakyReLU()) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_comma_model_prelu(): + model = Sequential() + + model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode="same", input_shape=(row, col, ch))) + model.add(PReLU()) + model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(PReLU()) + model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Flatten()) + # model.add(Dropout(.5)) + model.add(PReLU()) + model.add(Dense(512)) + # model.add(Dropout(.5)) + model.add(PReLU()) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_comma_model2(): + # additional dense layer + + model = Sequential() + + model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode="same", input_shape=(row, col, ch))) + model.add(Activation('relu')) + model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Flatten()) + model.add(Activation('relu')) + model.add(Dense(512)) + model.add(Activation('relu')) + model.add(Dense(256)) + model.add(Activation('relu')) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_comma_model3(): + # additional conv layer + model = Sequential() + + model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode="same", input_shape=(row, col, ch))) + model.add(Activation('relu')) + model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 3, 3, border_mode="same")) + model.add(Flatten()) + model.add(Activation('relu')) + model.add(Dense(512)) + model.add(Activation('relu')) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_comma_model4(): + # 2 additional conv layers + model = Sequential() + + model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode="same", input_shape=(row, col, ch))) + model.add(Activation('relu')) + model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 3, 3, border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 3, 3, border_mode="same")) + model.add(Flatten()) + model.add(Activation('relu')) + model.add(Dense(512)) + model.add(Activation('relu')) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_comma_model5(): + # more filters in first 2 conv layers + model = Sequential() + + model.add(Convolution2D(32, 8, 8, subsample=(4, 4), border_mode="same", input_shape=(row, col, ch))) + model.add(Activation('relu')) + model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Flatten()) + model.add(Activation('relu')) + model.add(Dense(512)) + model.add(Activation('relu')) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_comma_model6(): + # remove one conv layer + model = Sequential() + + model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode="same", input_shape=(row, col, ch))) + model.add(Activation('relu')) + model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Flatten()) + model.add(Activation('relu')) + model.add(Dense(256)) + model.add(Activation('relu')) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_comma_model_bn(): + model = Sequential() + + model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode="same", input_shape=(row, col, ch))) + model.add(BatchNormalization()) + model.add(Activation('relu')) + model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(BatchNormalization()) + model.add(Activation('relu')) + model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(BatchNormalization()) + model.add(Activation('relu')) + model.add(Flatten()) + + model.add(Dense(512)) + model.add(BatchNormalization()) + model.add(Activation('relu')) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_nvidia_model1(): + model = Sequential() + + model.add(Convolution2D(24, 5, 5, subsample=(2, 2), border_mode="same", input_shape=(row, col, ch))) + model.add(Activation('relu')) + model.add(Convolution2D(36, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(48, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 3, 3, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 3, 3, subsample=(2, 2), border_mode="same")) + model.add(Flatten()) + model.add(Activation('relu')) + model.add(Dense(100)) + model.add(Activation('relu')) + model.add(Dense(50)) + model.add(Activation('relu')) + model.add(Dense(10)) + model.add(Activation('relu')) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_nvidia_model2(): + model = Sequential() + + model.add(Convolution2D(24, 5, 5, subsample=(2, 2), border_mode="same", input_shape=(row, col, ch))) + model.add(Activation('relu')) + model.add(Convolution2D(36, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(48, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 3, 3, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 3, 3, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 3, 3, subsample=(2, 2), border_mode="same")) + model.add(Flatten()) + model.add(Activation('relu')) + model.add(Dense(100)) + model.add(Activation('relu')) + model.add(Dense(50)) + model.add(Activation('relu')) + model.add(Dense(10)) + model.add(Activation('relu')) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_nvidia_model3(): + model = Sequential() + + model.add(Convolution2D(24, 5, 5, subsample=(2, 2), border_mode="same", input_shape=(row, col, ch))) + model.add(Activation('relu')) + model.add(Convolution2D(36, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(48, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 3, 3, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 3, 3, subsample=(2, 2), border_mode="same")) + model.add(Activation('relu')) + model.add(Convolution2D(64, 3, 3, subsample=(2, 2), border_mode="same")) + model.add(Flatten()) + model.add(Activation('relu')) + model.add(Dense(256)) + model.add(Activation('relu')) + model.add(Dense(128)) + model.add(Activation('relu')) + model.add(Dense(64)) + model.add(Activation('relu')) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_comma_model_large(): + model = Sequential() + + model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode="same", input_shape=(row, col, ch))) + # model.add(ELU()) + model.add(Activation('relu')) + model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode="same")) + # model.add(ELU()) + model.add(Activation('relu')) + model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Flatten()) + # model.add(Dropout(.5)) + # model.add(ELU()) + model.add(Activation('relu')) + model.add(Dense(1024)) + # model.add(Dropout(.5)) + # model.add(ELU()) + model.add(Activation('relu')) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def create_comma_model_large_dropout(): + model = Sequential() + + model.add(Convolution2D(16, 8, 8, subsample=(4, 4), border_mode="same", input_shape=(row, col, ch))) + # model.add(ELU()) + model.add(Activation('relu')) + model.add(Convolution2D(32, 5, 5, subsample=(2, 2), border_mode="same")) + # model.add(ELU()) + model.add(Activation('relu')) + model.add(Convolution2D(64, 5, 5, subsample=(2, 2), border_mode="same")) + model.add(Flatten()) + # model.add(Dropout(.5)) + # model.add(ELU()) + model.add(Activation('relu')) + model.add(Dense(1024)) + model.add(Dropout(.5)) + # model.add(ELU()) + model.add(Activation('relu')) + model.add(Dense(1)) + + model.compile(optimizer="adam", loss="mse") + + print('Model is created and compiled..') + return model + + +def my_train_generator(): + num_iters = X_train.shape[0] / batch_size + while 1: + # print "Shuffling data..." + # train_idx_shf = np.random.permutation(X_train.shape[0]) + # X_train = X_train[train_idx_shf] + # y_train = y_train[train_idx_shf] + for i in range(int(num_iters)): + # idx = np.random.choice(X_train.shape[0], size=batch_size, replace=False) + idx = train_idx_shf[i * batch_size:(i + 1) * batch_size] + tmp = X_train[idx].astype('float32') + tmp -= X_train_mean + tmp /= 255.0 + yield tmp, y_train[idx] + + +def my_test_generator(): + num_iters = X_test.shape[0] / batch_size + while 1: + for i in range(int(num_iters)): + tmp = X_test[i * batch_size:(i + 1) * batch_size].astype('float32') + tmp -= X_train_mean + tmp /= 255.0 + yield tmp, y_test[i * batch_size:(i + 1) * batch_size] + + +if __name__ == "__main__": + config = TrainConfig() + + ch = config.num_channels + row = config.img_height + col = config.img_width + num_epoch = config.num_epoch + batch_size = config.batch_size + data_path = config.data_path + + print("Loading training data...") + print("Data path: " + data_path + "/X_train_round2_" + config.data_name + ".npy") + + X_train1 = np.load(data_path + "/X_train_round2_" + config.data_name + "_part1.npy") + y_train1 = np.load(data_path + "/y_train_round2_" + config.data_name + "_part1.npy") + X_train2 = np.load(data_path + "/X_train_round2_" + config.data_name + "_part2.npy") + y_train2 = np.load(data_path + "/y_train_round2_" + config.data_name + "_part2.npy") + X_train3 = np.load(data_path + "/X_train_round2_" + config.data_name + "_part3.npy") + y_train3 = np.load(data_path + "/y_train_round2_" + config.data_name + "_part3.npy") + X_train4 = np.load(data_path + "/X_train_round2_" + config.data_name + "_part4.npy") + y_train4 = np.load(data_path + "/y_train_round2_" + config.data_name + "_part4.npy") + X_train5 = np.load(data_path + "/X_train_round2_" + config.data_name + "_part5.npy") + y_train5 = np.load(data_path + "/y_train_round2_" + config.data_name + "_part5.npy") + + # use part4 as validation set + if config.val_part == 4: + X_train = np.concatenate((X_train1, X_train2, X_train3, X_train5), axis=0) + y_train = np.concatenate((y_train1, y_train2, y_train3, y_train5), axis=0) + X_test = X_train4 + y_test = y_train4 + # use part3 as validation set + elif config.val_part == 3: + X_train = np.concatenate((X_train1, X_train2, X_train4, X_train5), axis=0) + y_train = np.concatenate((y_train1, y_train2, y_train4, y_train5), axis=0) + X_test = X_train3 + y_test = y_train3 + # use last frames from all parts as validation set + elif config.val_part == 6: + X_train = np.concatenate((X_train1[:3800], X_train2[:13500], X_train3[:1680], X_train4[:3600], X_train5[:6300]), + axis=0) + y_train = np.concatenate((y_train1[:3800], y_train2[:13500], y_train3[:1680], y_train4[:3600], y_train5[:6300]), + axis=0) + X_test = np.concatenate((X_train1[3800:], X_train2[13500:], X_train3[1680:], X_train4[3600:], X_train5[6300:]), + axis=0) + y_test = np.concatenate((y_train1[3800:], y_train2[13500:], y_train3[1680:], y_train4[3600:], y_train5[6300:]), + axis=0) + # use part3 as validation set, but also use phase1 data for training + elif config.val_part == 33: + X_train = np.load(data_path + "/X_train_round1_" + config.data_name + ".npy") + y_train = np.load(data_path + "/y_train_round1_" + config.data_name + ".npy") + + X_train = np.concatenate((X_train, X_train1, X_train2, X_train4, X_train5), axis=0) + y_train = np.concatenate((y_train, y_train1, y_train2, y_train4, y_train5), axis=0) + X_test = X_train3 + y_test = y_train3 + # use part3 as validation set, but also use data from phase1 for training where steering angle is larger than 0.05 + # I split phase 1 data into internal train/val set, use val set as well + elif config.val_part == 333: + # phase 1 internal training set + X_train = np.load(data_path + "X_train_round1_" + config.data_name + ".npy") + y_train = np.load(data_path + "y_train_round1_" + config.data_name + ".npy") + + idx = np.abs(y_train) > 0.05 + X_train = X_train[idx] + y_train = y_train[idx] + + # phase 1 internal test set + X_train_test = np.load(data_path + "X_test_" + config.data_name + ".npy") + y_train_test = np.load(data_path + "y_test_" + config.data_name + ".npy") + + idx = np.abs(y_train_test) > 0.05 + X_train_test = X_train_test[idx] + y_train_test = y_train_test[idx] + + X_train = np.concatenate((X_train, X_train_test, X_train1, X_train2, X_train4, X_train5), axis=0) + y_train = np.concatenate((y_train, y_train_test, y_train1, y_train2, y_train4, y_train5), axis=0) + X_test = X_train3 + y_test = y_train3 + + print("X_train shape:" + str(X_train.shape)) + print("X_test shape:" + str(X_test.shape)) + print("y_train shape:" + str(y_train.shape)) + print("y_test shape:" + str(y_test.shape)) + + np.random.seed(1235) + train_idx_shf = np.random.permutation(X_train.shape[0]) + + print("Computing training set mean...") + X_train_mean = np.mean(X_train, axis=0, keepdims=True) + + print("Saving training set mean...") + np.save(config.X_train_mean_path, X_train_mean) + + print("Creating model...") + if config.model_name == "comma_prelu": + model = create_comma_model_prelu() + elif config.model_name == "comma_lrelu": + model = create_comma_model_lrelu() + elif config.model_name == "comma_bn": + model = create_comma_model_bn() + elif config.model_name == "comma2": + model = create_comma_model2() + elif config.model_name == "comma3": + model = create_comma_model3() + elif config.model_name == "comma4": + model = create_comma_model4() + elif config.model_name == "comma5": + model = create_comma_model5() + elif config.model_name == "comma6": + model = create_comma_model6() + elif config.model_name == "comma_large": + model = create_comma_model_large() + elif config.model_name == "comma_large_dropout": + model = create_comma_model_large_dropout() + elif config.model_name == "nvidia1": + model = create_nvidia_model1() + elif config.model_name == "nvidia2": + model = create_nvidia_model2() + elif config.model_name == "nvidia3": + model = create_nvidia_model3() + + model.summary() + + # checkpoint + filepath = data_path + "models/weights_" + config.data_name + "_" + config.model_name + "-{epoch:02d}-{val_loss:.5f}.hdf5" + checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True) + callbacks_list = [checkpoint] + + iters_train = X_train.shape[0] + iters_train -= iters_train % batch_size + iters_test = X_test.shape[0] + iters_test -= iters_test % batch_size + + model.fit_generator(my_train_generator(), + nb_epoch=num_epoch, + samples_per_epoch=iters_train, + validation_data=my_test_generator(), + nb_val_samples=iters_test, + callbacks=callbacks_list, + nb_worker=1 + ) diff --git a/requirements-37.txt b/requirements-37.txt index d4e892b..b77a23b 100644 --- a/requirements-37.txt +++ b/requirements-37.txt @@ -19,3 +19,5 @@ Shapely==1.7.1 six==1.15.0 typing-extensions==3.7.4.3 zipp==3.4.0 +tensorflow==2.4.0 +opencv-python==4.4.0.46 diff --git a/sample_test_generators/config.py b/sample_test_generators/config.py new file mode 100644 index 0000000..41213fe --- /dev/null +++ b/sample_test_generators/config.py @@ -0,0 +1,2 @@ +class TestGeneratorConfig: + tracks_path = "./runs" diff --git a/sample_test_generators/previous_generated_tracks_loader.py b/sample_test_generators/previous_generated_tracks_loader.py new file mode 100644 index 0000000..55c0f29 --- /dev/null +++ b/sample_test_generators/previous_generated_tracks_loader.py @@ -0,0 +1,44 @@ +import logging as log +import json +from glob import glob +from sample_test_generators.config import TestGeneratorConfig + +from code_pipeline.tests_generation import RoadTestFactory + + +class PreviousGeneratedTracksLoader(): + """ + Load tracks from previously generated runs. + """ + + def __init__(self, time_budget=None, executor=None, map_size=None): + self.time_budget = time_budget + self.executor = executor + self.map_size = map_size + config = TestGeneratorConfig() + self.tracks_path = config.tracks_path + + def start(self): + log.info("Load generated tracks") + + roads = [] + tracks_files = glob(self.tracks_path + "/*.json") + for file in tracks_files: + json_file = open(file, "r") + json_data = json.load(json_file) + roads.append(json_data["road_points"]) + + # Create RoadTests for all loaded tracks + tests = [] + for road_points in roads: + tests.append(RoadTestFactory.create_road_test(road_points)) + + # Execute all generated RoadTests + for test in tests: + test_outcome, description, execution_data = self.executor.execute_test(test) + # Print test outcome + log.info("test_outcome %s", test_outcome) + log.info("description %s", description) + + import time + time.sleep(10)