Implementation of NVIDIA End to end deep learning for self driving cars.

Keras/Autopilot_V2/AutopilotApp_V2.py

@@ -0,0 +1,45 @@
+import numpy as np
+import cv2
+from keras.models import load_model
+
+model = load_model('Autopilot.h5')
+
+def keras_predict(model, image):
+    processed = keras_process_image(image)
+    steering_angle = float(model.predict(processed, batch_size=1))
+    steering_angle = steering_angle * 60
+    return steering_angle
+
+
+def keras_process_image(img):
+    image_x = 100
+    image_y = 100
+    img = cv2.resize(img, (image_x, image_y))
+    img = np.array(img, dtype=np.float32)
+    img = np.reshape(img, (-1, image_x, image_y, 1))
+    return img
+
+
+steer = cv2.imread('steering_wheel_image.jpg', 0)
+rows, cols = steer.shape
+smoothed_angle = 0
+
+cap = cv2.VideoCapture('run.mp4')
+while (cap.isOpened()):
+    ret, frame = cap.read()
+    gray = cv2.resize((cv2.cvtColor(frame, cv2.COLOR_RGB2HSV))[:, :, 1], (100, 100))
+    steering_angle = keras_predict(model, gray)
+    print(steering_angle)
+    cv2.imshow('frame', cv2.resize(frame, (600, 400), interpolation=cv2.INTER_AREA))
+    smoothed_angle += 0.2 * pow(abs((steering_angle - smoothed_angle)), 2.0 / 3.0) * (
+        steering_angle - smoothed_angle) / abs(
+        steering_angle - smoothed_angle)
+    M = cv2.getRotationMatrix2D((cols / 2, rows / 2), -smoothed_angle, 1)
+    dst = cv2.warpAffine(steer, M, (cols, rows))
+    cv2.imshow("steering wheel", dst)
+
+    if cv2.waitKey(1) & 0xFF == ord('q'):
+        break
+
+cap.release()
+cv2.destroyAllWindows()

Keras/Autopilot_V2/LoadData_V2.py

@@ -0,0 +1,45 @@
+from __future__ import division
+import cv2
+import os
+import numpy as np
+import scipy
+import pickle
+import matplotlib.pyplot as plt
+from itertools import islice
+
+LIMIT = None
+
+DATA_FOLDER = 'driving_dataset'
+TRAIN_FILE = os.path.join(DATA_FOLDER, 'data.txt')
+
+def preprocess(img):
+    resized = cv2.resize((cv2.cvtColor(img, cv2.COLOR_RGB2HSV))[:, :, 1], (100, 100))
+    return resized
+
+def return_data():
+
+    X = []
+    y = []
+    features = []
+
+    with open(TRAIN_FILE) as fp:
+        for line in islice(fp, LIMIT):
+            path, angle = line.strip().split()
+            full_path = os.path.join(DATA_FOLDER, path)
+            X.append(full_path)
+            # using angles from -pi to pi to avoid rescaling the atan in the network
+            y.append(float(angle) * scipy.pi / 180)
+
+    for i in range(len(X)):
+        img = plt.imread(X[i])
+        features.append(preprocess(img))
+
+    features = np.array(features).astype('float32')
+    labels = np.array(y).astype('float32')
+
+    with open("features", "wb") as f:
+        pickle.dump(features, f, protocol=4)
+    with open("labels", "wb") as f:
+        pickle.dump(labels, f, protocol=4)
+
+return_data()

Keras/Autopilot_V2/Train_pilot_V2.py

@@ -0,0 +1,78 @@
+import numpy as np
+from keras.layers import Dense, Activation, Flatten, Conv2D, Lambda
+from keras.layers import MaxPooling2D, Dropout
+from keras.utils import print_summary
+from keras.models import Sequential
+from keras.callbacks import ModelCheckpoint
+import keras.backend as K
+import pickle
+
+from sklearn.model_selection import train_test_split
+from sklearn.utils import shuffle
+
+
+def keras_model(image_x, image_y):
+    model = Sequential()
+    model.add(Lambda(lambda x: x / 127.5 - 1., input_shape=(image_x, image_y, 1)))
+    model.add(Conv2D(32, (3, 3), padding='same'))
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), padding='valid'))
+    model.add(Conv2D(32, (3, 3), padding='same'))
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), padding='valid'))
+
+    model.add(Conv2D(64, (3, 3), padding='same'))
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), padding='valid'))
+    model.add(Conv2D(64, (3, 3), padding='same'))
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), padding='valid'))
+
+    model.add(Conv2D(128, (3, 3), padding='same'))
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), padding='valid'))
+    model.add(Conv2D(128, (3, 3), padding='same'))
+    model.add(Activation('relu'))
+    model.add(MaxPooling2D((2, 2), padding='valid'))
+
+    model.add(Flatten())
+    model.add(Dropout(0.5))
+    model.add(Dense(1024))
+    model.add(Dense(256))
+    model.add(Dense(64))
+    model.add(Dense(1))
+
+    model.compile(optimizer='adam', loss="mse")
+    filepath = "Autopilot_10.h5"
+    checkpoint = ModelCheckpoint(filepath, verbose=1, save_best_only=True)
+    callbacks_list = [checkpoint]
+
+    return model, callbacks_list
+
+
+def loadFromPickle():
+    with open("features", "rb") as f:
+        features = np.array(pickle.load(f))
+    with open("labels", "rb") as f:
+        labels = np.array(pickle.load(f))
+
+    return features, labels
+
+
+def main():
+    features, labels = loadFromPickle()
+    features, labels = shuffle(features, labels)
+    train_x, test_x, train_y, test_y = train_test_split(features, labels, random_state=0,
+                                                        test_size=0.3)
+    train_x = train_x.reshape(train_x.shape[0], 100, 100, 1)
+    test_x = test_x.reshape(test_x.shape[0], 100, 100, 1)
+    model, callbacks_list = keras_model(100, 100)
+    model.fit(train_x, train_y, validation_data=(test_x, test_y), epochs=3, batch_size=32,
+              callbacks=callbacks_list)
+    print_summary(model)
+
+    model.save('Autopilot_10.h5')
+
+
+main()
+K.clear_session();

Keras/Autopilot_V2/readme.md

@@ -0,0 +1,46 @@
+## Autopilot
+This code helps in getting the steering angle of self driving car. The inspiraion is taken from [End to End Learning for Self-Driving Cars](https://devblogs.nvidia.com/deep-learning-self-driving-cars/) module from NVIDIA
+
+The End to End Learning for Self-Driving Cars research paper can be found at (https://arxiv.org/abs/1604.07316)
+This repository uses convnets to predict steering angle according to the road. 
+
+1) [Autopilot Version 2](https://github.com/akshaybahadur21/Autopilot/tree/master/Autopilot_V2)
+
+### Code Requirements
+You can install Conda for python which resolves all the dependencies for machine learning.
+
+##### pip install requirements.txt
+
+### Description
+An autonomous car (also known as a driverless car, self-driving car, and robotic car) is a vehicle that is capable of sensing its environment and navigating without human input. Autonomous cars combine a variety of techniques to perceive their surroundings, including radar, laser light, GPS, odometry, and computer vision. Advanced control systems interpret sensory information to identify appropriate navigation paths, as well as obstacles and relevant signage
+
+## Autopilot V2 (NVIDIA Dataset based on real world)
+
+### Dataset
+Download the dataset at [here](https://github.com/SullyChen/driving-datasets) and extract into the repository folder
+
+### Python  Implementation
+
+1) Network Used- Convolutional Network
+2) Inspiration - End to End Learning for Self-Driving Cars by Nvidia
+
+If you face any problem, kindly raise an issue
+
+### Procedure
+
+1) First, run `LoadData_V2.py` which will get dataset from folder and store it in a pickle file after preprocessing.
+2) Now you need to have the data, run `Train_pilot.py` which will load data from pickle. After this, the training process begins.
+3) For testing it on the video, run `AutopilotApp_V2.py`
+
+<img src="https://github.com/akshaybahadur21/Autopilot/blob/master/v2.gif">
+
+### References:
+
+ - Mariusz Bojarski, Davide Del Testa, Daniel Dworakowski, Bernhard Firner, Beat Flepp, Prasoon Goyal, Lawrence D. Jackel, Mathew Monfort, Urs Muller, Jiakai Zhang, Xin Zhang, Jake Zhao, Karol Zieba. [End to End Learning for Self-Driving Cars](https://arxiv.org/abs/1604.07316)
+ - [Behavioral Cloning Project](https://github.com/udacity/CarND-Behavioral-Cloning-P3) 
+ - This implementation also took a lot of inspiration from the Sully Chen github repository: https://github.com/SullyChen/Autopilot-TensorFlow  
+
+
+
+
+