[ { "path": "README.md", "content": "# Self-Driving Car for GTA V\n### Overview\nThe aim of this project is to create a self-driving car using a virtual similator (particularly GTA V).\n\n### [Youtube Video](https://www.youtube.com/watch?v=BRK0wm7rrfQ)\n

\n \n

\n" }, { "path": "data_collection/data_balancing.py", "content": "import h5py\n\nfrom data_collection.data_collect import path as source_path\n\ndest_path = \"F:\\Graduation_Project\\\\training_data_balanced.h5\"\n\ndestination = h5py.File(dest_path, 'w')\ndestination.create_dataset('img', (0, 240, 320, 3), dtype='u1', maxshape=(None, 240, 320, 3), chunks=(30, 240, 320, 3))\ndestination.create_dataset('controls', (0, 2), dtype='i1', maxshape=(None, 2), chunks=(30, 2))\ndestination.create_dataset('metrics', (0, 2), dtype='u1', maxshape=(None, 2), chunks=(30, 2))\n\n\ndef save(data_img, controls, metrics):\n if data_img: # if the list is not empty\n destination[\"img\"].resize((destination[\"img\"].shape[0] + len(data_img)), axis=0)\n destination[\"img\"][-len(data_img):] = data_img\n destination[\"controls\"].resize((destination[\"controls\"].shape[0] + len(controls)), axis=0)\n destination[\"controls\"][-len(controls):] = controls\n destination[\"metrics\"].resize((destination[\"metrics\"].shape[0] + len(metrics)), axis=0)\n destination[\"metrics\"][-len(metrics):] = metrics\n\n\ndef main():\n source = h5py.File(source_path, 'r')\n images = []\n controls = []\n metrics = []\n\n tuples = 0\n straights = 0\n for i in range(source['img'].shape[0]):\n # if speed is not 0 and not arrived at the destination\n if source['metrics'][i][0] != 0 and source['metrics'][i][1] != 6:\n # save only each 5th straight drive frame\n if source['controls'][i][1] == 0:\n add = (straights % 5 == 0)\n straights += 1\n # save all turns\n else:\n add = True\n\n if add:\n images.append(source['img'][i])\n controls.append(source['controls'][i])\n metrics.append(source['metrics'][i])\n tuples += 1\n\n if tuples % 10000 == 0: # every 2.5 GB\n print(tuples)\n save(images, controls, metrics)\n images = []\n controls = []\n metrics = []\n\n save(images, controls, metrics)\n print(\"Copied: {:d} tuples from the source file\".format(tuples))\n\n source.close()\n destination.close()\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "data_collection/data_collect.py", "content": "\"\"\"\nData collection module (saves data in H5 format).\nSaves screen captures and pressed keys into a file\nfor further trainings of NN.\n\"\"\"\n\nimport os\nimport threading\nimport time\nimport winsound\n\nimport h5py\n\nfrom data_collection.gamepad_cap import Gamepad\nfrom data_collection.img_process import img_process\nfrom data_collection.key_cap import key_check\n\nlock = threading.Lock()\n\n# open the data file\npath = \"F:\\Graduation_Project\\\\training_data.h5\"\ndata_file = None\nif os.path.isfile(path):\n data_file = h5py.File(path, 'a')\nelse:\n data_file = h5py.File(path, 'w')\n # Write data in chunks for faster writing and reading by NN\n data_file.create_dataset('img', (0, 240, 320, 3), dtype='u1',\n maxshape=(None, 240, 320, 3), chunks=(30, 240, 320, 3))\n data_file.create_dataset('controls', (0, 2), dtype='i1', maxshape=(None, 2), chunks=(30, 2))\n data_file.create_dataset('metrics', (0, 2), dtype='u1', maxshape=(None, 2), chunks=(30, 2))\n\n\ndef save(data_img, controls, metrics):\n with lock: # make sure that data is consistent\n if data_img: # if the list is not empty\n # last_time = time.time()\n data_file[\"img\"].resize((data_file[\"img\"].shape[0] + len(data_img)), axis=0)\n data_file[\"img\"][-len(data_img):] = data_img\n data_file[\"controls\"].resize((data_file[\"controls\"].shape[0] + len(controls)), axis=0)\n data_file[\"controls\"][-len(controls):] = controls\n data_file[\"metrics\"].resize((data_file[\"metrics\"].shape[0] + len(metrics)), axis=0)\n data_file[\"metrics\"][-len(metrics):] = metrics\n # print('Saving took {} seconds'.format(time.time() - last_time))\n\n\ndef delete(session):\n frames = session if session < 500 else 500\n data_file[\"img\"].resize((data_file[\"img\"].shape[0] - frames), axis=0)\n data_file[\"controls\"].resize((data_file[\"controls\"].shape[0] - frames), axis=0)\n data_file[\"metrics\"].resize((data_file[\"metrics\"].shape[0] - frames), axis=0)\n\n\ndef main():\n # initialize gamepad\n gamepad = Gamepad()\n gamepad.open()\n\n # last_time = time.time() # to measure the number of frames\n alert_time = time.time() # to signal about exceeding speed limit\n close = False # to exit execution\n pause = True # to pause execution\n session = 0 # number of frames recorded in one session\n training_img = [] # lists for storing training data\n controls = []\n metrics = []\n\n print(\"Press RB on your gamepad to start recording\")\n while not close:\n while not pause:\n # read throttle and steering values from the gamepad\n throttle, steering = gamepad.get_state()\n # get screen, speed and direction\n ignore, screen, speed, direction = img_process(\"Grand Theft Auto V\")\n\n training_img.append(screen)\n controls.append([throttle, steering])\n metrics.append([speed, direction])\n session += 1\n\n if speed > 60 and time.time() - alert_time > 1:\n winsound.PlaySound('.\\\\resources\\\\alert.wav', winsound.SND_ASYNC)\n alert_time = time.time()\n\n # save the data every 30 iterations\n if len(training_img) % 30 == 0:\n # print(\"-\" * 30 + \"Saving\" + \"-\" * 30)\n threading.Thread(target=save, args=(training_img, controls, metrics)).start()\n training_img = []\n controls = []\n metrics = []\n\n time.sleep(0.015) # in order to slow down fps\n # print('Main loop took {} seconds'.format(time.time() - last_time))\n # last_time = time.time()\n\n if gamepad.get_RB():\n pause = True\n print('Paused. Save the last 15 seconds?')\n\n keys = key_check()\n while ('Y' not in keys) and ('N' not in keys):\n keys = key_check()\n\n if 'N' in keys:\n delete(session)\n training_img = []\n controls = []\n metrics = []\n print('Deleted.')\n else:\n print('Saved.')\n\n print('To exit the program press LB.')\n session = 0\n time.sleep(0.5)\n\n if gamepad.get_RB():\n pause = False\n print('Unpaused')\n time.sleep(1)\n elif gamepad.get_LB():\n gamepad.close()\n close = True\n print('Saving data and closing the program.')\n save(training_img, controls, metrics)\n\n data_file.close()\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "data_collection/gamepad_cap.py", "content": "\"\"\"\nModule for reading information from an Xbox gamepad\n\"\"\"\n\nimport threading\n\nfrom inputs import get_gamepad\n\n# Gamepad part\nAXIS_MAX = 32767\nAXIS_MIN = -32768\nTRIGGER_MAX = 255\nTRIGGER_MIN = -255\n\nAXIS_MAX_NORM = 10 / AXIS_MAX\nAXIS_MIN_NORM = -10 / AXIS_MIN\nTRIGGER_MAX_NORM = 10 / TRIGGER_MAX\nTRIGGER_MIN_NORM = -10 / TRIGGER_MIN\n\nDEADZONE = 3\n\n\nclass Gamepad:\n def __init__(self):\n self.x_axis = 0\n self.y_axisP = 0\n self.y_axisN = 0\n self.RB = 0\n self.LB = 0\n self.stop = False\n\n def open(self):\n self.stop = False\n threading.Thread(target=self.run).start()\n\n def run(self):\n while not self.stop:\n events = get_gamepad()\n for event in events:\n if event.code == \"ABS_X\":\n self.x_axis = event.state\n elif event.code == \"ABS_RZ\":\n self.y_axisP = event.state\n elif event.code == \"ABS_Z\":\n self.y_axisN = -event.state\n elif event.code == \"BTN_TR\":\n self.RB = event.state\n elif event.code == \"BTN_TL\":\n self.LB = event.state\n else:\n pass # we're not interested in the remain signals\n\n def get_state(self):\n xAxis = self.x_axis\n yAxis = self.y_axisP if self.y_axisP > 60 else self.y_axisN\n\n # normalize x axis\n if xAxis > 0:\n xAxis = int(round(xAxis * AXIS_MAX_NORM))\n else:\n xAxis = int(round(xAxis * AXIS_MIN_NORM))\n if -DEADZONE < xAxis < DEADZONE:\n xAxis = 0\n # normalize y axis\n if yAxis > 0:\n yAxis = int(round(yAxis * TRIGGER_MAX_NORM))\n else:\n yAxis = int(round(yAxis * TRIGGER_MIN_NORM))\n if -DEADZONE < yAxis < DEADZONE:\n yAxis = 0\n\n # return throttle and then steering\n return yAxis, xAxis\n\n def get_RB(self):\n return self.RB\n\n def get_LB(self):\n return self.LB\n\n def close(self):\n self.stop = True\n" }, { "path": "data_collection/histogram.py", "content": "\"\"\"\nHistogram of turns (for future balancing of data)\n\"\"\"\n\nimport h5py\nimport matplotlib.pyplot as plt\nimport numpy as np\n\nfrom data_collection.data_collect import path\n\nn_bins = [x - 0.5 for x in range(-10, 12)]\n\ndata = h5py.File(path, 'r')\n\nfig, axs = plt.subplots()\naxs.hist([d[1] for d in data['controls'][:]], bins=n_bins)\n\ndata.close()\nplt.xticks(np.arange(-10, 11, step=1))\nplt.show()\n" }, { "path": "data_collection/img_process.py", "content": "\"\"\"\nModule for preprocessing screen captures\n\"\"\"\n\nimport win32gui\nimport win32ui\n\nimport cv2\nimport numpy as np\nimport win32con\n\n\ndef initKNN(data, labels, shape):\n knn = cv2.ml.KNearest_create()\n train = np.load(data).reshape(-1, shape).astype(np.float32)\n train_labels = np.load(labels)\n knn.train(train, cv2.ml.ROW_SAMPLE, train_labels)\n return knn\n\n\nknnDigits = initKNN('..\\data_collection\\\\resources\\digits.npy',\n '..\\data_collection\\\\resources\\digits_labels.npy', 40)\nknnArrows = initKNN('..\\data_collection\\\\resources\\\\arrows.npy',\n '..\\data_collection\\\\resources\\\\arrows_labels.npy', 90)\n\n\n# Done by Frannecklp\ndef grab_screen(winName: str = \"Grand Theft Auto V\"):\n desktop = win32gui.GetDesktopWindow()\n\n # get area by a window name\n gtawin = win32gui.FindWindow(None, winName)\n # get the bounding box of the window\n left, top, x2, y2 = win32gui.GetWindowRect(gtawin)\n # cut window boarders\n top += 32\n left += 3\n y2 -= 4\n x2 -= 4\n width = x2 - left + 1\n height = y2 - top + 1\n\n # the device context(DC) for the entire window (title bar, menus, scroll bars, etc.)\n hwindc = win32gui.GetWindowDC(desktop)\n # Create a DC object from an integer handle\n srcdc = win32ui.CreateDCFromHandle(hwindc)\n # Create a memory device context that is compatible with the source DC\n memdc = srcdc.CreateCompatibleDC()\n # Create a bitmap object\n bmp = win32ui.CreateBitmap()\n # Create a bitmap compatible with the specified device context\n bmp.CreateCompatibleBitmap(srcdc, width, height)\n # Select an object into the device context.\n memdc.SelectObject(bmp)\n # Copy a bitmap from the source device context to this device context\n # parameters: destPos, size, dc, srcPos, rop(the raster operation))\n memdc.BitBlt((0, 0), (width, height), srcdc, (left, top), win32con.SRCCOPY)\n\n # the bitmap bits\n signedIntsArray = bmp.GetBitmapBits(True)\n # form a 1-D array initialized from text data in a string.\n img = np.fromstring(signedIntsArray, dtype='uint8')\n img.shape = (height, width, 4)\n\n # Delete all resources associated with the device context\n srcdc.DeleteDC()\n memdc.DeleteDC()\n # Releases the device context\n win32gui.ReleaseDC(desktop, hwindc)\n # Delete the bitmap and freeing all system resources associated with the object.\n # After the object is deleted, the specified handle is no longer valid.\n win32gui.DeleteObject(bmp.GetHandle())\n\n return cv2.cvtColor(img, cv2.COLOR_RGBA2RGB)\n\n\ndef predict(img, knn):\n ret, result, neighbours, dist = knn.findNearest(img, k=1)\n return result\n\n\ndef preprocess(img):\n gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n thr = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 7, -5)\n return thr\n\n\ndef convert_speed(num1, num2, num3):\n hundreds = 1\n tens = 1\n speed = 0\n\n if num3[0][0] != 10:\n hundreds = 10\n tens = 10\n speed += int(num3[0][0])\n if num2[0][0] != 10:\n speed += tens * int(num2[0][0])\n hundreds = tens * 10\n if num1[0][0] != 10:\n speed += hundreds * int(num1[0][0])\n\n return speed\n\n\ndef img_process(winName: str = \"Grand Theft Auto V\"):\n screen = grab_screen(winName)\n\n # Ji Hyun's computer\n numbers = preprocess(screen[567:575, 683:702, :])\n # Rustam's computer\n # numbers = preprocess(screen[573:581, 683:702, :])\n\n # three fields for numbers\n num1 = predict(numbers[:, :5].reshape(-1, 40).astype(np.float32), knnDigits)\n num2 = predict(numbers[:, 7:12].reshape(-1, 40).astype(np.float32), knnDigits)\n num3 = predict(numbers[:, -5:].reshape(-1, 40).astype(np.float32), knnDigits)\n\n # one field for direction arrows\n # Ji Hyun's computer\n direct = preprocess(screen[561:570, 18:28, :]).reshape(-1, 90).astype(np.float32)\n # Rustam's computer\n # direct = preprocess(screen[567:576, 18:28, :]).reshape(-1, 90).astype(np.float32)\n direct = int(predict(direct, knnArrows)[0][0])\n\n speed = convert_speed(num1, num2, num3)\n resized = cv2.resize(screen, (320, 240))\n\n return screen, resized, speed, direct\n" }, { "path": "data_collection/key_cap.py", "content": "# Citation: Box Of Hats (https://github.com/Box-Of-Hats)\n\n\"\"\"\nModule for reading keys from a keyboard\n\"\"\"\n\nimport win32api as wapi\n\nkeyList = [\"\\b\"]\nfor char in \"ABCDEFGHIJKLMNOPQRSTUVWXYZ 0123456789,.'£$/\\\\\":\n keyList.append(char)\n\n\ndef key_check():\n keys = []\n for key in keyList:\n if wapi.GetAsyncKeyState(ord(key)):\n keys.append(key)\n return keys\n" }, { "path": "drivers.txt", "content": "# For testing AI an XBox controller emulator is needed\n# https://github.com/shauleiz/ScpVBus/releases\n\nScpVBus\n\n# Installation:\n# In CMD (administrator): devcon.exe install ScpVBus.inf Root\\ScpVBus\n# Removal:\n# In CMD (administrator): devcon.exe remove Root\\ScpVBus\n" }, { "path": "driving/drive.py", "content": "\"\"\"\nCar driving module.\n\"\"\"\n\n# reading and writing files\nimport os\nimport time\n\nimport cv2\nimport numpy as np\n# load our saved model\nfrom keras.models import load_model\n\n# helper classes\nfrom data_collection.img_process import img_process\nfrom data_collection.key_cap import key_check\n# gamepad axes limits and gamepad module\nfrom driving.gamepad import AXIS_MIN, AXIS_MAX, TRIGGER_MAX, XInputDevice\nfrom object_detection.direction import Direct\n# YOLO algorithm\nfrom object_detection.object_detect import yolo_detection\n# lane detection algorithm\nfrom object_detection.lane_detect import detect_lane, draw_lane\nfrom training.utils import preprocess\n\nmodel_path = \"..\\\\training\"\ngamepad = None\n\n\ndef set_gamepad(controls):\n # trigger value\n trigger = int(round(controls[0][1] * TRIGGER_MAX))\n if trigger >= 0:\n # set left trigger to zero\n gamepad.SetTrigger('L', 0)\n gamepad.SetTrigger('R', trigger)\n else:\n # inverse value\n trigger = -trigger\n # set right trigger to zero\n gamepad.SetTrigger('L', trigger)\n gamepad.SetTrigger('R', 0)\n\n # axis value\n axis = 0\n if controls[0][0] >= 0:\n axis = int(round(controls[0][0] * AXIS_MAX))\n else:\n axis = int(round(controls[0][0] * (-AXIS_MIN)))\n gamepad.SetAxis('X', axis)\n\n\ndef drive(model):\n global gamepad\n gamepad = XInputDevice(1)\n gamepad.PlugIn()\n\n # last_time = time.time() # to measure the number of frames\n close = False # to exit execution\n pause = True # to pause execution\n stop = False # to stop the car\n throttle = 0\n left_line_max = 75\n right_line_max = 670\n\n print(\"Press T to start driving\")\n\n while not close:\n yolo_screen, resized, speed, direct = img_process(\"Grand Theft Auto V\")\n cv2.imshow(\"Driving-mode\", yolo_screen)\n cv2.waitKey(1)\n\n while not pause:\n # apply the preprocessing\n screen, resized, speed, direct = img_process(\"Grand Theft Auto V\")\n radar = cv2.cvtColor(resized[206:226, 25:45, :], cv2.COLOR_RGB2BGR)[:, :, 2:3]\n resized = preprocess(resized)\n left_line_color = [0, 255, 0]\n right_line_color = [0, 255, 0]\n\n # predict steering angle for the image\n # original + radar (small) + speed\n controls = model.predict([np.array([resized]), np.array([radar]), np.array([speed])], batch_size=1)\n # check that the car is following lane\n lane, stop_line = detect_lane(screen)\n # detect objects\n yolo_screen, color_detected, obj_distance = yolo_detection(screen, direct)\n\n if not stop:\n # adjusting speed\n if speed < 45:\n throttle = 0.4\n elif speed > 50:\n throttle = 0.0\n\n if 0 <= obj_distance <= 0.6:\n if speed < 5:\n throttle = 0\n else:\n throttle = -0.7 if obj_distance <= 0.4 else -0.3\n\n elif color_detected == \"Red\":\n if stop_line:\n if speed < 5:\n throttle = 0\n elif 0 <= stop_line[0][1] <= 50:\n throttle = -0.5\n elif 50 < stop_line[0][1] <= 120:\n throttle = -1\n # else:\n # throttle = -0.5\n elif speed > 5:\n throttle = -1\n else:\n throttle = 0\n cv2.destroyAllWindows()\n pause = True\n\n # adjusting steering angle\n if lane[0] and lane[0][0] > left_line_max:\n if abs(controls[0][0]) < 0.27:\n controls[0][0] = 0.27\n left_line_color = [0, 0, 255]\n elif lane[1] and lane[1][0] < right_line_max:\n if abs(controls[0][0]) < 0.27:\n controls[0][0] = -0.27\n right_line_color = [0, 0, 255]\n\n # set the gamepad values\n set_gamepad([[controls[0][0], throttle]])\n\n # print('Main loop took {} seconds'.format(time.time() - last_time))\n # last_time = time.time()\n\n screen[280:-130, :, :] = draw_lane(screen[280:-130, :, :], lane, stop_line,\n left_line_color, right_line_color)\n cv2.imshow(\"Driving-mode\", yolo_screen)\n cv2.waitKey(1)\n\n if direct == 6:\n print(\"Arrived at destination.\")\n stop = True\n\n # print('Main loop took {} seconds'.format(time.time() - last_time))\n # last_time = time.time()\n\n keys = key_check()\n if 'T' in keys:\n cv2.destroyAllWindows()\n pause = True\n # release gamepad keys\n set_gamepad([[0, 0]])\n print('Paused. To exit the program press Z.')\n time.sleep(0.5)\n\n keys = key_check()\n if 'T' in keys:\n pause = False\n stop = False\n print('Unpaused')\n time.sleep(1)\n elif 'Z' in keys:\n cv2.destroyAllWindows()\n close = True\n print('Closing the program.')\n gamepad.UnPlug()\n\n\ndef main():\n # load model\n location = os.path.join(model_path, 'base_model.h5')\n model = load_model(location)\n # control a car\n drive(model)\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "driving/gamepad.py", "content": "# This code based on Musi13's code (https://github.com/Musi13/pyvxbox)\n\n\"\"\"\nGamepad emulating module.\n\"\"\"\n\nimport sys\nfrom ctypes import *\n\ndll_path = \"vXboxInterface.dll\"\n\ntry:\n _vx = cdll.LoadLibrary(dll_path)\nexcept OSError as e:\n print(e)\n sys.exit(\"Unable to load vXbox SDK DLL. Ensure that %s is present\" % dll_path)\n\nif not _vx.isVBusExists():\n raise Exception('Xbox VBus does not exist')\n\nAXIS_MAX = 32767\nAXIS_MIN = -32768\nTRIGGER_MAX = 255\nBTN_ON = True\nBTN_OFF = False\n\n\nclass XInputDevice:\n def __init__(self, port):\n if _vx.isControllerExists(port):\n raise Exception('Port %d is already used' % port)\n self.UserIndex = port\n\n def PlugIn(self):\n _vx.PlugIn(self.UserIndex)\n\n def UnPlug(self, force=False):\n if not force:\n _vx.UnPlug(self.UserIndex)\n else:\n _vx.UnPlugForce(self.UserIndex)\n\n def SetBtn(self, button, value):\n function = {\n 'A': _vx.SetBtnA,\n 'B': _vx.SetBtnB,\n 'X': _vx.SetBtnX,\n 'Y': _vx.SetBtnY,\n 'Start': _vx.SetBtnStart,\n 'Back': _vx.SetBtnBack,\n 'LT': _vx.SetBtnLT,\n 'RT': _vx.SetBtnRT,\n 'LB': _vx.SetBtnLB,\n 'RB': _vx.SetBtnRB,\n 'GD': _vx.SetBtnGD\n }.get(button, None)\n if function is None:\n raise Exception('Unknown button %s' % str(button))\n function(self.UserIndex, value)\n\n def SetTrigger(self, trigger, value):\n function = {\n 'L': _vx.SetTriggerL,\n 'R': _vx.SetTriggerR\n }.get(trigger, None)\n if function is None:\n raise Exception('Unknown trigger %s' % str(trigger))\n function(self.UserIndex, value)\n\n def SetAxis(self, axis, value):\n function = {\n 'X': _vx.SetAxisX,\n 'Y': _vx.SetAxisY,\n 'Rx': _vx.SetAxisRx,\n 'Ry': _vx.SetAxisRy\n }.get(axis, None)\n if function is None:\n raise Exception('Unknown axis %s' % str(axis))\n function(self.UserIndex, value)\n\n def SetDpad(self, direction, value=0):\n function = {\n 'Up': _vx.SetDpadUp,\n 'Right': _vx.SetDpadRight,\n 'Down': _vx.SetDpadDown,\n 'Left': _vx.SetDpadLeft,\n '': _vx.SetDpad\n }.get(direction, None)\n if function is None:\n raise Exception('Unknown direction %s' % str(direction))\n if direction == '':\n function(self.UserIndex, value)\n else:\n function(self.UserIndex)\n\n def GetLedNumber(self, pLed):\n _vx.GetLedNumber(self.UserIndex, pLed)\n\n def GetVibration(self, pVib):\n _vx.GetVibration(self.UserIndex, pVib)\n" }, { "path": "game_plugins.txt", "content": "### List of plugins used in GTA V\n### for generating better conditions for AI\n\n# allows installation of plugins\nScript Hook V\n# for adjusting weather conditions, time, amount of car, pedestrians, etc.\nSimple Trainer for GTA V\n" }, { "path": "object_detection/direction.py", "content": "from enum import Enum\n\n\nclass Direct(Enum):\n STRAIGHT = 0\n LEFT = 1\n RIGHT = 2\n SLIGHTLY_LEFT = 3\n SLIGHTLY_RIGHT = 4\n U_TURN = 5\n ARRIVED = 6\n" }, { "path": "object_detection/lane_detect.py", "content": "import math\n\nimport cv2\nimport numpy as np\n\nfrom data_collection.img_process import grab_screen\n\nprev_lines = [[], [], []]\n\n\ndef crop(image):\n \"\"\"\n Crop the image (removing the sky at the top and the car front at the bottom)\n \"\"\"\n return image[280:-130, :, :]\n\n\ndef grayscale(img):\n \"\"\"\n Applies the Grayscale transform\n This will return an image with only one color channel\n \"\"\"\n return cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)\n\n\ndef canny(img, low_threshold=100, high_threshold=300):\n \"\"\"\n Applies the Canny transform\n \"\"\"\n return cv2.Canny(img, low_threshold, high_threshold)\n\n\ndef gaussian_blur(img, kernel_size):\n \"\"\"\n Applies a Gaussian Noise kernel\n \"\"\"\n return cv2.GaussianBlur(img, (kernel_size, kernel_size), sigmaX=30, sigmaY=30)\n\n\ndef region_of_interest(img, vertices):\n \"\"\"\n Applies an image mask.\n\n Only keeps the region of the image defined by the polygon\n formed from `vertices`. The rest of the image is set to black.\n `vertices` should be a numpy array of integer points.\n \"\"\"\n # defining a blank mask to start with\n mask = np.zeros_like(img)\n\n # defining a 3 channel or 1 channel color to fill the mask with depending on the input image\n if len(img.shape) > 2:\n channel_count = img.shape[2] # i.e. 3 or 4 depending on your image\n ignore_mask_color = (255,) * channel_count\n else:\n ignore_mask_color = 255\n\n # filling pixels inside the polygon defined by \"vertices\" with the fill color\n cv2.fillPoly(mask, vertices, ignore_mask_color)\n\n # returning the image only where mask pixels are nonzero\n masked_image = cv2.bitwise_and(img, mask)\n return masked_image\n\n\ndef construct_lane(lines):\n \"\"\"\n NOTE: this is the function you might want to use as a starting point once you want to\n average/extrapolate the line segments you detect to map out the full\n extent of the lane (going from the result shown in raw-lines-example.mp4\n to that shown in P1_example.mp4).\n\n Think about things like separating line segments by their\n slope ((y2-y1)/(x2-x1)) to decide which segments are part of the left\n line vs. the right line. Then, you can average the position of each of\n the lines and extrapolate to the top and bottom of the lane.\n\n This function draws `lines` with `color` and `thickness`.\n Lines are drawn on the image inplace (mutates the image).\n If you want to make the lines semi-transparent, think about combining\n this function with the add_images() function below\n \"\"\"\n left_line_x = []\n left_line_y = []\n right_line_x = []\n right_line_y = []\n stop_line_x_first = []\n stop_line_y_first = []\n stop_line_x_second = []\n stop_line_y_second = []\n\n lane = [[], []]\n stop_line = []\n\n min_y = 0\n max_y = 190\n\n if lines is not None:\n for line in lines:\n for x1, y1, x2, y2 in line:\n slope = (y2 - y1) / (x2 - x1) if x1 != x2 else 0 # <-- Calculating the slope.\n if 0.05 < math.fabs(slope) < 0.3: # not interested\n continue\n if math.fabs(slope) <= 0.05: # stop line\n if (y1 > 20) and (y2 > 20):\n # we need to detect two stop lines (top and bottom)\n if not stop_line_x_first or abs(stop_line_y_first[0] - y1) < 15:\n stop_line_x_first.extend([x1, x2])\n stop_line_y_first.extend([y1, y2])\n else:\n stop_line_x_second.extend([x1, x2])\n stop_line_y_second.extend([y1, y2])\n elif slope <= 0: # <-- If the slope is negative, left group.\n left_line_x.extend([x1, x2])\n left_line_y.extend([y1, y2])\n else: # <-- Otherwise, right group.\n right_line_x.extend([x1, x2])\n right_line_y.extend([y1, y2])\n\n offset = 7\n if left_line_x:\n poly_left = np.poly1d(np.polyfit(\n left_line_y,\n left_line_x,\n deg=1\n ))\n\n x1 = int(poly_left(max_y))\n x2 = int(poly_left(min_y))\n if prev_lines[0]:\n # recalculate x1\n if abs(x1 - prev_lines[0][0]) > offset:\n x1 = prev_lines[0][0] - offset if prev_lines[0][0] > x1 else prev_lines[0][0] + offset\n # recalculate x2\n if abs(x2 - prev_lines[0][1]) > offset:\n x2 = prev_lines[0][1] - offset if prev_lines[0][1] > x2 else prev_lines[0][1] + offset\n\n prev_lines[0] = [x1, x2]\n lane[0] = [x1, max_y, x2, min_y]\n elif prev_lines[0]:\n lane[0] = [prev_lines[0][0], max_y, prev_lines[0][1], min_y]\n prev_lines[0] = []\n\n if right_line_x:\n poly_right = np.poly1d(np.polyfit(\n right_line_y,\n right_line_x,\n deg=1\n ))\n\n x1 = int(poly_right(max_y))\n x2 = int(poly_right(min_y))\n if prev_lines[1]:\n # recalculate x1\n if abs(x1 - prev_lines[1][0]) > offset:\n x1 = prev_lines[1][0] - offset if prev_lines[1][0] > x1 else prev_lines[1][0] + offset\n # recalculate x2\n if abs(x2 - prev_lines[1][1]) > offset:\n x2 = prev_lines[1][1] - offset if prev_lines[1][1] > x2 else prev_lines[1][1] + offset\n\n prev_lines[1] = [x1, x2]\n lane[1] = [x1, max_y, x2, min_y]\n elif prev_lines[1]:\n lane[1] = [prev_lines[1][0], max_y, prev_lines[1][1], min_y]\n prev_lines[1] = []\n\n if stop_line_x_second:\n poly_stop = np.poly1d(np.polyfit(\n stop_line_x_first,\n stop_line_y_first,\n deg=1\n ))\n\n y1 = int(poly_stop(50))\n y2 = int(poly_stop(750))\n if prev_lines[2]:\n # recalculate y1\n if abs(y1 - prev_lines[2][0]) > offset:\n y1 = prev_lines[2][0] - offset if prev_lines[2][0] > y1 else prev_lines[2][0] + offset\n # recalculate y2\n if abs(y2 - prev_lines[2][1]) > offset:\n y2 = prev_lines[2][1] - offset if prev_lines[2][1] > y2 else prev_lines[2][1] + offset\n\n prev_lines[2] = [y1, y2]\n stop_line.append([50, y1, 750, y2])\n elif prev_lines[2]:\n stop_line.append([50, prev_lines[2][0], 750, prev_lines[2][1]])\n prev_lines[2] = []\n\n return lane, stop_line\n\n\ndef hough_lines(img, rho=6, theta=np.pi / 120, threshold=160, min_line_len=60, max_line_gap=10):\n \"\"\"\n `img` should be the output of a Canny transform.\n\n Returns an image with hough lines drawn.\n \"\"\"\n lines = cv2.HoughLinesP(img, rho, theta, threshold, np.array([]), minLineLength=min_line_len,\n maxLineGap=max_line_gap)\n return lines\n\n\n# Python 3 has support for cool math symbols.\ndef add_images(img, initial_img):\n \"\"\"\n `img` is the output of the hough_lines(), An image with lines drawn on it.\n Should be a blank image (all black) with lines drawn on it.\n\n `initial_img` should be the image before any processing.\n\n The result image is computed as follows:\n\n initial_img * α + img * β + γ\n NOTE: initial_img and img must be the same shape!\n \"\"\"\n return cv2.add(initial_img, img)\n\n\ndef draw_lane(original_img, lane, stop_line, left_color, right_color, thickness=5):\n img = np.zeros((original_img.shape[0], original_img.shape[1], 3), dtype=np.uint8)\n polygon_points = None\n offset_from_lane_edge = 8\n\n # draw lane lines\n if lane[0]:\n for x1, y1, x2, y2 in [lane[0]]:\n cv2.line(img, (int(x1), int(y1)), (int(x2), int(y2)), left_color, thickness)\n if lane[1]:\n for x1, y1, x2, y2 in [lane[1]]:\n cv2.line(img, (int(x1), int(y1)), (int(x2), int(y2)), right_color, thickness)\n\n # color the lane\n if lane[0] and lane[1]:\n lane_color = [40, 60, 0]\n for x1, y1, x2, y2 in [lane[0]]:\n p1 = (x1 + offset_from_lane_edge, y1)\n p2 = (x2 + offset_from_lane_edge, y2)\n\n for x1, y1, x2, y2 in [lane[1]]:\n p3 = (x2 - offset_from_lane_edge, y2)\n p4 = (x1 - offset_from_lane_edge, y1)\n\n polygon_points = np.array([[p1, p2, p3, p4]], np.int32)\n cv2.fillPoly(img, polygon_points, lane_color)\n\n # draw stop line\n if stop_line:\n for x1, y1, x2, y2 in stop_line:\n cv2.line(img, (int(x1), int(y1)), (int(x2), int(y2)), [0, 0, 255], thickness * 3)\n if polygon_points is not None:\n for px1, py1, px2, py2 in [lane[0]]:\n p1 = (px1 - offset_from_lane_edge, py1)\n p2 = (px2 - offset_from_lane_edge, py2)\n\n for px1, py1, px2, py2 in [lane[1]]:\n p3 = (px2 + offset_from_lane_edge, py2)\n p4 = (px1 + offset_from_lane_edge, py1)\n\n polygon_points = np.array([[p1, p2, p3, p4]], np.int32)\n\n img = region_of_interest(img, polygon_points)\n\n return add_images(img, original_img)\n\n\ndef detect_lane(screen):\n # 0. Crop the image\n image = crop(screen)\n # 1. convert to gray\n image = grayscale(image)\n # 2. apply gaussian filter\n image = gaussian_blur(image, 7)\n # 3. canny\n image = canny(image, 50, 100)\n # 4. ROI\n image = region_of_interest(image, np.array([[(0, 190), (0, 70), (187, 0),\n (613, 0), (800, 70), (800, 190)]], np.int32))\n # 5. Hough lines\n lines = hough_lines(image)\n # 6. construct lane\n return construct_lane(lines)\n\n\ndef main():\n while True:\n original_img = grab_screen()\n # 1. convert to gray\n image = grayscale(crop(original_img))\n # 2. apply gaussian filter\n image = gaussian_blur(image, 7)\n # 3. canny\n image = canny(image, 50, 100)\n # 4. ROI\n image = region_of_interest(image, np.array([[(0, 190), (0, 70), (187, 0),\n (613, 0), (800, 70), (800, 190)]], np.int32))\n # 5. Hough lines\n lines = hough_lines(image)\n # 6. construct lane\n lane, stop_line = construct_lane(lines)\n # 7. Place lane detection output on the original image\n original_img[280:-130, :, :] = draw_lane(original_img[280:-130, :, :], lane, stop_line, [0, 255, 0],\n [0, 255, 0])\n\n cv2.imshow(\"Frame\", original_img)\n key = cv2.waitKey(1) & 0xFF\n if key == ord(\"q\"):\n cv2.destroyAllWindows()\n break\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "object_detection/object_detect.py", "content": "import cv2\nimport numpy as np\nfrom darkflow.net.build import TFNet\nfrom shapely.geometry import box, Polygon\n\nfrom data_collection.img_process import grab_screen\nfrom object_detection.direction import Direct\n\n# set YOLO options\noptions = {\n 'model': 'cfg/yolo.cfg',\n 'load': 'yolov2.weights',\n 'threshold': 0.3,\n 'gpu': 0.5\n}\ntfnet = TFNet(options)\n\n# capture = cv2.VideoCapture('gta2.mp4')\nt = (0, 0, 0)\ncolors = [tuple(255 * np.random.rand(3)) for i in range(5)]\ncolors2 = [tuple(t) for j in range(15)]\n\n\ndef light_recog(frame, direct, traffic_lights):\n traffic_light = traffic_lights[0]\n\n # find out which traffic light to follow, if there are several\n if len(traffic_lights) > 1:\n # if we need to go to the right\n if direct == Direct.RIGHT or direct == Direct.SLIGHTLY_RIGHT:\n for tl in traffic_lights:\n if tl['topleft']['x'] > traffic_light['topleft']['x']:\n traffic_light = tl\n # straight or left\n else:\n for tl in traffic_lights:\n if tl['topleft']['x'] < traffic_light['topleft']['x']:\n traffic_light = tl\n\n # coordinates of the traffic light\n top_left = (traffic_light['topleft']['x'], traffic_light['topleft']['y'])\n bottom_right = (traffic_light['bottomright']['x'], traffic_light['bottomright']['y'])\n # crop the frame to the traffic light\n roi = frame[top_left[1]:bottom_right[1], top_left[0]:bottom_right[0]]\n hsv = cv2.cvtColor(roi, cv2.COLOR_BGR2HSV)\n color_detected = ''\n\n # possible color ranges for traffic lights\n red_lower = np.array([136, 87, 111], dtype=np.uint8)\n red_upper = np.array([180, 255, 255], dtype=np.uint8)\n\n yellow_lower = np.array([22, 60, 200], dtype=np.uint8)\n yellow_upper = np.array([60, 255, 255], dtype=np.uint8)\n\n green_lower = np.array([50, 100, 100], dtype=np.uint8)\n green_upper = np.array([70, 255, 255], dtype=np.uint8)\n\n # find what color the traffic light is showing\n red = cv2.inRange(hsv, red_lower, red_upper)\n yellow = cv2.inRange(hsv, yellow_lower, yellow_upper)\n green = cv2.inRange(hsv, green_lower, green_upper)\n\n kernel = np.ones((5, 5), np.uint8)\n\n red = cv2.dilate(red, kernel)\n res = cv2.bitwise_and(roi, roi, mask=red)\n green = cv2.dilate(green, kernel)\n res2 = cv2.bitwise_and(roi, roi, mask=green)\n\n (_, contours, hierarchy) = cv2.findContours(red, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n for contour in enumerate(contours):\n color_detected = \"Red\"\n\n (_, contours, hierarchy) = cv2.findContours(yellow, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n for contour in enumerate(contours):\n color_detected = \"Yellow\"\n\n (_, contours, hierarchy) = cv2.findContours(green, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n for contour in enumerate(contours):\n color_detected = \"Green\"\n\n if (0 <= top_left[1] and bottom_right[1] <= 437) and (244 <= top_left[0] and bottom_right[0] <= 630):\n frame = cv2.putText(frame, color_detected, bottom_right, cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2)\n\n frame = cv2.putText(frame, color_detected, bottom_right, cv2.FONT_HERSHEY_COMPLEX, 1, (255, 255, 255), 2)\n\n return frame, color_detected\n\n\ndef distance_to_car(frame, top_left, bottom_right):\n distance = None\n\n # myRoi_array= np.array([[(0, 490), (309, 269), (490, 270), (800,473)]])\n # process_img = region_of_interest(frame, myRoi_array)\n # cv2.imshow(\"precess_img\", process_img)\n\n # roi = Polygon([(15, 472), (330, 321), (470, 321), (796, 495)])\n roi = Polygon([(100, 470), (350, 280), (450, 280), (700, 470)])\n car = box(top_left[0], top_left[1], bottom_right[0], bottom_right[1])\n\n if roi.intersects(car):\n mid_x = (bottom_right[0] + top_left[0]) / 2\n mid_y = (top_left[1] + bottom_right[1]) / 2\n distance = round((1 - ((bottom_right[0] / 800) - (top_left[0] / 800))) ** 4, 1)\n frame = cv2.putText(frame, '{}'.format(distance), (int(mid_x), int(mid_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.7,\n (255, 255, 255), 2)\n cv2.putText(frame[top_left[1]:bottom_right[1], top_left[0]:bottom_right[0]],\n 'WARNING!', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 255), 3)\n\n return frame, distance\n\n\ndef distance_to_human(frame, top_left, bottom_right):\n distance = None\n\n roi = Polygon([(90, 470), (350, 280), (450, 280), (700, 470)])\n person = box(top_left[0], top_left[1], bottom_right[0], bottom_right[1])\n\n if roi.intersects(person):\n mid_x = (bottom_right[0] + top_left[0]) / 2\n mid_y = (top_left[1] + bottom_right[1]) / 2\n distance = round((1 - ((bottom_right[0] / 800) - (top_left[0] / 800))) ** 15, 1)\n frame = cv2.putText(frame, '{}'.format(distance), (int(mid_x), int(mid_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.7,\n (255, 255, 255), 2)\n cv2.putText(frame[top_left[1]:bottom_right[1], top_left[0]:bottom_right[0]],\n 'WARNING!', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 0, 255), 3)\n\n return frame, distance\n\n\ndef yolo_detection(screen, direct):\n # find objects on a frame by using YOLO\n results = tfnet.return_predict(screen[:-130, :, :])\n # create a list of detected traffic lights (might be several on a frame)\n traffic_lights = []\n color_detected = None\n distance = 1\n\n for color, color2, result in zip(colors, colors2, results):\n top_left = (result['topleft']['x'], result['topleft']['y'])\n bottom_right = (result['bottomright']['x'], result['bottomright']['y'])\n label = result['label']\n confidence = result['confidence']\n text = '{}: {:.0f}%'.format(label, confidence * 100)\n\n if label == 'traffic light' and confidence > 0.3:\n if 220 <= result['topleft']['x'] <= 630:\n traffic_lights.append(result)\n\n color = color2\n screen = cv2.rectangle(screen, top_left, bottom_right, color, 6)\n screen = cv2.putText(screen, text, top_left, cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2)\n\n if label == 'car' or label == 'bus' or label == 'truck' or label == 'train':\n screen, car_distance = distance_to_car(screen, top_left, bottom_right)\n\n if car_distance and 0 <= car_distance < distance:\n distance = car_distance\n\n screen = cv2.rectangle(screen, top_left, bottom_right, color, 6)\n screen = cv2.putText(screen, text, top_left, cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2)\n\n if label == 'person':\n screen, person_distance = distance_to_human(screen, top_left, bottom_right)\n\n if person_distance and 0 <= person_distance < distance:\n distance = person_distance\n\n screen = cv2.rectangle(screen, top_left, bottom_right, color, 6)\n screen = cv2.putText(screen, text, top_left, cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 0), 2)\n\n if traffic_lights:\n screen, color_detected = light_recog(screen, direct, traffic_lights)\n\n return screen, color_detected, distance\n\n\ndef main():\n while True:\n screen = grab_screen()\n screen, color_detected, obj_distance = yolo_detection(screen, 0)\n\n if color_detected:\n print(\"Color detected: \" + color_detected)\n if obj_distance != 1:\n print(\"Distance to obstacle: {}\".format(obj_distance))\n\n cv2.imshow(\"Frame\", screen)\n key = cv2.waitKey(1) & 0xFF\n if key == ord(\"q\"):\n cv2.destroyAllWindows()\n break\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "requirements.txt", "content": "### To install the packages type in the console:\n### pip install -r requirements.txt\n\nnumpy\nopencv-python\n# tensorflow\ntensorflow-gpu\n\n# Python for Window Extensions\npywin32\n# For data management\nh5py\n# A high-level neural networks API capable of running on top of TensorFlow\nKeras\n# Tools for data mining and data analysis\nscikit-learn\n# To read information from a gamepad\ninputs\n# for object detection module\nShapely\n# for YOLO\nCython\n" }, { "path": "training/model.py", "content": "\"\"\"\nNN model\n\"\"\"\n\nfrom keras.layers import Lambda, Conv2D, Dropout, Dense, Flatten, Concatenate, Input, MaxPooling2D\nfrom keras.models import Model\n\nfrom training.utils import INPUT_SHAPE, RADAR_SHAPE\n\n\n# original Nvidia model\n# def build_model(args):\n# \"\"\"\n# NVIDIA model used\n# Image normalization to avoid saturation and make gradients work better.\n# Convolution: 5x5, filter: 24, strides: 2x2, activation: ELU\n# Convolution: 5x5, filter: 36, strides: 2x2, activation: ELU\n# Convolution: 5x5, filter: 48, strides: 2x2, activation: ELU\n# Convolution: 3x3, filter: 64, strides: 1x1, activation: ELU\n# Convolution: 3x3, filter: 64, strides: 1x1, activation: ELU\n# Drop out (0.5)\n# Fully connected: neurons: 100, activation: ELU\n# Fully connected: neurons: 50, activation: ELU\n# Fully connected: neurons: 10, activation: ELU\n# Fully connected: neurons: 1 (output)\n# # the convolution layers are meant to handle feature engineering\n# the fully connected layer for predicting the steering angle.\n# dropout avoids overfitting\n# ELU(Exponential linear unit) function takes care of the Vanishing gradient problem.\n# \"\"\"\n# model = Sequential()\n# model.add(Lambda(lambda x: x / 127.5 - 1.0, input_shape=INPUT_SHAPE))\n# model.add(Conv2D(24, (5, 5), activation='elu', strides=(2, 2)))\n# model.add(Conv2D(36, (5, 5), activation='elu', strides=(2, 2)))\n# model.add(Conv2D(48, (5, 5), activation='elu', strides=(2, 2)))\n# model.add(Conv2D(64, (3, 3), activation='elu'))\n# model.add(Conv2D(64, (3, 3), activation='elu'))\n# model.add(Dropout(args.keep_prob))\n# model.add(Flatten())\n# model.add(Dense(100, activation='elu'))\n# model.add(Dense(50, activation='elu'))\n# model.add(Dense(10, activation='elu'))\n# model.add(Dense(1))\n# model.summary()\n#\n# return model\n\n\n# original + radar added\n# def build_model(args):\n# # image model\n# img_input = Input(shape=INPUT_SHAPE)\n# img_model = (Lambda(lambda x: x / 127.5 - 1.0, input_shape=INPUT_SHAPE))(img_input)\n# img_model = (Conv2D(24, (5, 5), activation='elu', strides=(2, 2)))(img_model)\n# img_model = (Conv2D(36, (5, 5), activation='elu', strides=(2, 2)))(img_model)\n# img_model = (Conv2D(48, (5, 5), activation='elu', strides=(2, 2)))(img_model)\n# img_model = (Conv2D(64, (3, 3), activation='elu'))(img_model)\n# img_model = (Conv2D(64, (3, 3), activation='elu'))(img_model)\n# img_model = (Dropout(args.keep_prob))(img_model)\n# img_model = (Flatten())(img_model)\n# img_model = (Dense(100, activation='elu'))(img_model)\n#\n# # radar model\n# radar_input = Input(shape=RADAR_SHAPE)\n# radar_model = (Conv2D(10, (5, 5), activation='elu'))(radar_input)\n# radar_model = (MaxPooling2D((2, 2)))(radar_model)\n# radar_model = (Conv2D(20, (5, 5), activation='elu'))(radar_model)\n# radar_model = (MaxPooling2D((2, 2)))(radar_model)\n# radar_model = (Dropout(args.keep_prob / 2))(radar_model)\n# radar_model = (Flatten())(radar_model)\n# radar_model = (Dense(30, activation='elu'))(radar_model)\n#\n# # combined model\n# out = Concatenate()([img_model, radar_model])\n# out = (Dense(50, activation='elu'))(out)\n# out = (Dense(10, activation='elu'))(out)\n# out = (Dense(1))(out)\n#\n# final_model = Model(inputs=[img_input, radar_input], outputs=out)\n# final_model.summary()\n#\n# return final_model\n\n\n# original + radar and speed info added\ndef build_model(args):\n # image model\n img_input = Input(shape=INPUT_SHAPE)\n img_model = (Lambda(lambda x: x / 127.5 - 1.0, input_shape=INPUT_SHAPE))(img_input)\n img_model = (Conv2D(24, (5, 5), activation='elu', strides=(2, 2)))(img_model)\n img_model = (Conv2D(36, (5, 5), activation='elu', strides=(2, 2)))(img_model)\n img_model = (Conv2D(48, (5, 5), activation='elu', strides=(2, 2)))(img_model)\n img_model = (Conv2D(64, (3, 3), activation='elu'))(img_model)\n img_model = (Conv2D(64, (3, 3), activation='elu'))(img_model)\n img_model = (Dropout(args.keep_prob))(img_model)\n img_model = (Flatten())(img_model)\n img_model = (Dense(100, activation='elu'))(img_model)\n\n # radar model\n radar_input = Input(shape=RADAR_SHAPE)\n radar_model = (Conv2D(32, (5, 5), activation='elu'))(radar_input)\n radar_model = (MaxPooling2D((2, 2), strides=(2, 2)))(radar_model)\n radar_model = (Conv2D(64, (5, 5), activation='elu'))(radar_model)\n radar_model = (MaxPooling2D((2, 2), strides=(2, 2)))(radar_model)\n radar_model = (Dropout(args.keep_prob / 2))(radar_model)\n radar_model = (Flatten())(radar_model)\n radar_model = (Dense(10, activation='elu'))(radar_model)\n\n # speed\n speed_input = Input(shape=(1,))\n\n # combined model\n out = Concatenate()([img_model, radar_model])\n out = (Dense(50, activation='elu'))(out)\n out = Concatenate()([out, speed_input])\n out = (Dense(10, activation='elu'))(out)\n out = (Dense(1))(out)\n\n final_model = Model(inputs=[img_input, radar_input, speed_input], outputs=out)\n final_model.summary()\n\n return final_model\n\n# original + throttle control\n# def build_model(args):\n# \"\"\"\n# NVIDIA model used\n# Image normalization to avoid saturation and make gradients work better.\n# Convolution: 5x5, filter: 24, strides: 2x2, activation: ELU\n# Convolution: 5x5, filter: 36, strides: 2x2, activation: ELU\n# Convolution: 5x5, filter: 48, strides: 2x2, activation: ELU\n# Convolution: 3x3, filter: 64, strides: 1x1, activation: ELU\n# Convolution: 3x3, filter: 64, strides: 1x1, activation: ELU\n# Drop out (0.5)\n# Fully connected: neurons: 100, activation: ELU\n# Fully connected: neurons: 50, activation: ELU\n# Fully connected: neurons: 10, activation: ELU\n# Fully connected: neurons: 1 (output)\n# # the convolution layers are meant to handle feature engineering\n# the fully connected layer for predicting the steering angle.\n# dropout avoids overfitting\n# ELU(Exponential linear unit) function takes care of the Vanishing gradient problem.\n# \"\"\"\n# # image model\n# img_input = Input(shape=INPUT_SHAPE)\n# img_model = (Lambda(lambda x: x / 127.5 - 1.0, input_shape=INPUT_SHAPE))(img_input)\n# img_model = (Conv2D(24, (5, 5), activation='elu', strides=(2, 2)))(img_model)\n# img_model = (Conv2D(36, (5, 5), activation='elu', strides=(2, 2)))(img_model)\n# img_model = (Conv2D(48, (5, 5), activation='elu', strides=(2, 2)))(img_model)\n# img_model = (Conv2D(64, (3, 3), activation='elu'))(img_model)\n# img_model = (Conv2D(64, (3, 3), activation='elu'))(img_model)\n# img_model = (Dropout(args.keep_prob))(img_model)\n# img_model = (Flatten())(img_model)\n# img_model = (Dense(100, activation='elu'))(img_model)\n#\n# # speed and direction model\n# metrics_input = Input(shape=(2,))\n# metrics_model = Dense(2, activation='elu')(metrics_input)\n#\n# # combined model\n# out = Concatenate()([img_model, metrics_model])\n# out = (Dense(50, activation='elu'))(out)\n# out = (Dense(10, activation='elu'))(out)\n# out = (Dense(2))(out)\n#\n# final_model = Model(inputs=[img_input, metrics_input], outputs=out)\n# final_model.summary()\n#\n# return final_model\n" }, { "path": "training/train.py", "content": "# This code based on Siraj Raval's code (https://github.com/llSourcell/How_to_simulate_a_self_driving_car)\n\n\"\"\"\nTraining module. Based on \"End to End Learning for Self-Driving Cars\" research paper by Nvidia.\n\"\"\"\n\nimport argparse\n\nimport h5py\nimport numpy as np\nfrom keras.callbacks import ModelCheckpoint\nfrom keras.models import load_model\nfrom keras.optimizers import Adam\nfrom sklearn.model_selection import train_test_split # to split out training and testing data\n\n# path with training files\nfrom data_collection.data_collect import path\nfrom training.model import build_model\n# helper class\nfrom training.utils import batch_generator\n\n# for debugging, allows for reproducible (deterministic) results\nnp.random.seed(0)\n\n\ndef load_data(args):\n \"\"\"\n Load training data and split it into training and validation set\n \"\"\"\n data = h5py.File(path, 'r')\n # list of all possible indexes\n indexes = list(range(data['img'].shape[0]))\n # split the data into a training (80), testing(20), and validation set\n indexes_train, indexes_valid = train_test_split(indexes, test_size=args.test_size, random_state=0)\n\n return data, indexes_train, indexes_valid\n\n\ndef load_weights(model):\n \"\"\"\n Load weights from previously trained model\n \"\"\"\n prev_model = load_model(\"..\\\\training\\\\base_model.h5\")\n model.set_weights(prev_model.get_weights())\n\n return model\n\n\ndef train_model(model, args, data, indexes_train, indexes_valid):\n \"\"\"\n Train the model\n \"\"\"\n # Saves the model after every epoch.\n # quantity to monitor, verbosity i.e logging mode (0 or 1),\n # if save_best_only is true the latest best model according to the quantity monitored will not be overwritten.\n # mode: one of {auto, min, max}. If save_best_only=True, the decision to overwrite the current save file is\n # made based on either the maximization or the minimization of the monitored quantity. For val_acc,\n # this should be max, for val_loss this should be min, etc. In auto mode, the direction is automatically\n # inferred from the name of the monitored quantity.\n checkpoint = ModelCheckpoint('model-{epoch:03d}.h5',\n monitor='val_loss',\n verbose=0,\n save_best_only=args.save_best_only,\n mode='auto')\n\n # calculate the difference between expected steering angle and actual steering angle\n # square the difference\n # add up all those differences for as many data points as we have\n # divide by the number of them\n # that value is our mean squared error! this is what we want to minimize via\n # gradient descent\n model.compile(loss='mean_squared_error', optimizer=Adam(lr=args.learning_rate))\n\n # Fits the model on data generated batch-by-batch by a Python generator.\n\n # The generator is run in parallel to the model, for efficiency.\n # For instance, this allows you to do real-time data augmentation on images on CPU in\n # parallel to training your model on GPU.\n # so we reshape our data into their appropriate batches and train our model simultaneously\n model.fit_generator(batch_generator(data, indexes_train, args.batch_size, True),\n steps_per_epoch=len(indexes_train) / args.batch_size,\n epochs=args.nb_epoch,\n max_queue_size=1,\n validation_data=batch_generator(data, indexes_valid, args.batch_size, False),\n validation_steps=len(indexes_valid) / args.batch_size,\n callbacks=[checkpoint],\n verbose=1)\n\n\n# for command line args\ndef s2b(s):\n \"\"\"\n Converts a string to boolean value\n \"\"\"\n s = s.lower()\n return s == 'true' or s == 'yes' or s == 'y' or s == '1'\n\n\ndef main():\n \"\"\"\n Load train/validation data set and train the model\n \"\"\"\n # The argparse module makes it easy to write user-friendly command-line interfaces.\n parser = argparse.ArgumentParser(description='Behavioral Cloning Training Program')\n parser.add_argument('-d', help='data directory', dest='data_dir', type=str, default=path)\n parser.add_argument('-t', help='test size fraction', dest='test_size', type=float, default=0.2)\n parser.add_argument('-k', help='drop out probability', dest='keep_prob', type=float, default=0.5)\n parser.add_argument('-n', help='number of epochs', dest='nb_epoch', type=int, default=200)\n parser.add_argument('-b', help='batch size', dest='batch_size', type=int, default=500)\n parser.add_argument('-o', help='save best models only', dest='save_best_only', type=s2b, default='true')\n parser.add_argument('-l', help='learning rate', dest='learning_rate', type=float, default=1.0e-4)\n args = parser.parse_args()\n\n # print parameters\n print('-' * 30)\n print('Parameters')\n print('-' * 30)\n for key, value in vars(args).items():\n print('{:<20} := {}'.format(key, value))\n print('-' * 30)\n\n # load data\n data = load_data(args)\n # build model\n model = build_model(args)\n # load previous weights\n model = load_weights(model)\n # train model on data, it saves as model.h5\n train_model(model, args, *data)\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "training/utils.py", "content": "# This code based on Siraj Raval's code (https://github.com/llSourcell/How_to_simulate_a_self_driving_car)\n\nimport math\n\nimport cv2\nimport numpy as np\nimport tensorflow as tf\n\nIMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS = 66, 200, 3\nINPUT_SHAPE = (IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS)\nRADAR_HEIGHT, RADAR_WIDTH, RADAR_CHANNELS = 20, 20, 1\nRADAR_SHAPE = (RADAR_HEIGHT, RADAR_WIDTH, RADAR_CHANNELS)\n\n\ndef crop(image):\n \"\"\"\n Crop the image (removing the sky at the top and the car front at the bottom)\n \"\"\"\n return image[90:-50, :, :]\n\n\ndef resize(image):\n \"\"\"\n Resize the image to the input shape used by the network model\n \"\"\"\n return cv2.resize(image, (IMAGE_WIDTH, IMAGE_HEIGHT), cv2.INTER_AREA)\n\n\ndef rgb2yuv(image):\n \"\"\"\n Convert the image from RGB to YUV (This is what the NVIDIA model does)\n \"\"\"\n return cv2.cvtColor(image, cv2.COLOR_RGB2YUV)\n\n\ndef preprocess(image):\n \"\"\"\n Combine all preprocess functions into one\n \"\"\"\n image = crop(image)\n image = resize(image)\n image = rgb2yuv(image)\n return image\n\n\n# def choose_image(data_dir, center, left, right, steering_angle):\n# \"\"\"\n# Randomly choose an image from the center, left or right, and adjust\n# the steering angle.\n# \"\"\"\n# choice = np.random.choice(3)\n# if choice == 0:\n# return load_image(data_dir, left), steering_angle + 0.2\n# elif choice == 1:\n# return load_image(data_dir, right), steering_angle - 0.2\n# return load_image(data_dir, center), steering_angle\n\n\n# flip image causes car riding on the opposite direction lane\n# def random_flip(image, steering_angle):\n# \"\"\"\n# Randomly flip the image left <-> right, and adjust the steering angle.\n# \"\"\"\n# if np.random.rand() < 0.5:\n# image = cv2.flip(image, 1)\n# steering_angle = -steering_angle\n# return image, steering_angle\n\n\ndef random_translate(image, steering_angle, range_x, range_y):\n \"\"\"\n Randomly shift the image vertically and horizontally (translation).\n \"\"\"\n trans_x = range_x * (np.random.rand() - 0.5)\n trans_y = range_y * (np.random.rand() - 0.5)\n\n # adjusting steering angle\n t_x = trans_x / 25\n if t_x > 0:\n t_x = math.ceil(t_x)\n if t_x > 2:\n steering_angle += (t_x - 2)\n if steering_angle > 10:\n steering_angle = 10\n else:\n t_x = math.floor(t_x)\n if t_x < -2:\n steering_angle += (t_x + 2)\n if steering_angle < -10:\n steering_angle = -10\n\n trans_m = np.float32([[1, 0, trans_x], [0, 1, trans_y]])\n height, width = image.shape[:2]\n # apply an affine transformation to an image\n image = cv2.warpAffine(image, trans_m, (width, height))\n return image, steering_angle\n\n\ndef random_shadow(image):\n \"\"\"\n Generates and adds random shadow\n \"\"\"\n # (x1, y1) and (x2, y2) forms a line\n # xm, ym gives all the locations of the image\n x1, y1 = IMAGE_WIDTH * np.random.rand(), 0\n x2, y2 = IMAGE_WIDTH * np.random.rand(), IMAGE_HEIGHT\n xm, ym = np.mgrid[0:IMAGE_HEIGHT, 0:IMAGE_WIDTH]\n\n # mathematically speaking, we want to set 1 below the line and zero otherwise\n # Our coordinate is up side down. So, the above the line:\n # (ym-y1)/(xm-x1) > (y2-y1)/(x2-x1)\n # as x2 == x1 causes zero-division problem, we'll write it in the below form:\n # (ym-y1)*(x2-x1) - (y2-y1)*(xm-x1) > 0\n mask = np.zeros_like(image[:, :, 1])\n mask[np.where((ym - y1) * (x2 - x1) - (y2 - y1) * (xm - x1) > 0)] = 1\n\n # choose which side should have shadow and adjust saturation\n cond = mask == np.random.randint(2)\n s_ratio = np.random.uniform(low=0.2, high=0.5)\n\n # adjust Saturation in HLS(Hue, Light, Saturation)\n hls = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)\n hls[:, :, 1][cond] = hls[:, :, 1][cond] * s_ratio\n return cv2.cvtColor(hls, cv2.COLOR_HLS2RGB)\n\n\ndef random_brightness(image):\n \"\"\"\n Randomly adjust brightness of the image.\n \"\"\"\n # HSV (Hue, Saturation, Value) is also called HSB ('B' for Brightness).\n hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)\n ratio = 1.0 + 0.4 * (np.random.rand() - 0.5)\n hsv[:, :, 2] = hsv[:, :, 2] * ratio\n return cv2.cvtColor(hsv, cv2.COLOR_HSV2RGB)\n\n\ndef augment(image, steering_angle, range_x=250, range_y=20):\n \"\"\"\n Generate an augmented image and adjust steering angle.\n (The steering angle is associated with the center image)\n \"\"\"\n # image, steering_angle = choose_image(data_dir, center, left, right, steering_angle)\n # image, steering_angle = random_flip(image, steering_angle)\n image, steering_angle = random_translate(image, steering_angle, range_x, range_y)\n image = random_shadow(image)\n image = random_brightness(image)\n return image, steering_angle\n\n\ndef batch_generator(data, indexes, batch_size, is_training):\n \"\"\"\n Generate training image give image paths and associated steering angles\n \"\"\"\n # preprocessing on the CPU\n with tf.device('/cpu:0'):\n images = np.empty([batch_size, IMAGE_HEIGHT, IMAGE_WIDTH, IMAGE_CHANNELS])\n radars = np.empty([batch_size, RADAR_HEIGHT, RADAR_WIDTH, RADAR_CHANNELS])\n # metrics = np.empty([batch_size, 2])\n # controls = np.empty([batch_size, 2])\n speeds = np.empty(batch_size)\n controls = np.empty(batch_size)\n while True:\n i = 0\n for index in np.random.permutation(indexes):\n camera = data['img'][index]\n radar = cv2.cvtColor(camera[206:226, 25:45, :], cv2.COLOR_RGB2BGR)\n steer = data['controls'][index][1]\n\n # augmentation\n if is_training:\n prob = np.random.rand()\n if (abs(steer) < 0.4 and prob > 0.2) or (prob < 0.6):\n camera, steer = augment(camera, steer)\n\n # add the image and steering angle to the batch\n images[i] = preprocess(camera)\n radars[i] = radar[:, :, 2:3]\n # controls[i] = [data['controls'][index][0] / 10, steer / 10] # normalized throttle and steering\n controls[i] = steer / 10\n speeds[i] = data['metrics'][index][0]\n # metrics[i] = data['metrics'][index]\n i += 1\n if i == batch_size:\n break\n # yield [images, metrics], controls\n yield [images, radars, speeds], controls\n" } ]