[ { "path": ".gitignore", "content": "**/*.jpeg\n**/*.jpg\n**/*.png\n**/.DS_Store\n**/env\n**/*.pyc\ntraining/data\ntraining/data_reviewed\n" }, { "path": "Makefile", "content": "PI_IP_ADDRESS=10.0.0.1\nPI_USERNAME=pi\nSTREAM_URL=rtsp://username:password@camera_host/endpoint\n\n.PHONY: run\nrun:\n\t@. env/bin/activate && cd src && export STREAM_URL=$(STREAM_URL) && python app.py\n\n.PHONY: install\ninstall:\n\t@cd scripts && bash install.sh\n\n.PHONY: copy\ncopy:\n\t@rsync -a $(shell pwd) --exclude env --exclude training $(PI_USERNAME)@$(PI_IP_ADDRESS):/home/$(PI_USERNAME)\n\n.PHONY: shell\nshell:\n\t@ssh $(PI_USERNAME)@$(PI_IP_ADDRESS)\n\n.PHONY: server\nserver:\n\t@echo \"Running in server mode\";\n\t@. env/bin/activate && cd src && python server.py\n" }, { "path": "README.md", "content": "# Package Theft Prevention Device\nAn AI-powered device to stop people from stealing my packages.\n\n## Installation\nTo install on a raspberry pi, clone the repository and run:\n```bash\nmake install\n```\n\n## Running\nTo run the system on the raspberry pi entirely, make sure self.server_mode is set to False in app.py, then use the following command.\n\n```bash\nmake run\n```\n\nFor more performance, you can run just the relay components on your Pi, and the inference/processing on a more powerful machine. To do this, run the following on the pi:\n\n```bash\nmake server\n```\n\nThen set self.server_mode = True in app.py, and set the environment variable ALARM_ENDPOINT to the alarm endpoint of your raspberry pi, which will look something like:\n\n```bash\nexport ALARM_ENDPOINT=\"http://your_pis_ip_address:8000/alarm/\"\n```\n\nThen run the following on the more powerful system\n\n```bash\nmake run\n```\n\n## Notes\nThe training dir doesn't actually contain code for training the model; I used GCP's vision AutoML. That dir has code for gathering images from an RTSP cam, and putting them in the format GCP needs.\n\nMy trained model is included as a .tflite file, though it probably won't work with your front door, you're best to train your own. Good luck!\n" }, { "path": "scripts/install.sh", "content": "#!/bin/bash\n# install.sh\n\ncd ../\n\necho \"deb https://packages.cloud.google.com/apt coral-edgetpu-stable main\" | sudo tee /etc/apt/sources.list.d/coral-edgetpu.list\ncurl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -\nsudo apt-get update\nsudo apt-get install -y python3-tflite-runtime ffmpeg git \\\n libsm6 libxext6 python-pip python3-pip git \\\n libatlas-base-dev python3-h5py libgtk2.0-dev libgtk-3-0 \\\n libilmbase-dev libopenexr-dev libgstreamer1.0-dev \\\n espeak gnustep-gui-runtime libsm6 \\\n libhdf5-dev libc-ares-dev libeigen3-dev\n\nsudo apt-get install -y openmpi-bin libopenmpi-dev\nsudo apt-get install -y libatlas-base-dev\n\n\npython3 -m pip install virtualenv\npython3 -m virtualenv -p python3 env\n. env/bin/activate\n\npython3 -m pip install keras_applications==1.0.8 --no-deps\npython3 -m pip install keras_preprocessing==1.1.0 --no-deps\npython3 -m pip install h5py==2.9.0\npython3 -m pip install -U six wheel mock RPi.GPIO==0.7.0\n\nwget https://github.com/lhelontra/tensorflow-on-arm/releases/download/v2.4.0/tensorflow-2.4.0-cp37-none-linux_armv7l.whl\npython3 -m pip uninstall -y tensorflow\npython3 -m pip install tensorflow-2.4.0-cp37-none-linux_armv7l.whl\n\n\ncd src && pip install -r requirements.txt\n" }, { "path": "src/app.py", "content": "import time\nimport os\nfrom classifier import Classifier\nfrom rtsparty import Stream\nfrom recognizer import FaceRecognizer\nimport cv2\nimport logging\nimport requests\nimport sys\n\n\nclass PackageSentry():\n \"\"\"Parent class for Package Sentry System\"\"\"\n\n def __init__(self):\n self._set_logging()\n self._set_defaults()\n\n def _set_defaults(self):\n \"\"\"Set all defaults\"\"\"\n logging.info('Starting stream')\n self.server_mode = False # Enable seerber mode; (separate the relay and processing components)\n self.alarm_endpoint = os.environ.get('ALARM_ENDPOINT', 'http://10.0.0.1:5000/alarm/')\n self.stream = Stream(os.environ.get('STREAM_URL'), live=True)\n self.recognizer = FaceRecognizer()\n logging.info('Starting classifier')\n self.classifier = Classifier()\n logging.info('Loading relay controller')\n if not self.server_mode:\n from relay_controller import RelayController\n self.relay_controller = RelayController()\n self.package_last_seen = None\n self.min_confidence = 0.5\n self.theft_tolerance_seconds = 0.1\n self.system_armed = False\n self.alarm_is_active = False\n self.package_detection_debounce_count = 0\n self.package_detection_debounce_threshold = 30\n self.known_person_timeout_seconds = 30\n self.known_persons_counter = 0\n self.known_person_last_seen = time.time()\n self.known_person_present = False\n\n def _set_logging(self):\n \"\"\"Set the log level for the system\"\"\"\n level = logging.INFO\n logging.basicConfig(stream=sys.stdout, level=level)\n\n def arm_system(self):\n \"\"\"Arm the system after a package has been left on the step\"\"\"\n logging.info('Arming System')\n self.system_armed = True\n\n def disarm_system(self):\n \"\"\"Arm the system after a package has been left on the step\"\"\"\n logging.info('Disarming System')\n self.system_armed = False\n\n def activate_alarm(self):\n \"\"\"A theft has occured, activate the alarm\"\"\"\n if self.alarm_is_active:\n return\n logging.info('Activating Alarm')\n self.alarm_is_active = True\n if not self.server_mode:\n self.relay_controller.activate_general_alarm()\n else:\n requests.get(self.alarm_endpoint)\n\n def _package_detected(self):\n \"\"\"A package has been identified in the frame\"\"\"\n logging.debug('Package detected')\n if self.package_detection_debounce_count > self.package_detection_debounce_threshold:\n if not self.system_armed:\n self.arm_system()\n else:\n self.package_detection_debounce_count = self.package_detection_debounce_count + 1\n self.package_last_seen = time.time()\n\n def _package_not_detected(self):\n \"\"\"The package is missing from the frame\"\"\"\n self.package_detection_debounce_count = 0\n if not self.system_armed:\n logging.debug('System not armed, ignoring')\n return\n if self.known_person_present:\n logging.debug('Known person removed package')\n self.disarm_system()\n return\n # This code gives the system a tolerance for frames that may be inaccurately\n # reporting the package as missing because of errors in the model or stream\n current_time = time.time()\n if current_time > self.package_last_seen + self.theft_tolerance_seconds:\n self.activate_alarm()\n\n def _check_for_known_persons(self, frame):\n \"\"\"Checks frame for known persons and responds accordingly\"\"\"\n # Only check once every n frrames\n check_every_n_frames = 60\n self.known_persons_counter = self.known_persons_counter + 1\n if self.known_persons_counter < check_every_n_frames:\n return\n self.known_persons_counter = 0\n logging.debug('Checking frame for faces')\n if self.recognizer.known_face_detected(frame):\n logging.info('Known person present')\n self.known_person_last_seen = time.time()\n self.known_person_present = True\n current_time = time.time()\n if current_time > self.known_person_timeout_seconds + self.known_person_last_seen:\n self.known_person_present = False\n\n def _check_frame(self):\n \"\"\"Check the frame for packages\"\"\"\n frame = self.stream.get_frame()\n if not self.stream.is_frame_empty(frame):\n self._check_for_known_persons(frame)\n if self.classifier.is_package_present(frame, self.min_confidence):\n self._package_detected()\n else:\n self._package_not_detected()\n\n def watch(self):\n \"\"\"Watch for theives and act accordingly\"\"\"\n logging.info('System watching')\n while True:\n self._check_frame()\n\ndef main():\n \"\"\"Run app\"\"\"\n try:\n ps = PackageSentry()\n ps.watch()\n except KeyboardInterrupt:\n print('Exiting')\n\n\nif __name__ == '__main__':\n main()\n" }, { "path": "src/classifier.py", "content": "import os\nimport logging\nimport numpy as np\nimport tensorflow as tf\nimport cv2\n\n\nclass Classifier():\n \"\"\"Identifies if there is or isn't a package on the porch\"\"\"\n\n def __init__(self):\n self._set_defaults()\n self._load_model()\n self._load_labels()\n\n def _set_defaults(self):\n parent_dir = os.path.dirname(os.path.realpath(__file__))\n self.model_file = os.path.join(parent_dir, 'models/model.tflite')\n self.label_file = os.path.join(parent_dir, 'models/dict.txt')\n self.input_mean = 127.5\n self.input_stdev = 127.5\n self.num_threads = None\n\n def _load_model(self):\n \"\"\"Load the model into memory\"\"\"\n self.model = tf.lite.Interpreter(\n model_path=self.model_file\n )\n self.model.allocate_tensors()\n self.input_details = self.model.get_input_details()\n self.output_details = self.model.get_output_details()\n self.is_floating_model = self.input_details[0]['dtype'] == np.float32\n self._set_default_height_width()\n\n def _load_labels(self):\n \"\"\"Load labels from the filesystem\"\"\"\n with open(self.label_file, 'r') as f:\n self.labels = [line.strip() for line in f.readlines()]\n\n def _set_default_height_width(self):\n \"\"\"Sets the default expected height and width\"\"\"\n self.default_height = self.input_details[0]['shape'][1]\n self.default_width = self.input_details[0]['shape'][2]\n\n def _normalize_input(self, frame):\n \"\"\"Normalizies the input frame\"\"\"\n frame = cv2.resize(frame, (self.default_width, self.default_height))\n return frame\n\n def _build_input_data(self, frame):\n \"\"\"Build the input data array\"\"\"\n return np.expand_dims(frame, axis=0)\n\n def classify_frame(self, frame):\n \"\"\"Classifies a frame\"\"\"\n logging.debug('Classifying image')\n input_data = self._build_input_data(self._normalize_input(frame))\n self.model.set_tensor(self.input_details[0]['index'], input_data)\n self.model.invoke()\n # dont judge this code plz\n results = np.squeeze(self.model.get_tensor(self.output_details[0]['index']))\n top_k = results.argsort()\n top_confidence = 0.0\n top_label = ''\n for i in top_k:\n confidence = float(results[i] / 255.0)\n label = self.labels[i]\n if confidence > top_confidence:\n top_confidence = confidence\n top_label = label\n logging.debug('Classification complete')\n return round(top_confidence, 3), top_label\n\n def is_package_present(self, frame, min_confidence=0.5):\n \"\"\"Determines if the package is currently present\"\"\"\n confidence, label = self.classify_frame(frame)\n logging.debug(str(confidence) + ' - ' + label)\n return label == 'package' and confidence > min_confidence\n" }, { "path": "src/faces/faces.txt", "content": "place .jpeg files here of faces the system should recognize and disarm for.\n" }, { "path": "src/models/dict.txt", "content": "package\nno_package\n" }, { "path": "src/models/readme.txt", "content": "My model was trained for my porch. It probably won't work for you - but I'm including it anyway. It's a tflite model.\n" }, { "path": "src/models/tflite_metadata.json", "content": "{\n \"batchSize\": 1,\n \"imageChannels\": 3,\n \"imageHeight\": 224,\n \"imageWidth\": 224,\n \"inferenceType\": \"QUANTIZED_UINT8\",\n \"inputTensor\": \"image\",\n \"inputType\": \"QUANTIZED_UINT8\",\n \"outputTensor\": \"scores\",\n \"supportedTfVersions\": [\n \"1.10\",\n \"1.11\",\n \"1.12\",\n \"1.13\"\n ]\n}" }, { "path": "src/recognizer.py", "content": "import os\nimport logging\nimport face_recognition\n\n\nclass FaceRecognizer():\n \"\"\"Face recognition module for package theft detection system\"\"\"\n\n def __init__(self):\n self._load_known_face()\n\n def _load_known_face(self):\n \"\"\"Loads known faces from the filesystem\"\"\"\n faces_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'faces')\n faces = [os.path.join(faces_dir, f) for f in os.listdir(faces_dir) if f.endswith('.jpeg')]\n known_images = [face_recognition.load_image_file(i) for i in faces]\n self.known_faces = []\n for image in known_images:\n encoding = face_recognition.face_encodings(image)\n if len(encoding) > 0:\n logging.debug('Adding known face')\n self.known_faces.append(encoding[0])\n\n def known_face_detected(self, frame):\n \"\"\"Retuns bool if a known face is detected\"\"\"\n faces_detected = face_recognition.face_encodings(frame)\n if len(faces_detected) > 0:\n unknown = face_recognition.face_encodings(frame)[0]\n results = face_recognition.compare_faces(self.known_faces, unknown)\n if True in results:\n logging.info('Known face detected')\n return True\n logging.info('Unknown face detected')\n return False\n" }, { "path": "src/relay_controller.py", "content": "import logging\nimport time\nimport threading\nimport RPi.GPIO as GPIO\n\n\nclass RelayController():\n \"\"\"Class for controlling the relay\"\"\"\n\n def __init__(self):\n self._set_defaults()\n self._setup_gpio()\n\n def __del__(self):\n GPIO.cleanup()\n\n def _set_defaults(self):\n \"\"\"Set defaults for the application\"\"\"\n self.silent = False\n self.sprinkler_running_timeout_seconds = 10\n self.air_solenoid_running_timeout_seconds = 2\n self.misc_running_timeout_seconds = 5\n self.sprinkler_pin_bcm = 27\n self.misc_pin_bcm = 17\n self.air_solenoid_pin_bcm = 22\n\n def _setup_gpio(self):\n \"\"\"Set up the GPIO defaults\"\"\"\n GPIO.setmode(GPIO.BCM)\n GPIO.setup(self.sprinkler_pin_bcm, GPIO.OUT)\n GPIO.setup(self.misc_pin_bcm, GPIO.OUT)\n GPIO.setup(self.air_solenoid_pin_bcm, GPIO.OUT)\n GPIO.output(self.sprinkler_pin_bcm, 1)\n GPIO.output(self.misc_pin_bcm, 1)\n GPIO.output(self.sprinkler_pin_bcm, 1)\n\n def _cycle_gpios(self):\n \"\"\"Cycle all active channels in the relay opn and off\"\"\"\n logging.info('Cycling Relay')\n time.sleep(1)\n channels = [\n self.sprinkler_pin_bcm,\n self.misc_pin_bcm,\n self.air_solenoid_pin_bcm,\n ]\n for c in channels:\n self._cycle_pin(c, 1)\n\n def _cycle_pin(self, pin, timeout):\n \"\"\"Cycle a relay channel on and off\"\"\"\n GPIO.output(pin, 0)\n time.sleep(timeout)\n GPIO.output(pin, 1)\n\n def activate_sprinkler(self):\n \"\"\"Cycles the sprinkler on and off\"\"\"\n time.sleep(1)\n logging.info('Activating Sprinkler')\n self._cycle_pin(self.sprinkler_pin_bcm, self.sprinkler_running_timeout_seconds)\n logging.info('Deactivating Sprinkler')\n\n def activate_misc_items(self):\n \"\"\"Activates all misc 12v items, the siren, lights, etc.\"\"\"\n logging.info('Activating Misc Items')\n self._cycle_pin(self.misc_pin_bcm, self.misc_running_timeout_seconds)\n logging.info('Deactivating Misc Items')\n\n def activate_solenoid(self):\n \"\"\"Activates the air solenoid.\"\"\"\n time.sleep(2)\n logging.info('Activating air solenoid')\n self._cycle_pin(self.air_solenoid_pin_bcm, self.air_solenoid_running_timeout_seconds)\n logging.info('Deactivating air solenoid')\n\n def activate_general_alarm(self):\n \"\"\"Activates all aspects of the alarm using background threads\"\"\"\n sprinkler_thread = threading.Thread(name='sprinkler_thread', target=self.activate_sprinkler)\n sprinkler_thread.setDaemon(True)\n if not self.silent:\n solenoid_thread = threading.Thread(name='solenoid_thread', target=self.activate_solenoid)\n misc_thread = threading.Thread(name='misc_thread', target=self.activate_misc_items)\n solenoid_thread.setDaemon(True)\n misc_thread.setDaemon(True)\n sprinkler_thread.start()\n if not self.silent:\n solenoid_thread.start()\n misc_thread.start()\n" }, { "path": "src/requirements.txt", "content": "numpy==1.21.0\ntensorflow\nrtsparty\npillow\nface-recognition==1.3.0\nflask\nrequests" }, { "path": "src/server.py", "content": "\"\"\"Components for running the raspberry pi in server mode\"\"\"\n\nimport os\nfrom flask import Flask, jsonify\nfrom relay_controller import RelayController\n\n\napp = Flask(__name__)\nrc = RelayController()\n\n\n@app.route('/')\ndef index():\n \"\"\"API index route\"\"\"\n return jsonify({'status': 'ok'})\n\n\n@app.route('/alarm/')\ndef alarm():\n \"\"\"Turn on the relay\"\"\"\n rc.activate_general_alarm()\n return jsonify({'status': 'alarm activated'})\n\n\nif __name__ == '__main__':\n app.run(debug=False, host='0.0.0.0', port=int(os.environ.get('PORT', '8000')))\n" }, { "path": "training/Makefile", "content": "RTSP_URL=rtsp://username:password@10.0.0.1/live\nSLEEP_SECONDS=10\nGCS_BASE=gs://your-bucket-here\n\n\n.PHONY: no-package-images\nno-package-images:\n\t@echo \"Taking photos with camera\"\n\t@echo \"There should be no packages at your front door\"\n\t@source env/bin/activate \\\n\t\t&& export PACKAGE_PRESENT=false \\\n\t\t&& export RTSP_URL=$(RTSP_URL) \\\n\t\t&& export SLEEP_SECONDS=$(SLEEP_SECONDS) \\\n\t\t&& python get_images_from_camera.py\n\n.PHONY: package-images\nnpackage-images:\n\t@echo \"Taking photos with camera\"\n\t@echo \"There should be some sort of package at your front door\"\n\t@source env/bin/activate \\\n\t\t&& export PACKAGE_PRESENT=false \\\n\t\t&& export RTSP_URL=$(RTSP_URL) \\\n\t\t&& export SLEEP_SECONDS=$(SLEEP_SECONDS) \\\n\t\t&& python get_images_from_camera.py\n\n.PHONY: generate-csv\ngenerate-csv:\n\t@echo \"Generating CSV\"\n\t@source env/bin/activate \\\n\t&& export GCS_BASE=$(GCS_BASE) \\\n\t&& python create_automl_csv.py\n" }, { "path": "training/create_automl_csv.py", "content": "import os\nimport pandas as pd\n\n\nreviewed_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data')\ngcs_base = os.environ.get('GCS_BASE')\n\n\ndef get_all_images(dir):\n return [x for x in os.listdir(dir) if x.endswith('.jpg')]\n\npackage_images = [os.path.join(gcs_base, 'data', 'package', x) for x in get_all_images(os.path.join(reviewed_dir, 'package'))]\nnopackage_images = [os.path.join(gcs_base, 'data', 'no_package', x) for x in get_all_images(os.path.join(reviewed_dir, 'no_package'))]\n\n\npackage_list = [[x, 'package'] for x in package_images]\nnopackage_list = [[x, 'no_package'] for x in nopackage_images]\n\ndf = pd.DataFrame(package_list + nopackage_list)\ndf.to_csv('training_data.csv', index=False, header=None)\n" }, { "path": "training/get_images_from_camera.py", "content": "\"\"\"Script for gathering training images from camera\"\"\"\nimport time\nimport pathlib\nimport os\nfrom rtsparty import Stream\nimport cv2\n\n\nprint('Establishing stream')\nstream = Stream(os.environ['RTSP_URL'], live=True)\n\n\ndef get_data_dir():\n \"\"\"Returns the data directory\"\"\"\n return os.path.dirname(os.path.realpath(__file__)), 'data'\n\n\ndef create_dirs():\n \"\"\"Creates directories if not exists\"\"\"\n pathlib.Path(os.path.join(get_data_dir(), 'package')).mkdir(parents=True, exist_ok=True)\n pathlib.Path(os.path.join(get_data_dir(), 'no_package')).mkdir(parents=True, exist_ok=True)\n\n\ndef get_file_name():\n \"\"\"Returns the appropriate file name for the image\"\"\"\n prefix = 'no_package'\n if bool(os.environ.get('PACKAGE_PRESENT', False)):\n prefix = 'package'\n data_dir = os.path.join(get_data_dir(), 'data', prefix)\n file_name = prefix + '__' + str(int(time.time())) + '.jpg'\n return os.path.join(data_dir, file_name)\n\n\ndef save_image_to_file():\n \"\"\"saves the image to the local filesystem\"\"\"\n frame = stream.get_frame()\n file_name = get_file_name()\n if stream.is_frame_empty(frame):\n return False\n cv2.imwrite(file_name, frame)\n return file_name\n\n\nif __name__ == '__main__':\n create_dirs()\n try:\n while True:\n saved = save_image_to_file()\n if saved:\n print(saved)\n time.sleep(5)\n else:\n print('Error, waiting 5 seconds')\n time.sleep(int(os.environ.get('SLEEP_SECONDS', '5')))\n except KeyboardInterrupt:\n print('Stopping')\n" }, { "path": "training/requirements.txt", "content": "rtsparty\npandas\n" } ]