Repository: rydercalmdown/package_theft_preventor
Branch: main
Commit: 280db84c3c5f
Files: 19
Total size: 19.5 KB

Directory structure:
gitextract_kv59wc3l/

├── .gitignore
├── Makefile
├── README.md
├── scripts/
│   └── install.sh
├── src/
│   ├── app.py
│   ├── classifier.py
│   ├── faces/
│   │   └── faces.txt
│   ├── models/
│   │   ├── dict.txt
│   │   ├── model.tflite
│   │   ├── readme.txt
│   │   └── tflite_metadata.json
│   ├── recognizer.py
│   ├── relay_controller.py
│   ├── requirements.txt
│   └── server.py
└── training/
    ├── Makefile
    ├── create_automl_csv.py
    ├── get_images_from_camera.py
    └── requirements.txt

================================================
FILE CONTENTS
================================================

================================================
FILE: .gitignore
================================================
**/*.jpeg
**/*.jpg
**/*.png
**/.DS_Store
**/env
**/*.pyc
training/data
training/data_reviewed


================================================
FILE: Makefile
================================================
PI_IP_ADDRESS=10.0.0.1
PI_USERNAME=pi
STREAM_URL=rtsp://username:password@camera_host/endpoint

.PHONY: run
run:
	@. env/bin/activate && cd src && export STREAM_URL=$(STREAM_URL) && python app.py

.PHONY: install
install:
	@cd scripts && bash install.sh

.PHONY: copy
copy:
	@rsync -a $(shell pwd) --exclude env --exclude training $(PI_USERNAME)@$(PI_IP_ADDRESS):/home/$(PI_USERNAME)

.PHONY: shell
shell:
	@ssh $(PI_USERNAME)@$(PI_IP_ADDRESS)

.PHONY: server
server:
	@echo "Running in server mode";
	@. env/bin/activate && cd src && python server.py


================================================
FILE: README.md
================================================
# Package Theft Prevention Device
An AI-powered device to stop people from stealing my packages.

## Installation
To install on a raspberry pi, clone the repository and run:
```bash
make install
```

## Running
To run the system on the raspberry pi entirely, make sure self.server_mode is set to False in app.py, then use the following command.

```bash
make run
```

For more performance, you can run just the relay components on your Pi, and the inference/processing on a more powerful machine. To do this, run the following on the pi:

```bash
make server
```

Then set self.server_mode = True in app.py, and set the environment variable ALARM_ENDPOINT to the alarm endpoint of your raspberry pi, which will look something like:

```bash
export ALARM_ENDPOINT="http://your_pis_ip_address:8000/alarm/"
```

Then run the following on the more powerful system

```bash
make run
```

## Notes
The training dir doesn't actually contain code for training the model; I used GCP's vision AutoML. That dir has code for gathering images from an RTSP cam, and putting them in the format GCP needs.

My trained model is included as a .tflite file, though it probably won't work with your front door, you're best to train your own. Good luck!


================================================
FILE: scripts/install.sh
================================================
#!/bin/bash
# install.sh

cd ../

echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | sudo tee /etc/apt/sources.list.d/coral-edgetpu.list
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
sudo apt-get update
sudo apt-get install -y python3-tflite-runtime ffmpeg git \
    libsm6 libxext6 python-pip python3-pip git \
    libatlas-base-dev python3-h5py libgtk2.0-dev libgtk-3-0 \
    libilmbase-dev libopenexr-dev libgstreamer1.0-dev \
    espeak gnustep-gui-runtime libsm6 \
    libhdf5-dev libc-ares-dev libeigen3-dev

sudo apt-get install -y openmpi-bin libopenmpi-dev
sudo apt-get install -y libatlas-base-dev


python3 -m pip install virtualenv
python3 -m virtualenv -p python3 env
. env/bin/activate

python3 -m pip install keras_applications==1.0.8 --no-deps
python3 -m pip install keras_preprocessing==1.1.0 --no-deps
python3 -m pip install h5py==2.9.0
python3 -m pip install -U six wheel mock RPi.GPIO==0.7.0

wget https://github.com/lhelontra/tensorflow-on-arm/releases/download/v2.4.0/tensorflow-2.4.0-cp37-none-linux_armv7l.whl
python3 -m pip uninstall -y tensorflow
python3 -m pip install tensorflow-2.4.0-cp37-none-linux_armv7l.whl


cd src && pip install -r requirements.txt


================================================
FILE: src/app.py
================================================
import time
import os
from classifier import Classifier
from rtsparty import Stream
from recognizer import FaceRecognizer
import cv2
import logging
import requests
import sys


class PackageSentry():
    """Parent class for Package Sentry System"""

    def __init__(self):
        self._set_logging()
        self._set_defaults()

    def _set_defaults(self):
        """Set all defaults"""
        logging.info('Starting stream')
        self.server_mode = False  # Enable seerber mode; (separate the relay and processing components)
        self.alarm_endpoint = os.environ.get('ALARM_ENDPOINT', 'http://10.0.0.1:5000/alarm/')
        self.stream = Stream(os.environ.get('STREAM_URL'), live=True)
        self.recognizer = FaceRecognizer()
        logging.info('Starting classifier')
        self.classifier = Classifier()
        logging.info('Loading relay controller')
        if not self.server_mode:
            from relay_controller import RelayController
            self.relay_controller = RelayController()
        self.package_last_seen = None
        self.min_confidence = 0.5
        self.theft_tolerance_seconds = 0.1
        self.system_armed = False
        self.alarm_is_active = False
        self.package_detection_debounce_count = 0
        self.package_detection_debounce_threshold = 30
        self.known_person_timeout_seconds = 30
        self.known_persons_counter = 0
        self.known_person_last_seen = time.time()
        self.known_person_present = False

    def _set_logging(self):
        """Set the log level for the system"""
        level = logging.INFO
        logging.basicConfig(stream=sys.stdout, level=level)

    def arm_system(self):
        """Arm the system after a package has been left on the step"""
        logging.info('Arming System')
        self.system_armed = True

    def disarm_system(self):
        """Arm the system after a package has been left on the step"""
        logging.info('Disarming System')
        self.system_armed = False

    def activate_alarm(self):
        """A theft has occured, activate the alarm"""
        if self.alarm_is_active:
            return
        logging.info('Activating Alarm')
        self.alarm_is_active = True
        if not self.server_mode:
            self.relay_controller.activate_general_alarm()
        else:
            requests.get(self.alarm_endpoint)

    def _package_detected(self):
        """A package has been identified in the frame"""
        logging.debug('Package detected')
        if self.package_detection_debounce_count > self.package_detection_debounce_threshold:
            if not self.system_armed:
                self.arm_system()
        else:
            self.package_detection_debounce_count = self.package_detection_debounce_count + 1
        self.package_last_seen = time.time()

    def _package_not_detected(self):
        """The package is missing from the frame"""
        self.package_detection_debounce_count = 0
        if not self.system_armed:
            logging.debug('System not armed, ignoring')
            return
        if self.known_person_present:
            logging.debug('Known person removed package')
            self.disarm_system()
            return
        # This code gives the system a tolerance for frames that may be inaccurately
        # reporting the package as missing because of errors in the model or stream
        current_time = time.time()
        if current_time > self.package_last_seen + self.theft_tolerance_seconds:
            self.activate_alarm()

    def _check_for_known_persons(self, frame):
        """Checks frame for known persons and responds accordingly"""
        # Only check once every n frrames
        check_every_n_frames = 60
        self.known_persons_counter = self.known_persons_counter + 1
        if self.known_persons_counter < check_every_n_frames:
            return
        self.known_persons_counter = 0
        logging.debug('Checking frame for faces')
        if self.recognizer.known_face_detected(frame):
            logging.info('Known person present')
            self.known_person_last_seen = time.time()
            self.known_person_present = True
        current_time = time.time()
        if current_time > self.known_person_timeout_seconds + self.known_person_last_seen:
            self.known_person_present = False

    def _check_frame(self):
        """Check the frame for packages"""
        frame = self.stream.get_frame()
        if not self.stream.is_frame_empty(frame):
            self._check_for_known_persons(frame)
            if self.classifier.is_package_present(frame, self.min_confidence):
                self._package_detected()
            else:
                self._package_not_detected()

    def watch(self):
        """Watch for theives and act accordingly"""
        logging.info('System watching')
        while True:
            self._check_frame()

def main():
    """Run app"""
    try:
        ps = PackageSentry()
        ps.watch()
    except KeyboardInterrupt:
        print('Exiting')


if __name__ == '__main__':
    main()


================================================
FILE: src/classifier.py
================================================
import os
import logging
import numpy as np
import tensorflow as tf
import cv2


class Classifier():
    """Identifies if there is or isn't a package on the porch"""

    def __init__(self):
        self._set_defaults()
        self._load_model()
        self._load_labels()

    def _set_defaults(self):
        parent_dir = os.path.dirname(os.path.realpath(__file__))
        self.model_file = os.path.join(parent_dir, 'models/model.tflite')
        self.label_file = os.path.join(parent_dir, 'models/dict.txt')
        self.input_mean = 127.5
        self.input_stdev = 127.5
        self.num_threads = None

    def _load_model(self):
        """Load the model into memory"""
        self.model = tf.lite.Interpreter(
            model_path=self.model_file
            )
        self.model.allocate_tensors()
        self.input_details = self.model.get_input_details()
        self.output_details = self.model.get_output_details()
        self.is_floating_model = self.input_details[0]['dtype'] == np.float32
        self._set_default_height_width()

    def _load_labels(self):
        """Load labels from the filesystem"""
        with open(self.label_file, 'r') as f:
            self.labels = [line.strip() for line in f.readlines()]

    def _set_default_height_width(self):
        """Sets the default expected height and width"""
        self.default_height = self.input_details[0]['shape'][1]
        self.default_width = self.input_details[0]['shape'][2]

    def _normalize_input(self, frame):
        """Normalizies the input frame"""
        frame = cv2.resize(frame, (self.default_width, self.default_height))
        return frame

    def _build_input_data(self, frame):
        """Build the input data array"""
        return np.expand_dims(frame, axis=0)

    def classify_frame(self, frame):
        """Classifies a frame"""
        logging.debug('Classifying image')
        input_data = self._build_input_data(self._normalize_input(frame))
        self.model.set_tensor(self.input_details[0]['index'], input_data)
        self.model.invoke()
        # dont judge this code plz
        results = np.squeeze(self.model.get_tensor(self.output_details[0]['index']))
        top_k = results.argsort()
        top_confidence = 0.0
        top_label = ''
        for i in top_k:
            confidence = float(results[i] / 255.0)
            label = self.labels[i]
            if confidence > top_confidence:
                top_confidence = confidence
                top_label = label
        logging.debug('Classification complete')
        return round(top_confidence, 3), top_label

    def is_package_present(self, frame, min_confidence=0.5):
        """Determines if the package is currently present"""
        confidence, label = self.classify_frame(frame)
        logging.debug(str(confidence) + ' - ' + label)
        return label == 'package' and confidence > min_confidence


================================================
FILE: src/faces/faces.txt
================================================
place .jpeg files here of faces the system should recognize and disarm for.


================================================
FILE: src/models/dict.txt
================================================
package
no_package


================================================
FILE: src/models/readme.txt
================================================
My model was trained for my porch. It probably won't work for you - but I'm including it anyway. It's a tflite model.


================================================
FILE: src/models/tflite_metadata.json
================================================
{
    "batchSize": 1,
    "imageChannels": 3,
    "imageHeight": 224,
    "imageWidth": 224,
    "inferenceType": "QUANTIZED_UINT8",
    "inputTensor": "image",
    "inputType": "QUANTIZED_UINT8",
    "outputTensor": "scores",
    "supportedTfVersions": [
        "1.10",
        "1.11",
        "1.12",
        "1.13"
    ]
}

================================================
FILE: src/recognizer.py
================================================
import os
import logging
import face_recognition


class FaceRecognizer():
    """Face recognition module for package theft detection system"""

    def __init__(self):
        self._load_known_face()

    def _load_known_face(self):
        """Loads known faces from the filesystem"""
        faces_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'faces')
        faces = [os.path.join(faces_dir, f) for f in os.listdir(faces_dir) if f.endswith('.jpeg')]
        known_images = [face_recognition.load_image_file(i) for i in faces]
        self.known_faces = []
        for image in known_images:
            encoding = face_recognition.face_encodings(image)
            if len(encoding) > 0:
                logging.debug('Adding known face')
                self.known_faces.append(encoding[0])

    def known_face_detected(self, frame):
        """Retuns bool if a known face is detected"""
        faces_detected = face_recognition.face_encodings(frame)
        if len(faces_detected) > 0:
            unknown = face_recognition.face_encodings(frame)[0]
            results = face_recognition.compare_faces(self.known_faces, unknown)
            if True in results:
                logging.info('Known face detected')
                return True
            logging.info('Unknown face detected')
            return False


================================================
FILE: src/relay_controller.py
================================================
import logging
import time
import threading
import RPi.GPIO as GPIO


class RelayController():
    """Class for controlling the relay"""

    def __init__(self):
        self._set_defaults()
        self._setup_gpio()

    def __del__(self):
        GPIO.cleanup()

    def _set_defaults(self):
        """Set defaults for the application"""
        self.silent = False
        self.sprinkler_running_timeout_seconds = 10
        self.air_solenoid_running_timeout_seconds = 2
        self.misc_running_timeout_seconds = 5
        self.sprinkler_pin_bcm = 27
        self.misc_pin_bcm = 17
        self.air_solenoid_pin_bcm = 22

    def _setup_gpio(self):
        """Set up the GPIO defaults"""
        GPIO.setmode(GPIO.BCM)
        GPIO.setup(self.sprinkler_pin_bcm, GPIO.OUT)
        GPIO.setup(self.misc_pin_bcm, GPIO.OUT)
        GPIO.setup(self.air_solenoid_pin_bcm, GPIO.OUT)
        GPIO.output(self.sprinkler_pin_bcm, 1)
        GPIO.output(self.misc_pin_bcm, 1)
        GPIO.output(self.sprinkler_pin_bcm, 1)

    def _cycle_gpios(self):
        """Cycle all active channels in the relay opn and off"""
        logging.info('Cycling Relay')
        time.sleep(1)
        channels = [
            self.sprinkler_pin_bcm,
            self.misc_pin_bcm,
            self.air_solenoid_pin_bcm,
        ]
        for c in channels:
            self._cycle_pin(c, 1)

    def _cycle_pin(self, pin, timeout):
        """Cycle a relay channel on and off"""
        GPIO.output(pin, 0)
        time.sleep(timeout)
        GPIO.output(pin, 1)

    def activate_sprinkler(self):
        """Cycles the sprinkler on and off"""
        time.sleep(1)
        logging.info('Activating Sprinkler')
        self._cycle_pin(self.sprinkler_pin_bcm, self.sprinkler_running_timeout_seconds)
        logging.info('Deactivating Sprinkler')

    def activate_misc_items(self):
        """Activates all misc 12v items, the siren, lights, etc."""
        logging.info('Activating Misc Items')
        self._cycle_pin(self.misc_pin_bcm, self.misc_running_timeout_seconds)
        logging.info('Deactivating Misc Items')

    def activate_solenoid(self):
        """Activates the air solenoid."""
        time.sleep(2)
        logging.info('Activating air solenoid')
        self._cycle_pin(self.air_solenoid_pin_bcm, self.air_solenoid_running_timeout_seconds)
        logging.info('Deactivating air solenoid')

    def activate_general_alarm(self):
        """Activates all aspects of the alarm using background threads"""
        sprinkler_thread = threading.Thread(name='sprinkler_thread', target=self.activate_sprinkler)
        sprinkler_thread.setDaemon(True)
        if not self.silent:
            solenoid_thread = threading.Thread(name='solenoid_thread', target=self.activate_solenoid)
            misc_thread = threading.Thread(name='misc_thread', target=self.activate_misc_items)
            solenoid_thread.setDaemon(True)
            misc_thread.setDaemon(True)
        sprinkler_thread.start()
        if not self.silent:
            solenoid_thread.start()
            misc_thread.start()


================================================
FILE: src/requirements.txt
================================================
numpy==1.21.0
tensorflow
rtsparty
pillow
face-recognition==1.3.0
flask
requests

================================================
FILE: src/server.py
================================================
"""Components for running the raspberry pi in server mode"""

import os
from flask import Flask, jsonify
from relay_controller import RelayController


app = Flask(__name__)
rc = RelayController()


@app.route('/')
def index():
    """API index route"""
    return jsonify({'status': 'ok'})


@app.route('/alarm/')
def alarm():
    """Turn on the relay"""
    rc.activate_general_alarm()
    return jsonify({'status': 'alarm activated'})


if __name__ == '__main__':
    app.run(debug=False, host='0.0.0.0', port=int(os.environ.get('PORT', '8000')))


================================================
FILE: training/Makefile
================================================
RTSP_URL=rtsp://username:password@10.0.0.1/live
SLEEP_SECONDS=10
GCS_BASE=gs://your-bucket-here


.PHONY: no-package-images
no-package-images:
	@echo "Taking photos with camera"
	@echo "There should be no packages at your front door"
	@source env/bin/activate \
		&& export PACKAGE_PRESENT=false \
		&& export RTSP_URL=$(RTSP_URL) \
		&& export SLEEP_SECONDS=$(SLEEP_SECONDS) \
		&& python get_images_from_camera.py

.PHONY: package-images
npackage-images:
	@echo "Taking photos with camera"
	@echo "There should be some sort of package at your front door"
	@source env/bin/activate \
		&& export PACKAGE_PRESENT=false \
		&& export RTSP_URL=$(RTSP_URL) \
		&& export SLEEP_SECONDS=$(SLEEP_SECONDS) \
		&& python get_images_from_camera.py

.PHONY: generate-csv
generate-csv:
	@echo "Generating CSV"
	@source env/bin/activate \
	&& export GCS_BASE=$(GCS_BASE) \
	&& python create_automl_csv.py


================================================
FILE: training/create_automl_csv.py
================================================
import os
import pandas as pd


reviewed_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data')
gcs_base = os.environ.get('GCS_BASE')


def get_all_images(dir):
    return [x for x in os.listdir(dir) if x.endswith('.jpg')]

package_images = [os.path.join(gcs_base, 'data', 'package', x) for x in get_all_images(os.path.join(reviewed_dir, 'package'))]
nopackage_images = [os.path.join(gcs_base, 'data', 'no_package', x) for x in get_all_images(os.path.join(reviewed_dir, 'no_package'))]


package_list = [[x, 'package'] for x in package_images]
nopackage_list = [[x, 'no_package'] for x in nopackage_images]

df = pd.DataFrame(package_list + nopackage_list)
df.to_csv('training_data.csv', index=False, header=None)


================================================
FILE: training/get_images_from_camera.py
================================================
"""Script for gathering training images from camera"""
import time
import pathlib
import os
from rtsparty import Stream
import cv2


print('Establishing stream')
stream = Stream(os.environ['RTSP_URL'], live=True)


def get_data_dir():
    """Returns the data directory"""
    return os.path.dirname(os.path.realpath(__file__)), 'data'


def create_dirs():
    """Creates directories if not exists"""
    pathlib.Path(os.path.join(get_data_dir(), 'package')).mkdir(parents=True, exist_ok=True)
    pathlib.Path(os.path.join(get_data_dir(), 'no_package')).mkdir(parents=True, exist_ok=True)


def get_file_name():
    """Returns the appropriate file name for the image"""
    prefix = 'no_package'
    if bool(os.environ.get('PACKAGE_PRESENT', False)):
        prefix = 'package'
    data_dir = os.path.join(get_data_dir(), 'data', prefix)
    file_name = prefix + '__' + str(int(time.time())) + '.jpg'
    return os.path.join(data_dir, file_name)


def save_image_to_file():
    """saves the image to the local filesystem"""
    frame = stream.get_frame()
    file_name = get_file_name()
    if stream.is_frame_empty(frame):
        return False
    cv2.imwrite(file_name, frame)
    return file_name


if __name__ == '__main__':
    create_dirs()
    try:
        while True:
            saved = save_image_to_file()
            if saved:
                print(saved)
                time.sleep(5)
            else:
                print('Error, waiting 5 seconds')
                time.sleep(int(os.environ.get('SLEEP_SECONDS', '5')))
    except KeyboardInterrupt:
        print('Stopping')


================================================
FILE: training/requirements.txt
================================================
rtsparty
pandas