[
{
"path": "Colab_train_emotic.ipynb",
"content": "{\n \"nbformat\": 4,\n \"nbformat_minor\": 0,\n \"metadata\": {\n \"colab\": {\n \"name\": \"emotic.ipynb\",\n \"provenance\": [],\n \"collapsed_sections\": [],\n \"authorship_tag\": \"ABX9TyNTRS+z3BPWqTSv2PkmiNrg\",\n \"include_colab_link\": true\n },\n \"kernelspec\": {\n \"name\": \"python3\",\n \"display_name\": \"Python 3\"\n },\n \"accelerator\": \"GPU\",\n \"widgets\": {\n \"application/vnd.jupyter.widget-state+json\": {\n \"a8cada3fef3846b2bffe52edacbc190d\": {\n \"model_module\": \"@jupyter-widgets/controls\",\n \"model_name\": \"HBoxModel\",\n \"model_module_version\": \"1.5.0\",\n \"state\": {\n \"_view_name\": \"HBoxView\",\n \"_dom_classes\": [],\n \"_model_name\": \"HBoxModel\",\n \"_view_module\": \"@jupyter-widgets/controls\",\n \"_model_module_version\": \"1.5.0\",\n \"_view_count\": null,\n \"_view_module_version\": \"1.5.0\",\n \"box_style\": \"\",\n \"layout\": \"IPY_MODEL_ae34432e333e4671b3f7f934de91027b\",\n \"_model_module\": \"@jupyter-widgets/controls\",\n \"children\": [\n \"IPY_MODEL_674e1fd300d042cbaf9f0e53e7ac4ecd\",\n \"IPY_MODEL_88b121247db64a3490f8c1b16f68c696\",\n \"IPY_MODEL_57ffb85897da4061b318afddca2eed81\"\n ]\n }\n },\n \"ae34432e333e4671b3f7f934de91027b\": {\n \"model_module\": \"@jupyter-widgets/base\",\n \"model_name\": \"LayoutModel\",\n \"model_module_version\": \"1.2.0\",\n \"state\": {\n \"_view_name\": \"LayoutView\",\n \"grid_template_rows\": null,\n \"right\": null,\n \"justify_content\": null,\n \"_view_module\": \"@jupyter-widgets/base\",\n \"overflow\": null,\n \"_model_module_version\": \"1.2.0\",\n \"_view_count\": null,\n \"flex_flow\": null,\n \"width\": null,\n \"min_width\": null,\n \"border\": null,\n \"align_items\": null,\n \"bottom\": null,\n \"_model_module\": \"@jupyter-widgets/base\",\n \"top\": null,\n \"grid_column\": null,\n \"overflow_y\": null,\n \"overflow_x\": null,\n \"grid_auto_flow\": null,\n \"grid_area\": null,\n \"grid_template_columns\": null,\n \"flex\": null,\n \"_model_name\": \"LayoutModel\",\n \"justify_items\": null,\n \"grid_row\": null,\n \"max_height\": null,\n \"align_content\": null,\n \"visibility\": null,\n \"align_self\": null,\n \"height\": null,\n \"min_height\": null,\n \"padding\": null,\n \"grid_auto_rows\": null,\n \"grid_gap\": null,\n \"max_width\": null,\n \"order\": null,\n \"_view_module_version\": \"1.2.0\",\n \"grid_template_areas\": null,\n \"object_position\": null,\n \"object_fit\": null,\n \"grid_auto_columns\": null,\n \"margin\": null,\n \"display\": null,\n \"left\": null\n }\n },\n \"674e1fd300d042cbaf9f0e53e7ac4ecd\": {\n \"model_module\": \"@jupyter-widgets/controls\",\n \"model_name\": \"HTMLModel\",\n \"model_module_version\": \"1.5.0\",\n \"state\": {\n \"_view_name\": \"HTMLView\",\n \"style\": \"IPY_MODEL_9cb235dbbe234dbe805b4aa00f7d54ae\",\n \"_dom_classes\": [],\n \"description\": \"\",\n \"_model_name\": \"HTMLModel\",\n \"placeholder\": \"\",\n \"_view_module\": \"@jupyter-widgets/controls\",\n \"_model_module_version\": \"1.5.0\",\n \"value\": \"100%\",\n \"_view_count\": null,\n \"_view_module_version\": \"1.5.0\",\n \"description_tooltip\": null,\n \"_model_module\": \"@jupyter-widgets/controls\",\n \"layout\": \"IPY_MODEL_5e048b1fa84146c8bd2b63a19239cb9e\"\n }\n },\n \"88b121247db64a3490f8c1b16f68c696\": {\n \"model_module\": \"@jupyter-widgets/controls\",\n \"model_name\": \"FloatProgressModel\",\n \"model_module_version\": \"1.5.0\",\n \"state\": {\n \"_view_name\": \"ProgressView\",\n \"style\": \"IPY_MODEL_e6a413c0b59f466b9213c1904b1f57f8\",\n \"_dom_classes\": [],\n \"description\": \"\",\n \"_model_name\": \"FloatProgressModel\",\n \"bar_style\": \"success\",\n \"max\": 46830571,\n \"_view_module\": \"@jupyter-widgets/controls\",\n \"_model_module_version\": \"1.5.0\",\n \"value\": 46830571,\n \"_view_count\": null,\n \"_view_module_version\": \"1.5.0\",\n \"orientation\": \"horizontal\",\n \"min\": 0,\n \"description_tooltip\": null,\n \"_model_module\": \"@jupyter-widgets/controls\",\n \"layout\": \"IPY_MODEL_e0d0abfa1e9441f58722b064823c8119\"\n }\n },\n \"57ffb85897da4061b318afddca2eed81\": {\n \"model_module\": \"@jupyter-widgets/controls\",\n \"model_name\": \"HTMLModel\",\n \"model_module_version\": \"1.5.0\",\n \"state\": {\n \"_view_name\": \"HTMLView\",\n \"style\": \"IPY_MODEL_a1bbd4436c154378839f58483fa8c261\",\n \"_dom_classes\": [],\n \"description\": \"\",\n \"_model_name\": \"HTMLModel\",\n \"placeholder\": \"\",\n \"_view_module\": \"@jupyter-widgets/controls\",\n \"_model_module_version\": \"1.5.0\",\n \"value\": \" 44.7M/44.7M [00:00<00:00, 134MB/s]\",\n \"_view_count\": null,\n \"_view_module_version\": \"1.5.0\",\n \"description_tooltip\": null,\n \"_model_module\": \"@jupyter-widgets/controls\",\n \"layout\": \"IPY_MODEL_4f6e592ca3f34209af0ae78a635fc346\"\n }\n },\n \"9cb235dbbe234dbe805b4aa00f7d54ae\": {\n \"model_module\": \"@jupyter-widgets/controls\",\n \"model_name\": \"DescriptionStyleModel\",\n \"model_module_version\": \"1.5.0\",\n \"state\": {\n \"_view_name\": \"StyleView\",\n \"_model_name\": \"DescriptionStyleModel\",\n \"description_width\": \"\",\n \"_view_module\": \"@jupyter-widgets/base\",\n \"_model_module_version\": \"1.5.0\",\n \"_view_count\": null,\n \"_view_module_version\": \"1.2.0\",\n \"_model_module\": \"@jupyter-widgets/controls\"\n }\n },\n \"5e048b1fa84146c8bd2b63a19239cb9e\": {\n \"model_module\": \"@jupyter-widgets/base\",\n \"model_name\": \"LayoutModel\",\n \"model_module_version\": \"1.2.0\",\n \"state\": {\n \"_view_name\": \"LayoutView\",\n \"grid_template_rows\": null,\n \"right\": null,\n \"justify_content\": null,\n \"_view_module\": \"@jupyter-widgets/base\",\n \"overflow\": null,\n \"_model_module_version\": \"1.2.0\",\n \"_view_count\": null,\n \"flex_flow\": null,\n \"width\": null,\n \"min_width\": null,\n \"border\": null,\n \"align_items\": null,\n \"bottom\": null,\n \"_model_module\": \"@jupyter-widgets/base\",\n \"top\": null,\n \"grid_column\": null,\n \"overflow_y\": null,\n \"overflow_x\": null,\n \"grid_auto_flow\": null,\n \"grid_area\": null,\n \"grid_template_columns\": null,\n \"flex\": null,\n \"_model_name\": \"LayoutModel\",\n \"justify_items\": null,\n \"grid_row\": null,\n \"max_height\": null,\n \"align_content\": null,\n \"visibility\": null,\n \"align_self\": null,\n \"height\": null,\n \"min_height\": null,\n \"padding\": null,\n \"grid_auto_rows\": null,\n \"grid_gap\": null,\n \"max_width\": null,\n \"order\": null,\n \"_view_module_version\": \"1.2.0\",\n \"grid_template_areas\": null,\n \"object_position\": null,\n \"object_fit\": null,\n \"grid_auto_columns\": null,\n \"margin\": null,\n \"display\": null,\n \"left\": null\n }\n },\n \"e6a413c0b59f466b9213c1904b1f57f8\": {\n \"model_module\": \"@jupyter-widgets/controls\",\n \"model_name\": \"ProgressStyleModel\",\n \"model_module_version\": \"1.5.0\",\n \"state\": {\n \"_view_name\": \"StyleView\",\n \"_model_name\": \"ProgressStyleModel\",\n \"description_width\": \"\",\n \"_view_module\": \"@jupyter-widgets/base\",\n \"_model_module_version\": \"1.5.0\",\n \"_view_count\": null,\n \"_view_module_version\": \"1.2.0\",\n \"bar_color\": null,\n \"_model_module\": \"@jupyter-widgets/controls\"\n }\n },\n \"e0d0abfa1e9441f58722b064823c8119\": {\n \"model_module\": \"@jupyter-widgets/base\",\n \"model_name\": \"LayoutModel\",\n \"model_module_version\": \"1.2.0\",\n \"state\": {\n \"_view_name\": \"LayoutView\",\n \"grid_template_rows\": null,\n \"right\": null,\n \"justify_content\": null,\n \"_view_module\": \"@jupyter-widgets/base\",\n \"overflow\": null,\n \"_model_module_version\": \"1.2.0\",\n \"_view_count\": null,\n \"flex_flow\": null,\n \"width\": null,\n \"min_width\": null,\n \"border\": null,\n \"align_items\": null,\n \"bottom\": null,\n \"_model_module\": \"@jupyter-widgets/base\",\n \"top\": null,\n \"grid_column\": null,\n \"overflow_y\": null,\n \"overflow_x\": null,\n \"grid_auto_flow\": null,\n \"grid_area\": null,\n \"grid_template_columns\": null,\n \"flex\": null,\n \"_model_name\": \"LayoutModel\",\n \"justify_items\": null,\n \"grid_row\": null,\n \"max_height\": null,\n \"align_content\": null,\n \"visibility\": null,\n \"align_self\": null,\n \"height\": null,\n \"min_height\": null,\n \"padding\": null,\n \"grid_auto_rows\": null,\n \"grid_gap\": null,\n \"max_width\": null,\n \"order\": null,\n \"_view_module_version\": \"1.2.0\",\n \"grid_template_areas\": null,\n \"object_position\": null,\n \"object_fit\": null,\n \"grid_auto_columns\": null,\n \"margin\": null,\n \"display\": null,\n \"left\": null\n }\n },\n \"a1bbd4436c154378839f58483fa8c261\": {\n \"model_module\": \"@jupyter-widgets/controls\",\n \"model_name\": \"DescriptionStyleModel\",\n \"model_module_version\": \"1.5.0\",\n \"state\": {\n \"_view_name\": \"StyleView\",\n \"_model_name\": \"DescriptionStyleModel\",\n \"description_width\": \"\",\n \"_view_module\": \"@jupyter-widgets/base\",\n \"_model_module_version\": \"1.5.0\",\n \"_view_count\": null,\n \"_view_module_version\": \"1.2.0\",\n \"_model_module\": \"@jupyter-widgets/controls\"\n }\n },\n \"4f6e592ca3f34209af0ae78a635fc346\": {\n \"model_module\": \"@jupyter-widgets/base\",\n \"model_name\": \"LayoutModel\",\n \"model_module_version\": \"1.2.0\",\n \"state\": {\n \"_view_name\": \"LayoutView\",\n \"grid_template_rows\": null,\n \"right\": null,\n \"justify_content\": null,\n \"_view_module\": \"@jupyter-widgets/base\",\n \"overflow\": null,\n \"_model_module_version\": \"1.2.0\",\n \"_view_count\": null,\n \"flex_flow\": null,\n \"width\": null,\n \"min_width\": null,\n \"border\": null,\n \"align_items\": null,\n \"bottom\": null,\n \"_model_module\": \"@jupyter-widgets/base\",\n \"top\": null,\n \"grid_column\": null,\n \"overflow_y\": null,\n \"overflow_x\": null,\n \"grid_auto_flow\": null,\n \"grid_area\": null,\n \"grid_template_columns\": null,\n \"flex\": null,\n \"_model_name\": \"LayoutModel\",\n \"justify_items\": null,\n \"grid_row\": null,\n \"max_height\": null,\n \"align_content\": null,\n \"visibility\": null,\n \"align_self\": null,\n \"height\": null,\n \"min_height\": null,\n \"padding\": null,\n \"grid_auto_rows\": null,\n \"grid_gap\": null,\n \"max_width\": null,\n \"order\": null,\n \"_view_module_version\": \"1.2.0\",\n \"grid_template_areas\": null,\n \"object_position\": null,\n \"object_fit\": null,\n \"grid_auto_columns\": null,\n \"margin\": null,\n \"display\": null,\n \"left\": null\n }\n }\n }\n }\n },\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"view-in-github\",\n \"colab_type\": \"text\"\n },\n \"source\": [\n \"![\\\"Open](\\\"https://colab.research.google.com/assets/colab-badge.svg\\\")
\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"_5Xan2tnR89K\"\n },\n \"source\": [\n \" Emotions in context (Emotic)
\\n\",\n \" Using context information to recognize emotions in images\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"rbCWI0rkt8yp\"\n },\n \"source\": [\n \"Project context
\\n\",\n \"\\n\",\n \"Humans use their facial features or expressions to convey how they feel, such as a person may smile when happy and scowl when angry. Historically, computer vision research has focussed on analyzing and learning these facial features to recognize emotions. \\n\",\n \"However, these facial features are not universal and vary extensively across cultures and situations. \\n\",\n \"\\n\",\n \"\\n\",\n \"![](\\\"https://raw.githubusercontent.com/Tandon-A/emotic/master/assets/face.jpg\\\")
![](\\\"https://raw.githubusercontent.com/Tandon-A/emotic/master/assets/full_scene.jpg\\\")
\\n\",\n \" Fig 1: a) (Facial feature) The person looks angry or in pain b) (Whole scene) The person looks elated.\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"A scene context, as shown in the figure above, can provide additional information about the situations. This project explores the use of context in recognizing emotions in images. \\n\",\n \"\\n\",\n \"This project uses the EMOTIC dataset and follows the methodology as introduced in the paper 'Context based emotion recognition using EMOTIC dataset'.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"1YFaW8HlNWnE\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"7cc564d6-4503-4b5a-bac8-a4fe0bdbcb65\"\n },\n \"source\": [\n \"# Linking Google drive to use preprocessed data \\n\",\n \"from google.colab import drive\\n\",\n \"\\n\",\n \"# This will prompt for authorization.\\n\",\n \"drive.mount('/content/drive')\\n\",\n \"#/content/drive/My Drive//\"\n ],\n \"execution_count\": 1,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"Mounted at /content/drive\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"FhzX7KUihZqu\"\n },\n \"source\": [\n \"# I. Prepare places pretrained model\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"uYgeeri3wdCM\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"59be98ac-4cc9-403c-e116-bac36e368e8b\"\n },\n \"source\": [\n \"# Get Resnet18 model trained on places dataset. \\n\",\n \"!mkdir ./places\\n\",\n \"!wget http://places2.csail.mit.edu/models_places365/resnet18_places365.pth.tar -O ./places/resnet18_places365.pth.tar\"\n ],\n \"execution_count\": 2,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"--2021-08-17 17:32:18-- http://places2.csail.mit.edu/models_places365/resnet18_places365.pth.tar\\n\",\n \"Resolving places2.csail.mit.edu (places2.csail.mit.edu)... 128.30.195.26\\n\",\n \"Connecting to places2.csail.mit.edu (places2.csail.mit.edu)|128.30.195.26|:80... connected.\\n\",\n \"HTTP request sent, awaiting response... 200 OK\\n\",\n \"Length: 45506139 (43M) [application/x-tar]\\n\",\n \"Saving to: ‘./places/resnet18_places365.pth.tar’\\n\",\n \"\\n\",\n \"./places/resnet18_p 100%[===================>] 43.40M 24.3MB/s in 1.8s \\n\",\n \"\\n\",\n \"2021-08-17 17:32:20 (24.3 MB/s) - ‘./places/resnet18_places365.pth.tar’ saved [45506139/45506139]\\n\",\n \"\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"RhWL6Qi_w4qp\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"4803750e-9487-4589-ef86-d8244ed698ca\"\n },\n \"source\": [\n \"# Saving the model weights to use ahead in the notebook\\n\",\n \"import torch\\n\",\n \"from torch.autograd import Variable as V\\n\",\n \"import torchvision.models as models\\n\",\n \"from PIL import Image\\n\",\n \"from torchvision import transforms as trn\\n\",\n \"from torch.nn import functional as F\\n\",\n \"import os\\n\",\n \"\\n\",\n \"# the architecture to use\\n\",\n \"arch = 'resnet18'\\n\",\n \"model_weight = os.path.join('./places', 'resnet18_places365.pth.tar')\\n\",\n \"\\n\",\n \"# create the network architecture\\n\",\n \"model = models.__dict__[arch](num_classes=365)\\n\",\n \"\\n\",\n \"#model_weight = '%s_places365.pth.tar' % arch\\n\",\n \"\\n\",\n \"checkpoint = torch.load(model_weight, map_location=lambda storage, loc: storage) # model trained in GPU could be deployed in CPU machine like this!\\n\",\n \"state_dict = {str.replace(k,'module.',''): v for k,v in checkpoint['state_dict'].items()} # the data parallel layer will add 'module' before each layer name\\n\",\n \"model.load_state_dict(state_dict)\\n\",\n \"model.eval()\\n\",\n \"\\n\",\n \"model.cpu()\\n\",\n \"torch.save(model.state_dict(), './places/resnet18_state_dict.pth')\\n\",\n \"print ('completed cell')\"\n ],\n \"execution_count\": 3,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"completed cell\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"ykNjfrUuhpbq\"\n },\n \"source\": [\n \"# II. General imports\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"vi-O8QgwvOQY\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"6f5857a3-f3af-4dbb-dd7f-8539fab5b9e7\"\n },\n \"source\": [\n \"import matplotlib.pyplot as plt\\n\",\n \"import numpy as np\\n\",\n \"import os\\n\",\n \"from PIL import Image\\n\",\n \"import scipy.io\\n\",\n \"from sklearn.metrics import average_precision_score, precision_recall_curve\\n\",\n \"\\n\",\n \"import torch \\n\",\n \"import torch.nn as nn \\n\",\n \"import torch.nn.functional as F\\n\",\n \"import torch.optim as optim \\n\",\n \"from torch.utils.data import Dataset, DataLoader \\n\",\n \"from torchsummary import summary\\n\",\n \"from torchvision import transforms\\n\",\n \"import torchvision.models as models\\n\",\n \"from torch.optim.lr_scheduler import StepLR\\n\",\n \"\\n\",\n \"print ('completed cell')\"\n ],\n \"execution_count\": 4,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"completed cell\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"AD0pBBBYh2vW\"\n },\n \"source\": [\n \"# III. Emotic classes\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"ZfPKerg4TWkR\"\n },\n \"source\": [\n \"## Emotic Model \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"ZWt88EcJVu0c\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"cd2365da-0d45-4616-800c-e1fd6f565c29\"\n },\n \"source\": [\n \"class Emotic(nn.Module):\\n\",\n \" ''' Emotic Model'''\\n\",\n \" def __init__(self, num_context_features, num_body_features):\\n\",\n \" super(Emotic,self).__init__()\\n\",\n \" self.num_context_features = num_context_features\\n\",\n \" self.num_body_features = num_body_features\\n\",\n \" self.fc1 = nn.Linear((self.num_context_features + num_body_features), 256)\\n\",\n \" self.bn1 = nn.BatchNorm1d(256)\\n\",\n \" self.d1 = nn.Dropout(p=0.5)\\n\",\n \" self.fc_cat = nn.Linear(256, 26)\\n\",\n \" self.fc_cont = nn.Linear(256, 3)\\n\",\n \" self.relu = nn.ReLU()\\n\",\n \"\\n\",\n \" \\n\",\n \" def forward(self, x_context, x_body):\\n\",\n \" context_features = x_context.view(-1, self.num_context_features)\\n\",\n \" body_features = x_body.view(-1, self.num_body_features)\\n\",\n \" fuse_features = torch.cat((context_features, body_features), 1)\\n\",\n \" fuse_out = self.fc1(fuse_features)\\n\",\n \" fuse_out = self.bn1(fuse_out)\\n\",\n \" fuse_out = self.relu(fuse_out)\\n\",\n \" fuse_out = self.d1(fuse_out) \\n\",\n \" cat_out = self.fc_cat(fuse_out)\\n\",\n \" cont_out = self.fc_cont(fuse_out)\\n\",\n \" return cat_out, cont_out\\n\",\n \"\\n\",\n \"print ('completed cell')\"\n ],\n \"execution_count\": 5,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"completed cell\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"zdzZGj6AxLaC\"\n },\n \"source\": [\n \"## Emotic Dataset\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"eKG5dNMXxlnm\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"890ab105-8973-4be7-be1f-670a816d6b79\"\n },\n \"source\": [\n \"class Emotic_PreDataset(Dataset):\\n\",\n \" ''' Custom Emotic dataset class. Use preprocessed data stored in npy files. '''\\n\",\n \" def __init__(self, x_context, x_body, y_cat, y_cont, transform, context_norm, body_norm):\\n\",\n \" super(Emotic_PreDataset,self).__init__()\\n\",\n \" self.x_context = x_context\\n\",\n \" self.x_body = x_body\\n\",\n \" self.y_cat = y_cat \\n\",\n \" self.y_cont = y_cont\\n\",\n \" self.transform = transform \\n\",\n \" self.context_norm = transforms.Normalize(context_norm[0], context_norm[1]) # Normalizing the context image with context mean and context std\\n\",\n \" self.body_norm = transforms.Normalize(body_norm[0], body_norm[1]) # Normalizing the body image with body mean and body std\\n\",\n \"\\n\",\n \" def __len__(self):\\n\",\n \" return len(self.y_cat)\\n\",\n \" \\n\",\n \" def __getitem__(self, index):\\n\",\n \" image_context = self.x_context[index]\\n\",\n \" image_body = self.x_body[index]\\n\",\n \" cat_label = self.y_cat[index]\\n\",\n \" cont_label = self.y_cont[index]\\n\",\n \" return self.context_norm(self.transform(image_context)), self.body_norm(self.transform(image_body)), torch.tensor(cat_label, dtype=torch.float32), torch.tensor(cont_label, dtype=torch.float32)/10.0\\n\",\n \"\\n\",\n \"print ('completed cell')\"\n ],\n \"execution_count\": 6,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"completed cell\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"JFuEQruAxQrK\"\n },\n \"source\": [\n \"## Emotic Losses\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"ObffJVXkqsJg\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"9665ef7f-44a7-4ddf-db6f-a4a0e6430061\"\n },\n \"source\": [\n \"class DiscreteLoss(nn.Module):\\n\",\n \" ''' Class to measure loss between categorical emotion predictions and labels.'''\\n\",\n \" def __init__(self, weight_type='mean', device=torch.device('cpu')):\\n\",\n \" super(DiscreteLoss, self).__init__()\\n\",\n \" self.weight_type = weight_type\\n\",\n \" self.device = device\\n\",\n \" if self.weight_type == 'mean':\\n\",\n \" self.weights = torch.ones((1,26))/26.0\\n\",\n \" self.weights = self.weights.to(self.device)\\n\",\n \" elif self.weight_type == 'static':\\n\",\n \" self.weights = torch.FloatTensor([0.1435, 0.1870, 0.1692, 0.1165, 0.1949, 0.1204, 0.1728, 0.1372, 0.1620,\\n\",\n \" 0.1540, 0.1987, 0.1057, 0.1482, 0.1192, 0.1590, 0.1929, 0.1158, 0.1907,\\n\",\n \" 0.1345, 0.1307, 0.1665, 0.1698, 0.1797, 0.1657, 0.1520, 0.1537]).unsqueeze(0)\\n\",\n \" self.weights = self.weights.to(self.device)\\n\",\n \" \\n\",\n \" def forward(self, pred, target):\\n\",\n \" if self.weight_type == 'dynamic':\\n\",\n \" self.weights = self.prepare_dynamic_weights(target)\\n\",\n \" self.weights = self.weights.to(self.device)\\n\",\n \" loss = (((pred - target)**2) * self.weights)\\n\",\n \" return loss.sum() \\n\",\n \"\\n\",\n \" def prepare_dynamic_weights(self, target):\\n\",\n \" target_stats = torch.sum(target, dim=0).float().unsqueeze(dim=0).cpu()\\n\",\n \" weights = torch.zeros((1,26))\\n\",\n \" weights[target_stats != 0 ] = 1.0/torch.log(target_stats[target_stats != 0].data + 1.2)\\n\",\n \" weights[target_stats == 0] = 0.0001\\n\",\n \" return weights\\n\",\n \"\\n\",\n \"\\n\",\n \"class ContinuousLoss_L2(nn.Module):\\n\",\n \" ''' Class to measure loss between continuous emotion dimension predictions and labels. Using l2 loss as base. '''\\n\",\n \" def __init__(self, margin=1):\\n\",\n \" super(ContinuousLoss_L2, self).__init__()\\n\",\n \" self.margin = margin\\n\",\n \" \\n\",\n \" def forward(self, pred, target):\\n\",\n \" labs = torch.abs(pred - target)\\n\",\n \" loss = labs ** 2 \\n\",\n \" loss[ (labs < self.margin) ] = 0.0\\n\",\n \" return loss.sum()\\n\",\n \"\\n\",\n \"\\n\",\n \"class ContinuousLoss_SL1(nn.Module):\\n\",\n \" ''' Class to measure loss between continuous emotion dimension predictions and labels. Using smooth l1 loss as base. '''\\n\",\n \" def __init__(self, margin=1):\\n\",\n \" super(ContinuousLoss_SL1, self).__init__()\\n\",\n \" self.margin = margin\\n\",\n \" \\n\",\n \" def forward(self, pred, target):\\n\",\n \" labs = torch.abs(pred - target)\\n\",\n \" loss = 0.5 * (labs ** 2)\\n\",\n \" loss[ (labs > self.margin) ] = labs[ (labs > self.margin) ] - 0.5\\n\",\n \" return loss.sum()\\n\",\n \"\\n\",\n \"print ('completed cell')\"\n ],\n \"execution_count\": 7,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"completed cell\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"-AMUYcy5h9cM\"\n },\n \"source\": [\n \"# IV. Load preprocessed data\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"VSadne_Bc5va\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"cea63663-6140-4666-8a80-3e69434b92d6\"\n },\n \"source\": [\n \"# Change data_src variable as per your drive\\n\",\n \"data_src = '/content/drive/My Drive/Colab/Emotic/data'\\n\",\n \"\\n\",\n \"\\n\",\n \"# Load training preprocessed data\\n\",\n \"train_context = np.load(os.path.join(data_src,'pre','train_context_arr.npy'))\\n\",\n \"train_body = np.load(os.path.join(data_src,'pre','train_body_arr.npy'))\\n\",\n \"train_cat = np.load(os.path.join(data_src,'pre','train_cat_arr.npy'))\\n\",\n \"train_cont = np.load(os.path.join(data_src,'pre','train_cont_arr.npy'))\\n\",\n \"\\n\",\n \"# Load validation preprocessed data \\n\",\n \"val_context = np.load(os.path.join(data_src,'pre','val_context_arr.npy'))\\n\",\n \"val_body = np.load(os.path.join(data_src,'pre','val_body_arr.npy'))\\n\",\n \"val_cat = np.load(os.path.join(data_src,'pre','val_cat_arr.npy'))\\n\",\n \"val_cont = np.load(os.path.join(data_src,'pre','val_cont_arr.npy'))\\n\",\n \"\\n\",\n \"# Load testing preprocessed data\\n\",\n \"test_context = np.load(os.path.join(data_src,'pre','test_context_arr.npy'))\\n\",\n \"test_body = np.load(os.path.join(data_src,'pre','test_body_arr.npy'))\\n\",\n \"test_cat = np.load(os.path.join(data_src,'pre','test_cat_arr.npy'))\\n\",\n \"test_cont = np.load(os.path.join(data_src,'pre','test_cont_arr.npy'))\\n\",\n \"\\n\",\n \"# Categorical emotion classes\\n\",\n \"cat = ['Affection', 'Anger', 'Annoyance', 'Anticipation', 'Aversion', 'Confidence', 'Disapproval', 'Disconnection',\\n\",\n \" 'Disquietment', 'Doubt/Confusion', 'Embarrassment', 'Engagement', 'Esteem', 'Excitement', 'Fatigue', 'Fear',\\n\",\n \" 'Happiness', 'Pain', 'Peace', 'Pleasure', 'Sadness', 'Sensitivity', 'Suffering', 'Surprise', 'Sympathy', 'Yearning']\\n\",\n \"\\n\",\n \"cat2ind = {}\\n\",\n \"ind2cat = {}\\n\",\n \"for idx, emotion in enumerate(cat):\\n\",\n \" cat2ind[emotion] = idx\\n\",\n \" ind2cat[idx] = emotion\\n\",\n \"\\n\",\n \"print ('train ', 'context ', train_context.shape, 'body', train_body.shape, 'cat ', train_cat.shape, 'cont', train_cont.shape)\\n\",\n \"print ('val ', 'context ', val_context.shape, 'body', val_body.shape, 'cat ', val_cat.shape, 'cont', val_cont.shape)\\n\",\n \"print ('test ', 'context ', test_context.shape, 'body', test_body.shape, 'cat ', test_cat.shape, 'cont', test_cont.shape)\\n\",\n \"print ('completed cell')\"\n ],\n \"execution_count\": 8,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"train context (23266, 224, 224, 3) body (23266, 128, 128, 3) cat (23266, 26) cont (23266, 3)\\n\",\n \"val context (3315, 224, 224, 3) body (3315, 128, 128, 3) cat (3315, 26) cont (3315, 3)\\n\",\n \"test context (7203, 224, 224, 3) body (7203, 128, 128, 3) cat (7203, 26) cont (7203, 3)\\n\",\n \"completed cell\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"JySFyUFZNgPy\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"84ba41f4-7fee-466e-b1b1-ff2fce976395\"\n },\n \"source\": [\n \"batch_size = 26\\n\",\n \"\\n\",\n \"context_mean = [0.4690646, 0.4407227, 0.40508908]\\n\",\n \"context_std = [0.2514227, 0.24312855, 0.24266963]\\n\",\n \"body_mean = [0.43832874, 0.3964344, 0.3706214]\\n\",\n \"body_std = [0.24784276, 0.23621225, 0.2323653]\\n\",\n \"context_norm = [context_mean, context_std]\\n\",\n \"body_norm = [body_mean, body_std]\\n\",\n \"\\n\",\n \"\\n\",\n \"train_transform = transforms.Compose([transforms.ToPILImage(), \\n\",\n \" transforms.RandomHorizontalFlip(), \\n\",\n \" transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), \\n\",\n \" transforms.ToTensor()])\\n\",\n \"test_transform = transforms.Compose([transforms.ToPILImage(), \\n\",\n \" transforms.ToTensor()])\\n\",\n \"\\n\",\n \"train_dataset = Emotic_PreDataset(train_context, train_body, train_cat, train_cont, \\\\\\n\",\n \" train_transform, context_norm, body_norm)\\n\",\n \"val_dataset = Emotic_PreDataset(val_context, val_body, val_cat, val_cont, \\\\\\n\",\n \" test_transform, context_norm, body_norm)\\n\",\n \"test_dataset = Emotic_PreDataset(test_context, test_body, test_cat, test_cont, \\\\\\n\",\n \" test_transform, context_norm, body_norm)\\n\",\n \"\\n\",\n \"train_loader = DataLoader(train_dataset, batch_size, shuffle=True, drop_last=True)\\n\",\n \"val_loader = DataLoader(val_dataset, batch_size, shuffle=False)\\n\",\n \"test_loader = DataLoader(test_dataset, batch_size, shuffle=False) \\n\",\n \"\\n\",\n \"print ('train loader ', len(train_loader), 'val loader ', len(val_loader), 'test', len(test_loader))\\n\",\n \"print ('completed cell')\"\n ],\n \"execution_count\": 9,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"train loader 894 val loader 128 test 278\\n\",\n \"completed cell\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"wvPoFnAliZBC\"\n },\n \"source\": [\n \"# V. Prepare emotic model\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"cMSaPqJyVyEW\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\",\n \"height\": 83,\n \"referenced_widgets\": [\n \"a8cada3fef3846b2bffe52edacbc190d\",\n \"ae34432e333e4671b3f7f934de91027b\",\n \"674e1fd300d042cbaf9f0e53e7ac4ecd\",\n \"88b121247db64a3490f8c1b16f68c696\",\n \"57ffb85897da4061b318afddca2eed81\",\n \"9cb235dbbe234dbe805b4aa00f7d54ae\",\n \"5e048b1fa84146c8bd2b63a19239cb9e\",\n \"e6a413c0b59f466b9213c1904b1f57f8\",\n \"e0d0abfa1e9441f58722b064823c8119\",\n \"a1bbd4436c154378839f58483fa8c261\",\n \"4f6e592ca3f34209af0ae78a635fc346\"\n ]\n },\n \"outputId\": \"b1b68154-bcfc-438a-c711-31b84177d56c\"\n },\n \"source\": [\n \"model_path_places = './places'\\n\",\n \"\\n\",\n \"model_context = models.__dict__[arch](num_classes=365)\\n\",\n \"context_state_dict = torch.load(os.path.join(model_path_places, 'resnet18_state_dict.pth'))\\n\",\n \"model_context.load_state_dict(context_state_dict)\\n\",\n \"\\n\",\n \"model_body = models.resnet18(pretrained=True)\\n\",\n \"\\n\",\n \"emotic_model = Emotic(list(model_context.children())[-1].in_features, list(model_body.children())[-1].in_features)\\n\",\n \"model_context = nn.Sequential(*(list(model_context.children())[:-1]))\\n\",\n \"model_body = nn.Sequential(*(list(model_body.children())[:-1]))\\n\",\n \"\\n\",\n \"\\n\",\n \"# print (summary(model_context, (3,224,224), device=\\\"cpu\\\"))\\n\",\n \"# print (summary(model_body, (3,128,128), device=\\\"cpu\\\"))\\n\",\n \"\\n\",\n \"print ('completed cell')\"\n ],\n \"execution_count\": 10,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"Downloading: \\\"https://download.pytorch.org/models/resnet18-f37072fd.pth\\\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\\n\"\n ],\n \"name\": \"stderr\"\n },\n {\n \"output_type\": \"display_data\",\n \"data\": {\n \"application/vnd.jupyter.widget-view+json\": {\n \"model_id\": \"a8cada3fef3846b2bffe52edacbc190d\",\n \"version_minor\": 0,\n \"version_major\": 2\n },\n \"text/plain\": [\n \" 0%| | 0.00/44.7M [00:00\"\n ]\n },\n \"metadata\": {\n \"tags\": [],\n \"needs_background\": \"light\"\n }\n }\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"cDa4nuQvjGSa\"\n },\n \"source\": [\n \"# VII. Test model\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"AFCcFv4mnmRi\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"bf220347-2681-4466-dc70-060c0291b5cc\"\n },\n \"source\": [\n \"def test_scikit_ap(cat_preds, cat_labels):\\n\",\n \" ap = np.zeros(26, dtype=np.float32)\\n\",\n \" for i in range(26):\\n\",\n \" ap[i] = average_precision_score(cat_labels[i, :], cat_preds[i, :])\\n\",\n \" print ('ap', ap, ap.shape, ap.mean())\\n\",\n \" return ap.mean()\\n\",\n \"\\n\",\n \"\\n\",\n \"def test_emotic_vad(cont_preds, cont_labels):\\n\",\n \" vad = np.zeros(3, dtype=np.float32)\\n\",\n \" for i in range(3):\\n\",\n \" vad[i] = np.mean(np.abs(cont_preds[i, :] - cont_labels[i, :]))\\n\",\n \" print ('vad', vad, vad.shape, vad.mean())\\n\",\n \" return vad.mean()\\n\",\n \"\\n\",\n \"\\n\",\n \"def get_thresholds(cat_preds, cat_labels):\\n\",\n \" thresholds = np.zeros(26, dtype=np.float32)\\n\",\n \" for i in range(26):\\n\",\n \" p, r, t = precision_recall_curve(cat_labels[i, :], cat_preds[i, :])\\n\",\n \" for k in range(len(p)):\\n\",\n \" if p[k] == r[k]:\\n\",\n \" thresholds[i] = t[k]\\n\",\n \" break\\n\",\n \" np.save('./thresholds.npy', thresholds)\\n\",\n \" return thresholds\\n\",\n \"\\n\",\n \"print ('completed cell')\"\n ],\n \"execution_count\": 14,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"completed cell\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"KOeZRVdbUPNx\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"e20ad71b-9d42-47f5-cda8-0bd08abb27c4\"\n },\n \"source\": [\n \"def test_data(models, device, data_loader, num_images):\\n\",\n \" model_context, model_body, emotic_model = models\\n\",\n \" cat_preds = np.zeros((num_images, 26))\\n\",\n \" cat_labels = np.zeros((num_images, 26))\\n\",\n \" cont_preds = np.zeros((num_images, 3))\\n\",\n \" cont_labels = np.zeros((num_images, 3))\\n\",\n \"\\n\",\n \" with torch.no_grad():\\n\",\n \" model_context.to(device)\\n\",\n \" model_body.to(device)\\n\",\n \" emotic_model.to(device)\\n\",\n \" model_context.eval()\\n\",\n \" model_body.eval()\\n\",\n \" emotic_model.eval()\\n\",\n \" indx = 0\\n\",\n \" print ('starting testing')\\n\",\n \" for images_context, images_body, labels_cat, labels_cont in iter(data_loader):\\n\",\n \" images_context = images_context.to(device)\\n\",\n \" images_body = images_body.to(device)\\n\",\n \"\\n\",\n \" pred_context = model_context(images_context)\\n\",\n \" pred_body = model_body(images_body)\\n\",\n \" pred_cat, pred_cont = emotic_model(pred_context, pred_body)\\n\",\n \"\\n\",\n \" cat_preds[ indx : (indx + pred_cat.shape[0]), :] = pred_cat.to(\\\"cpu\\\").data.numpy()\\n\",\n \" cat_labels[ indx : (indx + labels_cat.shape[0]), :] = labels_cat.to(\\\"cpu\\\").data.numpy()\\n\",\n \" cont_preds[ indx : (indx + pred_cont.shape[0]), :] = pred_cont.to(\\\"cpu\\\").data.numpy() * 10\\n\",\n \" cont_labels[ indx : (indx + labels_cont.shape[0]), :] = labels_cont.to(\\\"cpu\\\").data.numpy() * 10 \\n\",\n \" indx = indx + pred_cat.shape[0]\\n\",\n \"\\n\",\n \" cat_preds = cat_preds.transpose()\\n\",\n \" cat_labels = cat_labels.transpose()\\n\",\n \" cont_preds = cont_preds.transpose()\\n\",\n \" cont_labels = cont_labels.transpose()\\n\",\n \" scipy.io.savemat('./cat_preds.mat',mdict={'cat_preds':cat_preds})\\n\",\n \" scipy.io.savemat('./cat_labels.mat',mdict={'cat_labels':cat_labels})\\n\",\n \" scipy.io.savemat('./cont_preds.mat',mdict={'cont_preds':cont_preds})\\n\",\n \" scipy.io.savemat('./cont_labels.mat',mdict={'cont_labels':cont_labels})\\n\",\n \" print ('completed testing')\\n\",\n \" ap_mean = test_scikit_ap(cat_preds, cat_labels)\\n\",\n \" vad_mean = test_emotic_vad(cont_preds, cont_labels)\\n\",\n \" print (ap_mean, vad_mean)\\n\",\n \" return ap_mean, vad_mean \\n\",\n \"\\n\",\n \"print ('completed cell')\"\n ],\n \"execution_count\": 15,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"completed cell\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"id\": \"qIUQLrXBZ2RR\",\n \"outputId\": \"c958d8ba-6e32-438f-c5c5-816d9b9ed829\"\n },\n \"source\": [\n \"model_context = torch.load('./models/model_context1.pth')\\n\",\n \"model_body = torch.load('./models/model_body1.pth')\\n\",\n \"emotic_model = torch.load('./models/model_emotic1.pth')\\n\",\n \"\\n\",\n \"print ('completed cell')\"\n ],\n \"execution_count\": 16,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"completed cell\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"oB69Xo-kLldG\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"b6be064a-25b2-43d3-e7e7-51a7fc8a9304\"\n },\n \"source\": [\n \"val_ap, val_vad = test_data([model_context, model_body, emotic_model], device, val_loader, val_dataset.__len__())\\n\",\n \"test_ap, test_vad = test_data([model_context, model_body, emotic_model], device, test_loader, test_dataset.__len__())\\n\",\n \"\\n\",\n \"print ('validation Mean average precision=%.4f Mean VAD MAE=%.4f' %(val_ap, val_vad))\\n\",\n \"print ('testing Mean average precision=%.4f Mean VAD MAE=%.4f' %(test_ap, test_vad))\"\n ],\n \"execution_count\": 17,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"starting testing\\n\",\n \"completed testing\\n\",\n \"ap [0.3983917 0.18015468 0.22337271 0.95204633 0.17163357 0.7866947\\n\",\n \" 0.23361506 0.37178904 0.19096893 0.20868655 0.06009851 0.98069084\\n\",\n \" 0.26645675 0.7951143 0.13405906 0.08186857 0.8081806 0.16670538\\n\",\n \" 0.29040682 0.49211633 0.20419936 0.08260126 0.18704712 0.14419095\\n\",\n \" 0.3501988 0.11717057] (26,) 0.34147915\\n\",\n \"vad [0.70697206 0.8584789 0.86687875] (3,) 0.81077653\\n\",\n \"0.34147915 0.81077653\\n\",\n \"starting testing\\n\",\n \"completed testing\\n\",\n \"ap [0.29003292 0.08763415 0.14132965 0.56043494 0.07053518 0.75399864\\n\",\n \" 0.11882206 0.2385993 0.16040386 0.173684 0.01993784 0.86009395\\n\",\n \" 0.15641297 0.69662005 0.09915597 0.06025878 0.66563565 0.06506737\\n\",\n \" 0.21911173 0.4214436 0.17897978 0.05904196 0.1752331 0.08228464\\n\",\n \" 0.13343503 0.0820521 ] (26,) 0.2527015\\n\",\n \"vad [0.8996919 1.0314642 0.943558 ] (3,) 0.95823807\\n\",\n \"0.2527015 0.95823807\\n\",\n \"validation Mean average precision=0.3415 Mean VAD MAE=0.8108\\n\",\n \"testing Mean average precision=0.2527 Mean VAD MAE=0.9582\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"T-fc5LNp4len\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\"\n },\n \"outputId\": \"8de112fa-a4bd-43c0-ff44-895b1ae32fe1\"\n },\n \"source\": [\n \"cat_labels = scipy.io.loadmat('./cat_labels.mat')\\n\",\n \"cat_preds = scipy.io.loadmat('./cat_preds.mat')\\n\",\n \"cat_preds = cat_preds['cat_preds']\\n\",\n \"cat_labels = cat_labels['cat_labels']\\n\",\n \"print (cat_preds.shape, cat_labels.shape)\\n\",\n \"\\n\",\n \"#thesholds calculation for inference \\n\",\n \"thresholds = get_thresholds(cat_preds, cat_labels)\\n\",\n \"print (thresholds, thresholds.shape)\\n\",\n \"\\n\",\n \"print ('completed cell')\"\n ],\n \"execution_count\": 18,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"(26, 7203) (26, 7203)\\n\",\n \"[0.11334415 0.32935348 0.17811956 0.1820814 0.24816841 0.13238849\\n\",\n \" 0.23765785 0.10895684 0.07811652 0.07971309 0.14207679 0.47783324\\n\",\n \" 0.08085962 0.14741261 0.12622227 0.12906708 0.22126663 0.2721243\\n\",\n \" 0.10970519 0.10124312 0.18777776 0.14807722 0.2636854 0.09791826\\n\",\n \" 0.0983988 0.0875175 ] (26,)\\n\",\n \"completed cell\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {\n \"id\": \"owTpkHmOjLvr\"\n },\n \"source\": [\n \"# VIII. Average Precision computation using author's script\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"30PEDPHxrkXA\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\",\n \"height\": 101\n },\n \"outputId\": \"8d2ed78c-fadb-40fc-8f11-be409beb8ea0\"\n },\n \"source\": [\n \"!apt install octave\"\n ],\n \"execution_count\": null,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"Reading package lists... Done\\n\",\n \"Building dependency tree \\n\",\n \"Reading state information... Done\\n\",\n \"octave is already the newest version (4.2.2-1ubuntu1).\\n\",\n \"0 upgraded, 0 newly installed, 0 to remove and 31 not upgraded.\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"6fWR4CTMr7Hf\",\n \"colab\": {\n \"base_uri\": \"https://localhost:8080/\",\n \"height\": 34\n },\n \"outputId\": \"b7539f27-3a07-4184-f67f-d4b3d84350f7\"\n },\n \"source\": [\n \"%%writefile eval.m\\n\",\n \"\\n\",\n \"gt = load('./cat_labels.mat')\\n\",\n \"gt = gt.cat_labels\\n\",\n \"\\n\",\n \"pred = load('./cat_preds.mat')\\n\",\n \"pred = pred.cat_preds\\n\",\n \"\\n\",\n \"categories{1} = 'Affection';\\n\",\n \"categories{2} = 'Anger';\\n\",\n \"categories{3} = 'Annoyance';\\n\",\n \"categories{4} = 'Anticipation';\\n\",\n \"categories{5} = 'Aversion';\\n\",\n \"categories{6} = 'Confidence';\\n\",\n \"categories{7} = 'Disapproval';\\n\",\n \"categories{8} = 'Disconnection';\\n\",\n \"categories{9} = 'Disquietment';\\n\",\n \"categories{10} = 'Doubt/Confusion';\\n\",\n \"categories{11} = 'Embarrassment';\\n\",\n \"categories{12} = 'Engagement';\\n\",\n \"categories{13} = 'Esteem';\\n\",\n \"categories{14} = 'Excitement';\\n\",\n \"categories{15} = 'Fatigue';\\n\",\n \"categories{16} = 'Fear';\\n\",\n \"categories{17} = 'Happiness';\\n\",\n \"categories{18} = 'Pain';\\n\",\n \"categories{19} = 'Peace';\\n\",\n \"categories{20} = 'Pleasure';\\n\",\n \"categories{21} = 'Sadness';\\n\",\n \"categories{22} = 'Sensitivity';\\n\",\n \"categories{23} = 'Suffering';\\n\",\n \"categories{24} = 'Surprise';\\n\",\n \"categories{25} = 'Sympathy';\\n\",\n \"categories{26} = 'Yearning';\\n\",\n \"\\n\",\n \"\\n\",\n \"for c = 1:length(categories)\\n\",\n \" confidence = pred(c,:)'; \\n\",\n \" testClass = gt(c,:)';\\n\",\n \" confidence = double(confidence);\\n\",\n \"\\n\",\n \" S = rand('state');\\n\",\n \" rand('state',0);\\n\",\n \" confidence = confidence + rand(size(confidence))*10^(-10);\\n\",\n \" rand('state',S)\\n\",\n \"\\n\",\n \" [S,j] = sort(-confidence);\\n\",\n \" C = testClass(j);\\n\",\n \" n = length(C);\\n\",\n \" \\n\",\n \" REL = sum(C);\\n\",\n \" if n>0\\n\",\n \" RETREL = cumsum(C);\\n\",\n \" RET = (1:n)';\\n\",\n \" else\\n\",\n \" RETREL = 0;\\n\",\n \" RET = 1;\\n\",\n \" end\\n\",\n \"\\n\",\n \" precision = 100*RETREL ./ RET;\\n\",\n \" recall = 100*RETREL / REL;\\n\",\n \" th = -S;\\n\",\n \"\\n\",\n \" % compute AP\\n\",\n \" mrec=[0 ; recall ; 100];\\n\",\n \" mpre=[0 ; precision ; 0];\\n\",\n \" for i=numel(mpre)-1:-1:1\\n\",\n \" mpre(i)=max(mpre(i),mpre(i+1));\\n\",\n \" end\\n\",\n \" i=find(mrec(2:end)~=mrec(1:end-1))+1;\\n\",\n \" averagePrecision=sum((mrec(i)-mrec(i-1)).*mpre(i))/100;\\n\",\n \" ap_list(c) = averagePrecision\\n\",\n \"end\\n\",\n \"\\n\",\n \"display('#######################################')\\n\",\n \"\\n\",\n \"display('Average precision of predictions');\\n\",\n \"for c = 1:length(categories)\\n\",\n \" sp = '............................';\\n\",\n \" cat = strcat(categories{c}, sp);\\n\",\n \" cat = cat(1:18);\\n\",\n \" display(cat);\\n\",\n \" display(ap_list(c));\\n\",\n \"end\"\n ],\n \"execution_count\": null,\n \"outputs\": [\n {\n \"output_type\": \"stream\",\n \"text\": [\n \"Overwriting eval.m\\n\"\n ],\n \"name\": \"stdout\"\n }\n ]\n },\n {\n \"cell_type\": \"code\",\n \"metadata\": {\n \"id\": \"fA1Oc48zvI_l\"\n },\n \"source\": [\n \"!octave -W eval.m\"\n ],\n \"execution_count\": null,\n \"outputs\": []\n }\n ]\n}"
},
{
"path": "LICENSE",
"content": "MIT License\n\nCopyright (c) 2020 Abhishek Tandon \n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
},
{
"path": "README.md",
"content": "# Emotic \n\nHumans use their facial features or expressions to convey how they feel, such as a person may smile when happy and scowl when angry. Historically, computer vision research has focussed on analyzing and learning these facial features to recognize emotions. \nHowever, these facial features are not universal and vary extensively across cultures and situations. \n\n\n![](\"https://raw.githubusercontent.com/Tandon-A/emotic/master/assets/face.jpg\")
![](\"https://raw.githubusercontent.com/Tandon-A/emotic/master/assets/full_scene.jpg\")
\n###### Fig 1: a) (Facial feature) The person looks angry or in pain b) (Whole scene) The person looks elated. \n\nA scene context, as shown in the figure above, can provide additional information about the situations. This project explores the use of context in recognizing emotions in images. \n\n## Pipeline \n\nThe project uses the EMOTIC dataset and follows the methodology as introduced in the paper *['Context based emotion recognition using EMOTIC dataset'](https://arxiv.org/pdf/2003.13401.pdf)*.\n\n![Pipeline](https://raw.githubusercontent.com/Tandon-A/emotic/master/assets/pipeline%20model.jpg \"Model Pipeline\") \n###### Fig 2: Model Pipeline ([Image source](https://arxiv.org/pdf/2003.13401.pdf))\n\nTwo feature extraction modules first extract features over an image. These features are then used by a third module to predict the continuous dimensions (valence, arousal and dominance) and the discrete emotion categories.\n\n## Emotic Dataset \n\nThe Emotic dataset can be used only for **non-commercial research and education purposes**.\nPlease, fill out the following form to request access to the dataset and the corresponding annotations.\n\n[Access Request for EMOTIC](https://forms.gle/wvhComeDHwQPD6TE6)\n\n## Usage\nDownload the Emotic dataset & annotations, and prepare the directory following the below structure: \n```\n├── ...\n│ ├── emotic\n│ | ├── ade20k\n│ | ├── emodb_small\n│ | ├── framesdb\n│ | ├── mscoco \n│ ├── Annotations\n│ | ├── Annotations.mat\n```\n\n1. To convert annotations from mat object to csv files and preprocess the data: \n\n```\n> python mat2py.py --data_dir proj/data/emotic19 --generate_npy\n```\n* data_dir: Path of the directory containing the emotic and annotations folder as described in the above data directory structure. \n* generate_npy: Argument to specify to generate npy files (later used for training and testing) along with CSV files. If not passed only CSV files are generated. \n\n2. To train the model: \n\n```\n> python main.py --mode train --data_path proj/data/emotic_pre --experiment_path proj/debug_exp\n```\n* mode: Mode to run the main file.\n* data_path: Path of the directory which contains the preprocessed data and CSV files generated in the first step. \n* experiment_path: Path of the experiment directory. The directory will save the results, models and logs. \n\n3. To test the model: \n\n```\n> python main.py --mode test --data_path proj/data/emotic_pre --experiment_path proj/debug_exp\n```\n* mode: Mode to run the main file.\n* data_path: Path of the directory which contains the preprocessed data and CSV files generated in the first step. \n* experiment_path: Path of the experiment directory. Models stored in the the directory are used for testing. \n\n4. To perform inference: \n\n```\n> python main.py --mode inference --inference_file proj/debug_exp/inference_file.txt --experiment_path proj/debug_exp\n```\n* mode: Mode to run the main file.\n* inference_file: Text file specifying images to perform inference. A row is: 'full_path_of_image x1 y1 x2 y2', where (x1,y1) and (x2,y2) specify the bounding box. Refer [sample_inference_list.txt](https://github.com/Tandon-A/emotic/blob/master/sample_inference_list.txt).\n* experiment_path: Path of the experiment directory. Models stored in the the directory are used for inference. \n \n \nYou can also train and test models on Emotic dataset by using the [Colab_train_emotic notebook](https://github.com/Tandon-A/emotic/blob/master/Colab_train_emotic.ipynb). [](https://colab.research.google.com/github/Tandon-A/emotic/blob/master/Colab_train_emotic.ipynb)\n\nThe **trained models and thresholds** to use for inference purposes are availble [here](https://drive.google.com/drive/folders/1e-JLA7V73CQD5pjTFCSWnKCmB0gCpV1D?usp=sharing). \n\n## Results \n\n![Result GIF 1](https://github.com/Tandon-A/emotic/blob/master/assets/eld11_gif2.gif \"Result GIF 1\")\n\n## Acknowledgements\n\n* [Places365-CNN](https://github.com/CSAILVision/places365) \n* [Pytorch-Yolo](https://github.com/eriklindernoren/PyTorch-YOLOv3)\n\n### Context Based Emotion Recognition using Emotic Dataset \n_Ronak Kosti, Jose Alvarez, Adria Recasens, Agata Lapedriza_
\n[[Paper]](https://arxiv.org/pdf/2003.13401.pdf) [[Project Webpage]](http://sunai.uoc.edu/emotic/) [[Authors' Implementation]](https://github.com/rkosti/emotic)\n\n```\n@article{kosti2020context,\n title={Context based emotion recognition using emotic dataset},\n author={Kosti, Ronak and Alvarez, Jose M and Recasens, Adria and Lapedriza, Agata},\n journal={arXiv preprint arXiv:2003.13401},\n year={2020}\n}\n```\n\n## Author \n[Abhishek Tandon](https://github.com/Tandon-A)\n\n\n"
},
{
"path": "emotic.py",
"content": "import torch \nimport torch.nn as nn \n\nclass Emotic(nn.Module):\n ''' Emotic Model'''\n def __init__(self, num_context_features, num_body_features):\n super(Emotic,self).__init__()\n self.num_context_features = num_context_features\n self.num_body_features = num_body_features\n self.fc1 = nn.Linear((self.num_context_features + num_body_features), 256)\n self.bn1 = nn.BatchNorm1d(256)\n self.d1 = nn.Dropout(p=0.5)\n self.fc_cat = nn.Linear(256, 26)\n self.fc_cont = nn.Linear(256, 3)\n self.relu = nn.ReLU()\n\n \n def forward(self, x_context, x_body):\n context_features = x_context.view(-1, self.num_context_features)\n body_features = x_body.view(-1, self.num_body_features)\n fuse_features = torch.cat((context_features, body_features), 1)\n fuse_out = self.fc1(fuse_features)\n fuse_out = self.bn1(fuse_out)\n fuse_out = self.relu(fuse_out)\n fuse_out = self.d1(fuse_out) \n cat_out = self.fc_cat(fuse_out)\n cont_out = self.fc_cont(fuse_out)\n return cat_out, cont_out\n"
},
{
"path": "emotic_dataset.py",
"content": "import ast\nimport numpy as np \nimport os \nfrom PIL import Image\n\nimport torch \nfrom torch.utils.data import Dataset\nfrom torchvision import transforms\n\n\nclass Emotic_PreDataset(Dataset):\n ''' Custom Emotic dataset class. Use preprocessed data stored in npy files. '''\n def __init__(self, x_context, x_body, y_cat, y_cont, transform, context_norm, body_norm):\n super(Emotic_PreDataset,self).__init__()\n self.x_context = x_context\n self.x_body = x_body\n self.y_cat = y_cat \n self.y_cont = y_cont\n self.transform = transform \n self.context_norm = transforms.Normalize(context_norm[0], context_norm[1]) # Normalizing the context image with context mean and context std\n self.body_norm = transforms.Normalize(body_norm[0], body_norm[1]) # Normalizing the body image with body mean and body std\n\n def __len__(self):\n return len(self.y_cat)\n \n def __getitem__(self, index):\n image_context = self.x_context[index]\n image_body = self.x_body[index]\n cat_label = self.y_cat[index]\n cont_label = self.y_cont[index]\n return self.context_norm(self.transform(image_context)), self.body_norm(self.transform(image_body)), torch.tensor(cat_label, dtype=torch.float32), torch.tensor(cont_label, dtype=torch.float32)/10.0\n\n\nclass Emotic_CSVDataset(Dataset):\n ''' Custom Emotic dataset class. Use csv files and generated data at runtime. '''\n def __init__(self, data_df, cat2ind, transform, context_norm, body_norm, data_src = './'):\n super(Emotic_CSVDataset,self).__init__()\n self.data_df = data_df\n self.data_src = data_src \n self.transform = transform \n self.cat2ind = cat2ind\n self.context_norm = transforms.Normalize(context_norm[0], context_norm[1]) # Normalizing the context image with context mean and context std\n self.body_norm = transforms.Normalize(body_norm[0], body_norm[1]) # Normalizing the body image with body mean and body std\n\n def __len__(self):\n return len(self.data_df)\n \n def __getitem__(self, index):\n row = self.data_df.loc[index]\n image_context = Image.open(os.path.join(self.data_src, row['Folder'], row['Filename']))\n bbox = ast.literal_eval(row['BBox'])\n image_body = image_context.crop((bbox[0], bbox[1], bbox[2], bbox[3]))\n image_context = image_context.resize((224, 224))\n image_body = image_body.resize((128, 128))\n cat_labels = ast.literal_eval(row['Categorical_Labels'])\n cont_labels = ast.literal_eval(row['Continuous_Labels'])\n one_hot_cat_labels = self.cat_to_one_hot(cat_labels)\n return self.context_norm(self.transform(image_context)), self.body_norm(self.transform(image_body)), torch.tensor(one_hot_cat_labels, dtype=torch.float32), torch.tensor(cont_labels, dtype=torch.float32)/10.0\n \n def cat_to_one_hot(self, cat):\n one_hot_cat = np.zeros(26)\n for em in cat:\n one_hot_cat[self.cat2ind[em]] = 1\n return one_hot_cat\n"
},
{
"path": "inference.py",
"content": "import cv2\nimport numpy as np \nimport os \n\nimport torch \nfrom torchvision import transforms\n\nfrom emotic import Emotic \n\n\ndef process_images(context_norm, body_norm, image_context_path=None, image_context=None, image_body=None, bbox=None):\n ''' Prepare context and body image. \n :param context_norm: List containing mean and std values for context images. \n :param body_norm: List containing mean and std values for body images. \n :param image_context_path: Path of the context image. \n :param image_context: Numpy array of the context image.\n :param image_body: Numpy array of the body image. \n :param bbox: List to specify the bounding box to generate the body image. bbox = [x1, y1, x2, y2].\n :return: Transformed image_context tensor and image_body tensor.\n '''\n if image_context is None and image_context_path is None:\n raise ValueError('both image_context and image_context_path cannot be none. Please specify one of the two.')\n if image_body is None and bbox is None: \n raise ValueError('both body image and bounding box cannot be none. Please specify one of the two')\n\n if image_context_path is not None:\n image_context = cv2.cvtColor(cv2.imread(image_context_path), cv2.COLOR_BGR2RGB)\n \n if bbox is not None:\n image_body = image_context[bbox[1]:bbox[3],bbox[0]:bbox[2]].copy()\n \n image_context = cv2.resize(image_context, (224,224))\n image_body = cv2.resize(image_body, (128,128))\n \n test_transform = transforms.Compose([transforms.ToPILImage(),transforms.ToTensor()])\n context_norm = transforms.Normalize(context_norm[0], context_norm[1]) \n body_norm = transforms.Normalize(body_norm[0], body_norm[1])\n\n image_context = context_norm(test_transform(image_context)).unsqueeze(0)\n image_body = body_norm(test_transform(image_body)).unsqueeze(0)\n\n return image_context, image_body \n\n\ndef infer(context_norm, body_norm, ind2cat, ind2vad, device, thresholds, models, image_context_path=None, image_context=None, image_body=None, bbox=None, to_print=True):\n ''' Perform inference over an image. \n :param context_norm: List containing mean and std values for context images. \n :param body_norm: List containing mean and std values for body images. \n :param ind2cat: Dictionary converting integer index to categorical emotion. \n :param ind2vad: Dictionary converting integer index to continuous emotion dimension (Valence, Arousal and Dominance).\n :param device: Torch device. Used to send tensors to GPU if available.\n :param image_context_path: Path of the context image. \n :param image_context: Numpy array of the context image.\n :param image_body: Numpy array of the body image. \n :param bbox: List to specify the bounding box to generate the body image. bbox = [x1, y1, x2, y2].\n :param to_print: Variable to display inference results.\n :return: Categorical Emotions list and continuous emotion dimensions numpy array.\n '''\n image_context, image_body = process_images(context_norm, body_norm, image_context_path=image_context_path, image_context=image_context, image_body=image_body, bbox=bbox)\n\n model_context, model_body, emotic_model = models\n\n with torch.no_grad():\n image_context = image_context.to(device)\n image_body = image_body.to(device)\n \n pred_context = model_context(image_context)\n pred_body = model_body(image_body)\n pred_cat, pred_cont = emotic_model(pred_context, pred_body)\n pred_cat = pred_cat.squeeze(0)\n pred_cont = pred_cont.squeeze(0).to(\"cpu\").data.numpy()\n\n bool_cat_pred = torch.gt(pred_cat, thresholds)\n \n cat_emotions = list()\n for i in range(len(bool_cat_pred)):\n if bool_cat_pred[i] == True:\n cat_emotions.append(ind2cat[i])\n\n if to_print == True:\n print ('\\n Image predictions')\n print ('Continuous Dimnesions Predictions') \n for i in range(len(pred_cont)):\n print ('Continuous %10s %.5f' %(ind2vad[i], 10*pred_cont[i]))\n print ('Categorical Emotion Predictions')\n for emotion in cat_emotions:\n print ('Categorical %16s' %(emotion))\n \n return cat_emotions, 10*pred_cont\n\n\ndef inference_emotic(images_list, model_path, result_path, context_norm, body_norm, ind2cat, ind2vad, args):\n ''' Infer on list of images defined in a text file. Save the results in inference_file.txt in the directory specified by the result_path. \n :param images_list: Text file specifying the images and their bounding box values to conduct inference. A row in the file is Path_of_image x1 y1 x2 y2. \n :param model_path: Directory path to load models and val_thresholds to perform inference.\n :param result_path: Directory path to save the results (text file containig categorical emotion and continuous emotion dimension prediction per image).\n :param context_norm: List containing mean and std values for context images. \n :param body_norm: List containing mean and std values for body images. \n :param ind2cat: Dictionary converting integer index to categorical emotion. \n :param ind2vad: Dictionary converting integer index to continuous emotion dimension (Valence, Arousal and Dominance).\n :param args: Runtime arguments.\n '''\n with open(images_list, 'r') as f:\n lines = f.readlines()\n \n device = torch.device(\"cuda:%s\" %(str(args.gpu)) if torch.cuda.is_available() else \"cpu\")\n thresholds = torch.FloatTensor(np.load(os.path.join(result_path, 'val_thresholds.npy'))).to(device) \n model_context = torch.load(os.path.join(model_path,'model_context1.pth')).to(device)\n model_body = torch.load(os.path.join(model_path,'model_body1.pth')).to(device)\n emotic_model = torch.load(os.path.join(model_path,'model_emotic1.pth')).to(device)\n model_context.eval()\n model_body.eval()\n emotic_model.eval()\n models = [model_context, model_body, emotic_model]\n\n\n result_file = os.path.join(result_path, 'inference_list.txt')\n with open(result_file, 'w') as f:\n pass\n \n for idx, line in enumerate(lines):\n image_context_path, x1, y1, x2, y2 = line.split('\\n')[0].split(' ')\n bbox = [int(x1), int(y1), int(x2), int(y2)]\n pred_cat, pred_cont = infer(context_norm, body_norm, ind2cat, ind2vad, device, thresholds, models, image_context_path=image_context_path, bbox=bbox)\n \n write_line = list()\n write_line.append(image_context_path)\n for emotion in pred_cat:\n write_line.append(emotion)\n for continuous in pred_cont:\n write_line.append(str('%.4f' %(continuous)))\n write_line = ' '.join(write_line) \n with open(result_file, 'a') as f:\n f.writelines(write_line)\n f.writelines('\\n')\n"
},
{
"path": "loss.py",
"content": "import torch\nimport torch.nn as nn\n\nclass DiscreteLoss(nn.Module):\n ''' Class to measure loss between categorical emotion predictions and labels.'''\n def __init__(self, weight_type='mean', device=torch.device('cpu')):\n super(DiscreteLoss, self).__init__()\n self.weight_type = weight_type\n self.device = device\n if self.weight_type == 'mean':\n self.weights = torch.ones((1,26))/26.0\n self.weights = self.weights.to(self.device)\n elif self.weight_type == 'static':\n self.weights = torch.FloatTensor([0.1435, 0.1870, 0.1692, 0.1165, 0.1949, 0.1204, 0.1728, 0.1372, 0.1620,\n 0.1540, 0.1987, 0.1057, 0.1482, 0.1192, 0.1590, 0.1929, 0.1158, 0.1907,\n 0.1345, 0.1307, 0.1665, 0.1698, 0.1797, 0.1657, 0.1520, 0.1537]).unsqueeze(0)\n self.weights = self.weights.to(self.device)\n \n def forward(self, pred, target):\n if self.weight_type == 'dynamic':\n self.weights = self.prepare_dynamic_weights(target)\n self.weights = self.weights.to(self.device)\n loss = (((pred - target)**2) * self.weights)\n return loss.sum() \n\n def prepare_dynamic_weights(self, target):\n target_stats = torch.sum(target, dim=0).float().unsqueeze(dim=0).cpu()\n weights = torch.zeros((1,26))\n weights[target_stats != 0 ] = 1.0/torch.log(target_stats[target_stats != 0].data + 1.2)\n weights[target_stats == 0] = 0.0001\n return weights\n\n\nclass ContinuousLoss_L2(nn.Module):\n ''' Class to measure loss between continuous emotion dimension predictions and labels. Using l2 loss as base. '''\n def __init__(self, margin=1):\n super(ContinuousLoss_L2, self).__init__()\n self.margin = margin\n \n def forward(self, pred, target):\n labs = torch.abs(pred - target)\n loss = labs ** 2 \n loss[ (labs < self.margin) ] = 0.0\n return loss.sum()\n\n\nclass ContinuousLoss_SL1(nn.Module):\n ''' Class to measure loss between continuous emotion dimension predictions and labels. Using smooth l1 loss as base. '''\n def __init__(self, margin=1):\n super(ContinuousLoss_SL1, self).__init__()\n self.margin = margin\n \n def forward(self, pred, target):\n labs = torch.abs(pred - target)\n loss = 0.5 * (labs ** 2)\n loss[ (labs > self.margin) ] = labs[ (labs > self.margin) ] - 0.5\n return loss.sum()\n\n\nif __name__ == '__main__':\n\n # Discrete Loss function test \n target = torch.zeros((2,26))\n target[0, 0:13] = 1\n target[1, 13:] = 2\n target[:, 13] = 0\n \n pred = torch.ones((2,26)) * 1\n target = target.cuda()\n pred = pred.cuda()\n pred.requires_grad = True\n target.requires_grad = False\n\n disc_loss = DiscreteLoss('dynamic', torch.device(\"cuda:0\"))\n loss = disc_loss(pred, target)\n print ('discrete loss class', loss, loss.shape, loss.dtype, loss.requires_grad) # loss = 37.1217\n\n #Continuous Loss function test\n target = torch.ones((2,3))\n target[0, :] = 0.9\n target[1, :] = 0.2\n target = target.cuda()\n pred = torch.ones((2,3))\n pred[0, :] = 0.7\n pred[1, :] = 0.25\n pred = pred.cuda()\n pred.requires_grad = True\n target.requires_grad = False\n\n cont_loss_SL1 = ContinuousLoss_SL1()\n loss = cont_loss_SL1(pred*10, target * 10)\n print ('continuous SL1 loss class', loss, loss.shape, loss.dtype, loss.requires_grad) # loss = 4.8750\n\n cont_loss_L2 = ContinuousLoss_L2()\n loss = cont_loss_L2(pred*10, target * 10)\n print ('continuous L2 loss class', loss, loss.shape, loss.dtype, loss.requires_grad) # loss = 12.0\n"
},
{
"path": "main.py",
"content": "import argparse\nimport os\n\nfrom emotic import Emotic\nfrom train import train_emotic\nfrom test import test_emotic\nfrom inference import inference_emotic\n\ndef parse_args():\n parser = argparse.ArgumentParser()\n parser.add_argument('--gpu', type=int, default=0, help='gpu id')\n parser.add_argument('--mode', type=str, default='train_test', choices=['train', 'test', 'train_test', 'inference'])\n parser.add_argument('--data_path', type=str, help='Path to preprocessed data npy files/ csv files')\n parser.add_argument('--experiment_path', type=str, required=True, help='Path to save experiment files (results, models, logs)')\n parser.add_argument('--model_dir_name', type=str, default='models', help='Name of the directory to save models')\n parser.add_argument('--result_dir_name', type=str, default='results', help='Name of the directory to save results(predictions, labels mat files)')\n parser.add_argument('--log_dir_name', type=str, default='logs', help='Name of the directory to save logs (train, val)')\n parser.add_argument('--inference_file', type=str, help='Text file containing image context paths and bounding box')\n parser.add_argument('--context_model', type=str, default='resnet18', choices=['resnet18', 'resnet50'], help='context model type')\n parser.add_argument('--body_model', type=str, default='resnet18', choices=['resnet18', 'resnet50'], help='body model type')\n parser.add_argument('--learning_rate', type=float, default=0.01)\n parser.add_argument('--weight_decay', type=float, default=5e-4)\n parser.add_argument('--cat_loss_weight', type=float, default=0.5, help='weight for discrete loss')\n parser.add_argument('--cont_loss_weight', type=float, default=0.5, help='weight fot continuous loss')\n parser.add_argument('--continuous_loss_type', type=str, default='Smooth L1', choices=['L2', 'Smooth L1'], help='type of continuous loss')\n parser.add_argument('--discrete_loss_weight_type', type=str, default='dynamic', choices=['dynamic', 'mean', 'static'], help='weight policy for discrete loss')\n parser.add_argument('--epochs', type=int, default=15)\n parser.add_argument('--batch_size', type=int, default=52) # use batch size = double(categorical emotion classes)\n # Generate args\n args = parser.parse_args()\n return args\n\n\ndef check_paths(args): \n ''' Check (create if they don't exist) experiment directories.\n :param args: Runtime arguments as passed by the user.\n :return: List containing result_dir_path, model_dir_path, train_log_dir_path, val_log_dir_path.\n '''\n folders= [args.result_dir_name, args.model_dir_name]\n paths = list()\n for folder in folders:\n folder_path = os.path.join(args.experiment_path, folder)\n if not os.path.exists(folder_path):\n os.makedirs(folder_path)\n paths.append(folder_path)\n \n log_folders = ['train', 'val']\n for folder in log_folders:\n folder_path = os.path.join(args.experiment_path, args.log_dir_name, folder)\n if not os.path.exists(folder_path):\n os.makedirs(folder_path)\n paths.append(folder_path)\n return paths\n\n\nif __name__ == '__main__':\n args = parse_args()\n print ('mode ', args.mode)\n\n result_path, model_path, train_log_path, val_log_path = check_paths(args)\n\n cat = ['Affection', 'Anger', 'Annoyance', 'Anticipation', 'Aversion', 'Confidence', 'Disapproval', 'Disconnection', \\\n 'Disquietment', 'Doubt/Confusion', 'Embarrassment', 'Engagement', 'Esteem', 'Excitement', 'Fatigue', 'Fear','Happiness', \\\n 'Pain', 'Peace', 'Pleasure', 'Sadness', 'Sensitivity', 'Suffering', 'Surprise', 'Sympathy', 'Yearning']\n cat2ind = {}\n ind2cat = {}\n for idx, emotion in enumerate(cat):\n cat2ind[emotion] = idx\n ind2cat[idx] = emotion\n \n vad = ['Valence', 'Arousal', 'Dominance']\n ind2vad = {}\n for idx, continuous in enumerate(vad):\n ind2vad[idx] = continuous\n\n context_mean = [0.4690646, 0.4407227, 0.40508908]\n context_std = [0.2514227, 0.24312855, 0.24266963]\n body_mean = [0.43832874, 0.3964344, 0.3706214]\n body_std = [0.24784276, 0.23621225, 0.2323653]\n context_norm = [context_mean, context_std]\n body_norm = [body_mean, body_std]\n\n if args.mode == 'train':\n if args.data_path is None:\n raise ValueError('Data path not provided. Please pass a valid data path for training')\n with open(os.path.join(args.experiment_path, 'config.txt'), 'w') as f:\n print(args, file=f)\n train_emotic(result_path, model_path, train_log_path, val_log_path, ind2cat, ind2vad, context_norm, body_norm, args)\n elif args.mode == 'test':\n if args.data_path is None:\n raise ValueError('Data path not provided. Please pass a valid data path for testing')\n test_emotic(result_path, model_path, ind2cat, ind2vad, context_norm, body_norm, args)\n elif args.mode == 'train_test':\n if args.data_path is None:\n raise ValueError('Data path not provided. Please pass a valid data path for training and testing')\n with open(os.path.join(args.experiment_path, 'config.txt'), 'w') as f:\n print(args, file=f)\n train_emotic(result_path, model_path, train_log_path, val_log_path, ind2cat, ind2vad, context_norm, body_norm, args)\n test_emotic(result_path, model_path, ind2cat, ind2vad, context_norm, body_norm, args)\n elif args.mode == 'inference':\n if args.inference_file is None:\n raise ValueError('Inference file not provided. Please pass a valid inference file for inference')\n inference_emotic(args.inference_file, model_path, result_path, context_norm, body_norm, ind2cat, ind2vad, args)\n else:\n raise ValueError('Unknown mode')\n"
},
{
"path": "mat2py.py",
"content": "import argparse \nimport csv \nimport cv2\nimport numpy as np \nimport os \nfrom scipy.io import loadmat \n\n\nclass emotic_train:\n def __init__(self, filename, folder, image_size, person):\n self.filename = filename\n self.folder = folder\n self.im_size = []\n self.bbox = []\n self.cat = []\n self.cont = []\n self.gender = person[3][0]\n self.age = person[4][0]\n self.cat_annotators = 0\n self.cont_annotators = 0\n self.set_imsize(image_size)\n self.set_bbox(person[0])\n self.set_cat(person[1])\n self.set_cont(person[2])\n self.check_cont()\n\n def set_imsize(self, image_size):\n image_size = np.array(image_size).flatten().tolist()[0]\n row = np.array(image_size[0]).flatten().tolist()[0]\n col = np.array(image_size[1]).flatten().tolist()[0]\n self.im_size.append(row)\n self.im_size.append(col)\n\n def validate_bbox(self, bbox):\n x1, y1, x2, y2 = bbox\n x1 = min(self.im_size[0], max(0, x1))\n x2 = min(self.im_size[0], max(0, x2))\n y1 = min(self.im_size[1], max(0, y1))\n y2 = min(self.im_size[1], max(0, y2))\n return [int(x1), int(y1), int(x2), int(y2)]\n\n def set_bbox(self, person_bbox):\n self.bbox = self.validate_bbox(np.array(person_bbox).flatten().tolist())\n\n def set_cat(self, person_cat):\n cat = np.array(person_cat).flatten().tolist()\n cat = np.array(cat[0]).flatten().tolist()\n self.cat = [np.array(c).flatten().tolist()[0] for c in cat]\n self.cat_annotators = 1\n\n def set_cont(self, person_cont):\n cont = np.array(person_cont).flatten().tolist()[0]\n self.cont = [np.array(c).flatten().tolist()[0] for c in cont]\n self.cont_annotators = 1\n\n def check_cont(self):\n for c in self.cont:\n if np.isnan(c):\n self.cont_annotators = 0\n break\n\nclass emotic_test:\n def __init__(self, filename, folder, image_size, person):\n self.filename = filename\n self.folder = folder\n self.im_size = []\n self.bbox = []\n self.cat = []\n self.cat_annotators = 0\n self.comb_cat = []\n self.cont_annotators = 0\n self.cont = []\n self.comb_cont = []\n self.gender = person[5][0]\n self.age = person[6][0]\n\n self.set_imsize(image_size)\n self.set_bbox(person[0])\n self.set_cat(person[1])\n self.set_comb_cat(person[2])\n self.set_cont(person[3])\n self.set_comb_cont(person[4])\n self.check_cont()\n\n def set_imsize(self, image_size):\n image_size = np.array(image_size).flatten().tolist()[0]\n row = np.array(image_size[0]).flatten().tolist()[0]\n col = np.array(image_size[1]).flatten().tolist()[0]\n self.im_size.append(row)\n self.im_size.append(col)\n\n def validate_bbox(self, bbox):\n x1, y1, x2, y2 = bbox\n x1 = min(self.im_size[0], max(0, x1))\n x2 = min(self.im_size[0], max(0, x2))\n y1 = min(self.im_size[1], max(0, y1))\n y2 = min(self.im_size[1], max(0, y2))\n return [int(x1), int(y1), int(x2), int(y2)]\n\n def set_bbox(self, person_bbox):\n self.bbox = self.validate_bbox(np.array(person_bbox).flatten().tolist())\n\n def set_cat(self, person_cat):\n self.cat_annotators = len(person_cat[0])\n for ann in range(self.cat_annotators):\n ann_cat = person_cat[0][ann]\n ann_cat = np.array(ann_cat).flatten().tolist()\n ann_cat = np.array(ann_cat[0]).flatten().tolist()\n ann_cat = [np.array(c).flatten().tolist()[0] for c in ann_cat]\n self.cat.append(ann_cat)\n\n def set_comb_cat(self, person_comb_cat):\n if self.cat_annotators != 0:\n self.comb_cat = [np.array(c).flatten().tolist()[0] for c in person_comb_cat[0]]\n else:\n self.comb_cat = []\n\n def set_comb_cont(self, person_comb_cont):\n if self.cont_annotators != 0:\n comb_cont = [np.array(c).flatten().tolist()[0] for c in person_comb_cont[0]]\n self.comb_cont = [np.array(c).flatten().tolist()[0] for c in comb_cont[0]]\n else:\n self.comb_cont = []\n\n def set_cont(self, person_cont):\n self.cont_annotators = len(person_cont[0])\n for ann in range(self.cont_annotators):\n ann_cont = person_cont[0][ann]\n ann_cont = np.array(ann_cont).flatten().tolist()\n ann_cont = np.array(ann_cont[0]).flatten().tolist()\n ann_cont = [np.array(c).flatten().tolist()[0] for c in ann_cont]\n self.cont.append(ann_cont)\n\n def check_cont(self):\n for c in self.comb_cont:\n if np.isnan(c):\n self.cont_annotators = 0\n break\n\n\ndef cat_to_one_hot(y_cat):\n '''\n One hot encode a categorical label. \n :param y_cat: Categorical label.\n :return: One hot encoded categorical label. \n '''\n one_hot_cat = np.zeros(26)\n for em in y_cat:\n one_hot_cat[cat2ind[em]] = 1\n return one_hot_cat\n\ndef prepare_data(data_mat, data_path_src, save_dir, dataset_type='train', generate_npy=False, debug_mode=False):\n '''\n Prepare csv files and save preprocessed data in npy files. \n :param data_mat: Mat data object for a label. \n :param data_path_src: Path of the parent directory containing the emotic images folders (mscoco, framesdb, emodb_small, ade20k)\n :param save_dir: Path of the directory to save the csv files and the npy files (if generate_npy files is True)\n :param dataset_type: Type of the dataset (train, val or test). Variable used in the name of csv files and npy files. \n :param generate_npy: If True the data is preprocessed and saved in npy files. Npy files are later used for training. \n '''\n data_set = list()\n\n if generate_npy:\n context_arr = list()\n body_arr = list()\n cat_arr = list()\n cont_arr = list()\n \n to_break = 0\n path_not_exist = 0\n cat_cont_zero = 0\n idx = 0\n for ex_idx, ex in enumerate(data_mat[0]):\n nop = len(ex[4][0])\n for person in range(nop):\n if dataset_type == 'train':\n et = emotic_train(ex[0][0],ex[1][0],ex[2],ex[4][0][person])\n else:\n et = emotic_test(ex[0][0],ex[1][0],ex[2],ex[4][0][person])\n try:\n image_path = os.path.join(data_path_src,et.folder,et.filename)\n if not os.path.exists(image_path):\n path_not_exist += 1\n print ('path not existing', ex_idx, image_path)\n continue\n else:\n context = cv2.cvtColor(cv2.imread(image_path),cv2.COLOR_BGR2RGB)\n body = context[et.bbox[1]:et.bbox[3],et.bbox[0]:et.bbox[2]].copy()\n context_cv = cv2.resize(context, (224,224))\n body_cv = cv2.resize(body, (128,128))\n except Exception as e:\n to_break += 1\n if debug_mode == True:\n print ('breaking at idx=%d, %d due to exception=%r' %(ex_idx, idx, e))\n continue\n if (et.cat_annotators == 0 or et.cont_annotators == 0):\n cat_cont_zero += 1\n continue\n data_set.append(et) \n if generate_npy == True:\n context_arr.append(context_cv)\n body_arr.append(body_cv)\n if dataset_type == 'train':\n cat_arr.append(cat_to_one_hot(et.cat))\n cont_arr.append(np.array(et.cont))\n else: \n cat_arr.append(cat_to_one_hot(et.comb_cat))\n cont_arr.append(np.array(et.comb_cont))\n if idx % 1000 == 0 and debug_mode==False:\n print (\" Preprocessing data. Index = \", idx)\n elif idx % 20 == 0 and debug_mode==True:\n print (\" Preprocessing data. Index = \", idx)\n idx = idx + 1\n # for debugging purposes\n if debug_mode == True and idx >= 104:\n print (' ######## Breaking data prep step', idx, ex_idx, ' ######')\n print (to_break, path_not_exist, cat_cont_zero)\n cv2.imwrite(os.path.join(save_dir, 'context1.png'), context_arr[-1])\n cv2.imwrite(os.path.join(save_dir, 'body1.png'), body_arr[-1])\n break\n print (to_break, path_not_exist, cat_cont_zero)\n \n csv_path = os.path.join(save_dir, \"%s.csv\" %(dataset_type))\n with open(csv_path, 'w') as csvfile:\n filewriter = csv.writer(csvfile, delimiter=',', dialect='excel')\n row = ['Index', 'Folder', 'Filename', 'Image Size', 'BBox', 'Categorical_Labels', 'Continuous_Labels', 'Gender', 'Age']\n filewriter.writerow(row)\n for idx, ex in enumerate(data_set):\n if dataset_type == 'train':\n row = [idx, ex.folder, ex.filename, ex.im_size, ex.bbox, ex.cat, ex.cont, ex.gender, ex.age]\n else:\n row = [idx, ex.folder, ex.filename, ex.im_size, ex.bbox, ex.comb_cat, ex.comb_cont, ex.gender, ex.age]\n filewriter.writerow(row)\n print ('wrote file ', csv_path)\n\n if generate_npy == True: \n context_arr = np.array(context_arr)\n body_arr = np.array(body_arr)\n cat_arr = np.array(cat_arr)\n cont_arr = np.array(cont_arr)\n print (len(data_set), context_arr.shape, body_arr.shape)\n np.save(os.path.join(save_dir,'%s_context_arr.npy' %(dataset_type)), context_arr)\n np.save(os.path.join(save_dir,'%s_body_arr.npy' %(dataset_type)), body_arr)\n np.save(os.path.join(save_dir,'%s_cat_arr.npy' %(dataset_type)), cat_arr)\n np.save(os.path.join(save_dir,'%s_cont_arr.npy' %(dataset_type)), cont_arr)\n print (context_arr.shape, body_arr.shape, cat_arr.shape, cont_arr.shape)\n print ('completed generating %s data files' %(dataset_type))\n \n\ndef parse_args():\n parser = argparse.ArgumentParser()\n parser.add_argument('--data_dir', type=str, required=True, help='Path to Emotic data and annotations')\n parser.add_argument('--save_dir_name', type=str, default='emotic_pre', help='Directory name in which preprocessed data will be stored')\n parser.add_argument('--label', type=str, default='all', choices=['train', 'val', 'test', 'all'])\n parser.add_argument('--generate_npy', action='store_true', help='Generate npy files')\n parser.add_argument('--debug_mode', action='store_true', help='Debug mode. Will only save a small subset of the data')\n # Generate args\n args = parser.parse_args()\n return args\n \nif __name__ == '__main__':\n args = parse_args()\n ann_path_src = os.path.join(args.data_dir, 'Annotations','Annotations.mat')\n data_path_src = os.path.join(args.data_dir, 'emotic')\n save_path = os.path.join(args.data_dir, args.save_dir_name)\n if not os.path.exists(save_path):\n os.makedirs(save_path)\n \n cat = ['Affection', 'Anger', 'Annoyance', 'Anticipation', 'Aversion', 'Confidence', 'Disapproval', 'Disconnection',\n 'Disquietment', 'Doubt/Confusion', 'Embarrassment', 'Engagement', 'Esteem', 'Excitement', 'Fatigue', 'Fear',\n 'Happiness', 'Pain', 'Peace', 'Pleasure', 'Sadness', 'Sensitivity', 'Suffering', 'Surprise', 'Sympathy', 'Yearning']\n cat2ind = {}\n ind2cat = {}\n for idx, emotion in enumerate(cat):\n cat2ind[emotion] = idx\n ind2cat[idx] = emotion\n \n print ('loading Annotations')\n mat = loadmat(ann_path_src)\n if args.label.lower() == 'all':\n labels = ['train', 'val', 'test']\n else:\n labels = [args.label.lower()]\n for label in labels:\n data_mat = mat[label]\n print ('starting label ', label)\n prepare_data(data_mat, data_path_src, save_path, dataset_type=label, generate_npy=args.generate_npy, debug_mode=args.debug_mode)\n"
},
{
"path": "prepare_models.py",
"content": "import os\nimport torch\nfrom torch.autograd import Variable as V\nimport torchvision.models as models\nfrom torch.nn import functional as F\n\n\ndef prep_models(context_model='resnet18', body_model='resnet18', model_dir='./'):\n ''' Download imagenet pretrained models for context_model and body_model.\n :param context_model: Model to use for conetxt features.\n :param body_model: Model to use for body features.\n :param model_dir: Directory path where to store pretrained models.\n :return: Yolo model after loading model weights\n '''\n model_name = '%s_places365.pth.tar' % context_model\n model_file = os.path.join(model_dir, model_name)\n if not os.path.exists(model_file):\n download_command = 'wget ' + 'http://places2.csail.mit.edu/models_places365/' + model_name +' -O ' + model_file\n os.system(download_command)\n\n save_file = os.path.join(model_dir,'%s_places365_py36.pth.tar' % context_model)\n from functools import partial\n import pickle\n pickle.load = partial(pickle.load, encoding=\"latin1\")\n pickle.Unpickler = partial(pickle.Unpickler, encoding=\"latin1\")\n model = torch.load(model_file, map_location=lambda storage, loc: storage, pickle_module=pickle)\n torch.save(model, save_file)\n\n # create the network architecture\n model_context = models.__dict__[context_model](num_classes=365)\n checkpoint = torch.load(save_file, map_location=lambda storage, loc: storage) # model trained in GPU could be deployed in CPU machine like this!\n if context_model == 'densenet161':\n state_dict = {str.replace(k,'module.',''): v for k,v in checkpoint['state_dict'].items()}\n state_dict = {str.replace(k,'norm.','norm'): v for k,v in state_dict.items()}\n state_dict = {str.replace(k,'conv.','conv'): v for k,v in state_dict.items()}\n state_dict = {str.replace(k,'normweight','norm.weight'): v for k,v in state_dict.items()}\n state_dict = {str.replace(k,'normrunning','norm.running'): v for k,v in state_dict.items()}\n state_dict = {str.replace(k,'normbias','norm.bias'): v for k,v in state_dict.items()}\n state_dict = {str.replace(k,'convweight','conv.weight'): v for k,v in state_dict.items()}\n else:\n state_dict = {str.replace(k,'module.',''): v for k,v in checkpoint['state_dict'].items()} # the data parallel layer will add 'module' before each layer name\n model_context.load_state_dict(state_dict)\n model_context.eval()\n model_context.cpu()\n torch.save(model_context, os.path.join(model_dir, 'context_model' + '.pth'))\n \n print ('completed preparing context model')\n\n model_body = models.__dict__[body_model](pretrained=True)\n model_body.cpu()\n torch.save(model_body, os.path.join(model_dir, 'body_model' + '.pth'))\n\n print ('completed preparing body model')\n return model_context, model_body\n\n\nif __name__ == '__main__':\n prep_models(model_dir='proj/debug_exp/models')\n\n\n"
},
{
"path": "sample_inference_list.txt",
"content": "/data/emotic19/emotic/mscoco/images/COCO_val2014_000000562243.jpg 86 58 564 628\n/data/emotic19/emotic/mscoco/images/COCO_train2014_000000288841.jpg 485 149 605 473"
},
{
"path": "test.py",
"content": "import numpy as np \nimport os \nimport scipy.io\nfrom sklearn.metrics import average_precision_score, precision_recall_curve\n\nimport torch \nimport torch.nn as nn \nfrom torch.utils.data import DataLoader \nimport torchvision.models as models\nfrom torchvision import transforms\n\nfrom emotic import Emotic \nfrom emotic_dataset import Emotic_PreDataset\n\n\ndef test_scikit_ap(cat_preds, cat_labels, ind2cat):\n ''' Calculate average precision per emotion category using sklearn library.\n :param cat_preds: Categorical emotion predictions. \n :param cat_labels: Categorical emotion labels. \n :param ind2cat: Dictionary converting integer index to categorical emotion.\n :return: Numpy array containing average precision per emotion category.\n '''\n ap = np.zeros(26, dtype=np.float32)\n for i in range(26):\n ap[i] = average_precision_score(cat_labels[i, :], cat_preds[i, :])\n print ('Category %16s %.5f' %(ind2cat[i], ap[i]))\n print ('Mean AP %.5f' %(ap.mean()))\n return ap \n\n\ndef test_vad(cont_preds, cont_labels, ind2vad):\n ''' Calcaulate VAD (valence, arousal, dominance) errors. \n :param cont_preds: Continuous emotion predictions. \n :param cont_labels: Continuous emotion labels. \n :param ind2vad: Dictionary converting integer index to continuous emotion dimension (Valence, Arousal and Dominance).\n :return: Numpy array containing mean absolute error per continuous emotion dimension. \n '''\n vad = np.zeros(3, dtype=np.float32)\n for i in range(3):\n vad[i] = np.mean(np.abs(cont_preds[i, :] - cont_labels[i, :]))\n print ('Continuous %10s %.5f' %(ind2vad[i], vad[i]))\n print ('Mean VAD Error %.5f' %(vad.mean()))\n return vad\n\n\ndef get_thresholds(cat_preds, cat_labels):\n ''' Calculate thresholds where precision is equal to recall. These thresholds are then later for inference.\n :param cat_preds: Categorical emotion predictions. \n :param cat_labels: Categorical emotion labels. \n :return: Numpy array containing thresholds per emotion category where precision is equal to recall.\n '''\n thresholds = np.zeros(26, dtype=np.float32)\n for i in range(26):\n p, r, t = precision_recall_curve(cat_labels[i, :], cat_preds[i, :])\n for k in range(len(p)):\n if p[k] == r[k]:\n thresholds[i] = t[k]\n break\n return thresholds\n\n\ndef test_data(models, device, data_loader, ind2cat, ind2vad, num_images, result_dir='./', test_type='val'):\n ''' Test models on data \n :param models: List containing model_context, model_body and emotic_model (fusion model) in that order.\n :param device: Torch device. Used to send tensors to GPU if available. \n :param data_loader: Dataloader iterating over dataset. \n :param ind2cat: Dictionary converting integer index to categorical emotion.\n :param ind2vad: Dictionary converting integer index to continuous emotion dimension (Valence, Arousal and Dominance)\n :param num_images: Number of images in the dataset. \n :param result_dir: Directory path to save results (predictions mat object and thresholds npy object).\n :param test_type: Test type variable. Variable used in the name of thresholds and predictio files.\n '''\n model_context, model_body, emotic_model = models\n cat_preds = np.zeros((num_images, 26))\n cat_labels = np.zeros((num_images, 26))\n cont_preds = np.zeros((num_images, 3))\n cont_labels = np.zeros((num_images, 3))\n\n with torch.no_grad():\n model_context.to(device)\n model_body.to(device)\n emotic_model.to(device)\n model_context.eval()\n model_body.eval()\n emotic_model.eval()\n indx = 0\n print ('starting testing')\n for images_context, images_body, labels_cat, labels_cont in iter(data_loader):\n images_context = images_context.to(device)\n images_body = images_body.to(device)\n\n pred_context = model_context(images_context)\n pred_body = model_body(images_body)\n pred_cat, pred_cont = emotic_model(pred_context, pred_body)\n\n cat_preds[ indx : (indx + pred_cat.shape[0]), :] = pred_cat.to(\"cpu\").data.numpy()\n cat_labels[ indx : (indx + labels_cat.shape[0]), :] = labels_cat.to(\"cpu\").data.numpy()\n cont_preds[ indx : (indx + pred_cont.shape[0]), :] = pred_cont.to(\"cpu\").data.numpy() * 10\n cont_labels[ indx : (indx + labels_cont.shape[0]), :] = labels_cont.to(\"cpu\").data.numpy() * 10\n indx = indx + pred_cat.shape[0]\n\n cat_preds = cat_preds.transpose()\n cat_labels = cat_labels.transpose()\n cont_preds = cont_preds.transpose()\n cont_labels = cont_labels.transpose()\n print ('completed testing')\n \n # Mat files used for emotic testing (matlab script)\n scipy.io.savemat(os.path.join(result_dir, '%s_cat_preds.mat' %(test_type)), mdict={'cat_preds':cat_preds})\n scipy.io.savemat(os.path.join(result_dir, '%s_cat_labels.mat' %(test_type)), mdict={'cat_labels':cat_labels})\n scipy.io.savemat(os.path.join(result_dir, '%s_cont_preds.mat' %(test_type)), mdict={'cont_preds':cont_preds})\n scipy.io.savemat(os.path.join(result_dir, '%s_cont_labels.mat' %(test_type)), mdict={'cont_labels':cont_labels})\n print ('saved mat files')\n\n test_scikit_ap(cat_preds, cat_labels, ind2cat)\n test_vad(cont_preds, cont_labels, ind2vad)\n thresholds = get_thresholds(cat_preds, cat_labels)\n np.save(os.path.join(result_dir, '%s_thresholds.npy' %(test_type)), thresholds)\n print ('saved thresholds')\n\n\ndef test_emotic(result_path, model_path, ind2cat, ind2vad, context_norm, body_norm, args):\n ''' Prepare test data and test models on the same.\n :param result_path: Directory path to save the results (val_predidictions mat object, val_thresholds npy object).\n :param model_path: Directory path to load pretrained base models and save the models after training. \n :param ind2cat: Dictionary converting integer index to categorical emotion. \n :param ind2vad: Dictionary converting integer index to continuous emotion dimension (Valence, Arousal and Dominance).\n :param context_norm: List containing mean and std values for context images. \n :param body_norm: List containing mean and std values for body images. \n :param args: Runtime arguments.\n ''' \n # Prepare models \n model_context = torch.load(os.path.join(model_path,'model_context1.pth'))\n model_body = torch.load(os.path.join(model_path,'model_body1.pth'))\n emotic_model = torch.load(os.path.join(model_path,'model_emotic1.pth'))\n print ('Succesfully loaded models')\n\n #Load data preprocessed npy files\n test_context = np.load(os.path.join(args.data_path, 'test_context_arr.npy'))\n test_body = np.load(os.path.join(args.data_path, 'test_body_arr.npy'))\n test_cat = np.load(os.path.join(args.data_path, 'test_cat_arr.npy'))\n test_cont = np.load(os.path.join(args.data_path, 'test_cont_arr.npy'))\n print ('test ', 'context ', test_context.shape, 'body', test_body.shape, 'cat ', test_cat.shape, 'cont', test_cont.shape)\n\n # Initialize Dataset and DataLoader \n test_transform = transforms.Compose([transforms.ToPILImage(),transforms.ToTensor()])\n test_dataset = Emotic_PreDataset(test_context, test_body, test_cat, test_cont, test_transform, context_norm, body_norm)\n test_loader = DataLoader(test_dataset, args.batch_size, shuffle=False)\n print ('test loader ', len(test_loader))\n \n device = torch.device(\"cuda:%s\" %(str(args.gpu)) if torch.cuda.is_available() else \"cpu\")\n test_data([model_context, model_body, emotic_model], device, test_loader, ind2cat, ind2vad, len(test_dataset), result_dir=result_path, test_type='test')\n"
},
{
"path": "train.py",
"content": "import numpy as np \nimport os \n\nimport torch\nimport torch.nn as nn \nimport torch.nn.functional as F\nimport torch.optim as optim \nfrom torch.optim.lr_scheduler import StepLR\nfrom torch.utils.data import DataLoader \nimport torchvision.models as models\nfrom torchvision import transforms\nfrom tensorboardX import SummaryWriter\n\nfrom emotic import Emotic \nfrom emotic_dataset import Emotic_PreDataset\nfrom loss import DiscreteLoss, ContinuousLoss_SL1, ContinuousLoss_L2\nfrom prepare_models import prep_models\nfrom test import test_data\n\n\ndef train_data(opt, scheduler, models, device, train_loader, val_loader, disc_loss, cont_loss, train_writer, val_writer, model_path, args):\n '''\n Training emotic model on train data using train loader.\n :param opt: Optimizer object.\n :param scheduler: Learning rate scheduler object.\n :param models: List containing model_context, model_body and emotic_model (fusion model) in that order. \n :param device: Torch device. Used to send tensors to GPU if available. \n :param train_loader: Dataloader iterating over train dataset. \n :param val_loader: Dataloader iterating over validation dataset. \n :param disc_loss: Discrete loss criterion. Loss measure between discrete emotion categories predictions and the target emotion categories. \n :param cont_loss: Continuous loss criterion. Loss measure between continuous VAD emotion predictions and the target VAD values.\n :param train_writer: SummaryWriter object to save train logs. \n :param val_writer: SummaryWriter object to save validation logs. \n :param model_path: Directory path to save the models after training. \n :param args: Runtime arguments.\n '''\n \n model_context, model_body, emotic_model = models\n\n emotic_model.to(device)\n model_context.to(device)\n model_body.to(device)\n\n print ('starting training')\n\n for e in range(args.epochs):\n\n running_loss = 0.0 \n running_cat_loss = 0.0 \n running_cont_loss = 0.0\n \n emotic_model.train()\n model_context.train()\n model_body.train()\n \n #train models for one epoch \n for images_context, images_body, labels_cat, labels_cont in iter(train_loader):\n images_context = images_context.to(device)\n images_body = images_body.to(device)\n labels_cat = labels_cat.to(device)\n labels_cont = labels_cont.to(device)\n\n opt.zero_grad()\n\n pred_context = model_context(images_context)\n pred_body = model_body(images_body)\n\n pred_cat, pred_cont = emotic_model(pred_context, pred_body)\n cat_loss_batch = disc_loss(pred_cat, labels_cat)\n cont_loss_batch = cont_loss(pred_cont * 10, labels_cont * 10)\n\n loss = (args.cat_loss_weight * cat_loss_batch) + (args.cont_loss_weight * cont_loss_batch)\n \n running_loss += loss.item()\n running_cat_loss += cat_loss_batch.item()\n running_cont_loss += cont_loss_batch.item()\n \n loss.backward()\n opt.step()\n\n if e % 1 == 0: \n print ('epoch = %d loss = %.4f cat loss = %.4f cont_loss = %.4f' %(e, running_loss, running_cat_loss, running_cont_loss))\n\n train_writer.add_scalar('losses/total_loss', running_loss, e)\n train_writer.add_scalar('losses/categorical_loss', running_cat_loss, e)\n train_writer.add_scalar('losses/continuous_loss', running_cont_loss, e)\n \n running_loss = 0.0 \n running_cat_loss = 0.0 \n running_cont_loss = 0.0 \n \n emotic_model.eval()\n model_context.eval()\n model_body.eval()\n \n with torch.no_grad():\n #validation for one epoch\n for images_context, images_body, labels_cat, labels_cont in iter(val_loader):\n images_context = images_context.to(device)\n images_body = images_body.to(device)\n labels_cat = labels_cat.to(device)\n labels_cont = labels_cont.to(device)\n\n pred_context = model_context(images_context)\n pred_body = model_body(images_body)\n\n pred_cat, pred_cont = emotic_model(pred_context, pred_body)\n cat_loss_batch = disc_loss(pred_cat, labels_cat)\n cont_loss_batch = cont_loss(pred_cont * 10, labels_cont * 10)\n loss = (args.cat_loss_weight * cat_loss_batch) + (args.cont_loss_weight * cont_loss_batch)\n \n running_loss += loss.item()\n running_cat_loss += cat_loss_batch.item()\n running_cont_loss += cont_loss_batch.item()\n\n if e % 1 == 0:\n print ('epoch = %d validation loss = %.4f cat loss = %.4f cont loss = %.4f ' %(e, running_loss, running_cat_loss, running_cont_loss))\n \n val_writer.add_scalar('losses/total_loss', running_loss, e)\n val_writer.add_scalar('losses/categorical_loss', running_cat_loss, e)\n val_writer.add_scalar('losses/continuous_loss', running_cont_loss, e)\n \n scheduler.step()\n \n print ('completed training')\n emotic_model.to(\"cpu\")\n model_context.to(\"cpu\")\n model_body.to(\"cpu\")\n torch.save(emotic_model, os.path.join(model_path, 'model_emotic1.pth'))\n torch.save(model_context, os.path.join(model_path, 'model_context1.pth'))\n torch.save(model_body, os.path.join(model_path, 'model_body1.pth'))\n print ('saved models')\n\n\ndef train_emotic(result_path, model_path, train_log_path, val_log_path, ind2cat, ind2vad, context_norm, body_norm, args):\n ''' Prepare dataset, dataloders, models. \n :param result_path: Directory path to save the results (val_predidictions mat object, val_thresholds npy object).\n :param model_path: Directory path to load pretrained base models and save the models after training. \n :param train_log_path: Directory path to save the training logs. \n :param val_log_path: Directoty path to save the validation logs. \n :param ind2cat: Dictionary converting integer index to categorical emotion. \n :param ind2vad: Dictionary converting integer index to continuous emotion dimension (Valence, Arousal and Dominance).\n :param context_norm: List containing mean and std values for context images. \n :param body_norm: List containing mean and std values for body images. \n :param args: Runtime arguments. \n '''\n # Load preprocessed data from npy files\n train_context = np.load(os.path.join(args.data_path, 'train_context_arr.npy'))\n train_body = np.load(os.path.join(args.data_path, 'train_body_arr.npy'))\n train_cat = np.load(os.path.join(args.data_path, 'train_cat_arr.npy'))\n train_cont = np.load(os.path.join(args.data_path, 'train_cont_arr.npy'))\n\n val_context = np.load(os.path.join(args.data_path, 'val_context_arr.npy'))\n val_body = np.load(os.path.join(args.data_path, 'val_body_arr.npy'))\n val_cat = np.load(os.path.join(args.data_path, 'val_cat_arr.npy'))\n val_cont = np.load(os.path.join(args.data_path, 'val_cont_arr.npy'))\n\n print ('train ', 'context ', train_context.shape, 'body', train_body.shape, 'cat ', train_cat.shape, 'cont', train_cont.shape)\n print ('val ', 'context ', val_context.shape, 'body', val_body.shape, 'cat ', val_cat.shape, 'cont', val_cont.shape)\n\n # Initialize Dataset and DataLoader \n train_transform = transforms.Compose([transforms.ToPILImage(),transforms.RandomHorizontalFlip(), transforms.ColorJitter(brightness=0.4, contrast=0.4, saturation=0.4), transforms.ToTensor()])\n test_transform = transforms.Compose([transforms.ToPILImage(),transforms.ToTensor()])\n\n train_dataset = Emotic_PreDataset(train_context, train_body, train_cat, train_cont, train_transform, context_norm, body_norm)\n val_dataset = Emotic_PreDataset(val_context, val_body, val_cat, val_cont, test_transform, context_norm, body_norm)\n\n train_loader = DataLoader(train_dataset, args.batch_size, shuffle=True)\n val_loader = DataLoader(val_dataset, args.batch_size, shuffle=False)\n\n print ('train loader ', len(train_loader), 'val loader ', len(val_loader))\n\n # Prepare models \n model_context, model_body = prep_models(context_model=args.context_model, body_model=args.body_model, model_dir=model_path)\n emotic_model = Emotic(list(model_context.children())[-1].in_features, list(model_body.children())[-1].in_features)\n model_context = nn.Sequential(*(list(model_context.children())[:-1]))\n model_body = nn.Sequential(*(list(model_body.children())[:-1]))\n\n for param in emotic_model.parameters():\n param.requires_grad = True\n for param in model_context.parameters():\n param.requires_grad = True\n for param in model_body.parameters():\n param.requires_grad = True\n \n device = torch.device(\"cuda:%s\" %(str(args.gpu)) if torch.cuda.is_available() else \"cpu\")\n opt = optim.Adam((list(emotic_model.parameters()) + list(model_context.parameters()) + list(model_body.parameters())), lr=args.learning_rate, weight_decay=args.weight_decay)\n scheduler = StepLR(opt, step_size=7, gamma=0.1)\n disc_loss = DiscreteLoss(args.discrete_loss_weight_type, device)\n if args.continuous_loss_type == 'Smooth L1':\n cont_loss = ContinuousLoss_SL1()\n else:\n cont_loss = ContinuousLoss_L2()\n\n train_writer = SummaryWriter(train_log_path)\n val_writer = SummaryWriter(val_log_path)\n\n # training\n train_data(opt, scheduler, [model_context, model_body, emotic_model], device, train_loader, val_loader, disc_loss, cont_loss, train_writer, val_writer, model_path, args)\n # validation\n test_data([model_context, model_body, emotic_model], device, val_loader, ind2cat, ind2vad, len(val_dataset), result_dir=result_path, test_type='val')\n"
},
{
"path": "yolo_inference.py",
"content": "import argparse \nimport cv2\nimport numpy as np \nimport os \n\nimport torch \nfrom torchvision import transforms\n\nfrom emotic import Emotic \nfrom inference import infer\nfrom yolo_utils import prepare_yolo, rescale_boxes, non_max_suppression\n\ndef parse_args():\n parser = argparse.ArgumentParser()\n parser.add_argument('--gpu', type=int, default=0, help='gpu id')\n parser.add_argument('--experiment_path', type=str, required=True, help='Path of experiment files (results, models, logs)')\n parser.add_argument('--model_dir', type=str, default='models', help='Folder to access the models')\n parser.add_argument('--result_dir', type=str, default='results', help='Path to save the results')\n parser.add_argument('--inference_file', type=str, help='Text file containing image context paths and bounding box')\n parser.add_argument('--video_file', type=str, help='Test video file')\n # Generate args\n args = parser.parse_args()\n return args\n\n\ndef get_bbox(yolo_model, device, image_context, yolo_image_size=416, conf_thresh=0.8, nms_thresh=0.4):\n ''' Use yolo to obtain bounding box of every person in context image. \n :param yolo_model: Yolo model to obtain bounding box of every person in context image. \n :param device: Torch device. Used to send tensors to GPU (if available) for faster processing. \n :yolo_image_size: Input image size for yolo model. \n :conf_thresh: Confidence threshold for yolo model. Predictions with object confidence > conf_thresh are returned. \n :nms_thresh: Non-maximal suppression threshold for yolo model. Predictions with IoU > nms_thresh are returned. \n :return: Numpy array of bounding boxes. Array shape = (no_of_persons, 4). \n '''\n test_transform = transforms.Compose([transforms.ToPILImage(),transforms.ToTensor()])\n image_yolo = test_transform(cv2.resize(image_context, (416, 416))).unsqueeze(0).to(device)\n\n with torch.no_grad():\n detections = yolo_model(image_yolo)\n nms_det = non_max_suppression(detections, conf_thresh, nms_thresh)[0]\n det = rescale_boxes(nms_det, yolo_image_size, (image_context.shape[:2]))\n \n bboxes = []\n for x1, y1, x2, y2, _, _, cls_pred in det:\n if cls_pred == 0: # checking if predicted_class = persons. \n x1 = int(min(image_context.shape[1], max(0, x1)))\n x2 = int(min(image_context.shape[1], max(x1, x2)))\n y1 = int(min(image_context.shape[0], max(15, y1)))\n y2 = int(min(image_context.shape[0], max(y1, y2)))\n bboxes.append([x1, y1, x2, y2])\n return np.array(bboxes)\n\n\ndef yolo_infer(images_list, result_path, model_path, context_norm, body_norm, ind2cat, ind2vad, args):\n ''' Infer on a list of images defined in images_list text file to obtain bounding boxes of persons in the images using yolo model.\n :param images_list: Text file specifying the images to conduct inference. A row in the file is Path_of_image. \n :param result_path: Directory path to save the results (images with the predicted emotion categories and continuous emotion dimesnions).\n :param model_path: Directory path to load models and val_thresholds to perform inference.\n :param context_norm: List containing mean and std values for context images. \n :param body_norm: List containing mean and std values for body images. \n :param ind2cat: Dictionary converting integer index to categorical emotion. \n :param ind2vad: Dictionary converting integer index to continuous emotion dimension (Valence, Arousal and Dominance).\n :param args: Runtime arguments.\n '''\n device = torch.device(\"cuda:%s\" %(str(args.gpu)) if torch.cuda.is_available() else \"cpu\")\n yolo = prepare_yolo(model_path)\n yolo = yolo.to(device)\n yolo.eval()\n\n thresholds = torch.FloatTensor(np.load(os.path.join(result_path, 'val_thresholds.npy'))).to(device) \n model_context = torch.load(os.path.join(model_path,'model_context1.pth')).to(device)\n model_body = torch.load(os.path.join(model_path,'model_body1.pth')).to(device)\n emotic_model = torch.load(os.path.join(model_path,'model_emotic1.pth')).to(device)\n models = [model_context, model_body, emotic_model]\n\n with open(images_list, 'r') as f:\n lines = f.readlines()\n \n for idx, line in enumerate(lines):\n image_context_path = line.split('\\n')[0].split(' ')[0]\n image_context = cv2.cvtColor(cv2.imread(image_context_path), cv2.COLOR_BGR2RGB)\n try:\n bbox_yolo = get_bbox(yolo, device, image_context)\n for pred_bbox in bbox_yolo:\n pred_cat, pred_cont = infer(context_norm, body_norm, ind2cat, ind2vad, device, thresholds, models, image_context=image_context, bbox=pred_bbox, to_print=False)\n write_text_vad = list()\n for continuous in pred_cont:\n write_text_vad.append(str('%.1f' %(continuous)))\n write_text_vad = 'vad ' + ' '.join(write_text_vad) \n image_context = cv2.rectangle(image_context, (pred_bbox[0], pred_bbox[1]),(pred_bbox[2] , pred_bbox[3]), (255, 0, 0), 3)\n cv2.putText(image_context, write_text_vad, (pred_bbox[0], pred_bbox[1] - 5), cv2.FONT_HERSHEY_PLAIN, 1, (255, 255, 255), 1)\n for i, emotion in enumerate(pred_cat):\n cv2.putText(image_context, emotion, (pred_bbox[0], pred_bbox[1] + (i+1)*12), cv2.FONT_HERSHEY_PLAIN, 1, (255, 255, 255), 1)\n except Exception as e:\n print ('Exception for image ',image_context_path)\n print (e)\n cv2.imwrite(os.path.join(result_path, 'img_%r.jpg' %(idx)), cv2.cvtColor(image_context, cv2.COLOR_RGB2BGR))\n print ('completed inference for image %d' %(idx))\n\n\ndef yolo_video(video_file, result_path, model_path, context_norm, body_norm, ind2cat, ind2vad, args):\n ''' Perform inference on a video. First yolo model is used to obtain bounding boxes of persons in every frame.\n After that the emotic model is used to obtain categoraical and continuous emotion predictions. \n :param video_file: Path of video file. \n :param result_path: Directory path to save the results (output video).\n :param model_path: Directory path to load models and val_thresholds to perform inference.\n :param context_norm: List containing mean and std values for context images. \n :param body_norm: List containing mean and std values for body images. \n :param ind2cat: Dictionary converting integer index to categorical emotion. \n :param ind2vad: Dictionary converting integer index to continuous emotion dimension (Valence, Arousal and Dominance).\n :param args: Runtime arguments.\n ''' \n device = torch.device(\"cuda:%s\" %(str(args.gpu)) if torch.cuda.is_available() else \"cpu\")\n yolo = prepare_yolo(model_path)\n yolo = yolo.to(device)\n yolo.eval()\n\n thresholds = torch.FloatTensor(np.load(os.path.join(result_path, 'val_thresholds.npy'))).to(device) \n model_context = torch.load(os.path.join(model_path,'model_context1.pth')).to(device)\n model_body = torch.load(os.path.join(model_path,'model_body1.pth')).to(device)\n emotic_model = torch.load(os.path.join(model_path,'model_emotic1.pth')).to(device)\n model_context.eval()\n model_body.eval()\n emotic_model.eval()\n models = [model_context, model_body, emotic_model]\n\n video_stream = cv2.VideoCapture(video_file)\n writer = None\n\n print ('Starting testing on video')\n while True:\n (grabbed, frame) = video_stream.read()\n if not grabbed:\n break\n image_context = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n\n try: \n bbox_yolo = get_bbox(yolo, device, image_context)\n for pred_idx, pred_bbox in enumerate(bbox_yolo):\n pred_cat, pred_cont = infer(context_norm, body_norm, ind2cat, ind2vad, device, thresholds, models, image_context=image_context, bbox=pred_bbox, to_print=False)\n write_text_vad = list()\n for continuous in pred_cont:\n write_text_vad.append(str('%.1f' %(continuous)))\n write_text_vad = 'vad ' + ' '.join(write_text_vad) \n image_context = cv2.rectangle(image_context, (pred_bbox[0], pred_bbox[1]),(pred_bbox[2] , pred_bbox[3]), (255, 0, 0), 3)\n cv2.putText(image_context, write_text_vad, (pred_bbox[0], pred_bbox[1] - 5), cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 255), 2)\n for i, emotion in enumerate(pred_cat):\n cv2.putText(image_context, emotion, (pred_bbox[0], pred_bbox[1] + (i+1)*12), cv2.FONT_HERSHEY_PLAIN, 1, (0, 0, 255), 2)\n except Exception:\n pass\n if writer is None:\n fourcc = cv2.VideoWriter_fourcc(*\"MJPG\")\n writer = cv2.VideoWriter(os.path.join(result_path, 'result_vid.avi'), fourcc, 30, (image_context.shape[1], image_context.shape[0]), True) \n writer.write(cv2.cvtColor(image_context, cv2.COLOR_RGB2BGR))\n writer.release()\n video_stream.release() \n print ('Completed video')\n\n\ndef check_paths(args):\n ''' Check (create if they don't exist) experiment directories.\n :param args: Runtime arguments as passed by the user.\n :return: result_dir_path, model_dir_path.\n '''\n if args.inference_file is not None: \n if not os.path.exists(args.inference_file):\n raise ValueError('inference file does not exist. Please pass a valid inference file')\n if args.video_file is not None: \n if not os.path.exists(args.video_file):\n raise ValueError('video file does not exist. Please pass a valid video file')\n if args.inference_file is None and args.video_file is None: \n raise ValueError(' both inference file and video file can\\'t be none. Please specify one and run again')\n model_path = os.path.join(args.experiment_path, args.model_dir)\n if not os.path.exists(model_path):\n raise ValueError('model path %s does not exist. Please pass a valid model_path' %(model_path))\n result_path = os.path.join(args.experiment_path, args.result_dir)\n if not os.path.exists(result_path):\n os.makedirs(result_path)\n return result_path, model_path\n\nif __name__=='__main__':\n args = parse_args()\n\n result_path, model_path = check_paths(args)\n\n cat = ['Affection', 'Anger', 'Annoyance', 'Anticipation', 'Aversion', 'Confidence', 'Disapproval', 'Disconnection', \\\n 'Disquietment', 'Doubt/Confusion', 'Embarrassment', 'Engagement', 'Esteem', 'Excitement', 'Fatigue', 'Fear','Happiness', \\\n 'Pain', 'Peace', 'Pleasure', 'Sadness', 'Sensitivity', 'Suffering', 'Surprise', 'Sympathy', 'Yearning']\n cat2ind = {}\n ind2cat = {}\n for idx, emotion in enumerate(cat):\n cat2ind[emotion] = idx\n ind2cat[idx] = emotion\n \n vad = ['Valence', 'Arousal', 'Dominance']\n ind2vad = {}\n for idx, continuous in enumerate(vad):\n ind2vad[idx] = continuous\n \n context_mean = [0.4690646, 0.4407227, 0.40508908]\n context_std = [0.2514227, 0.24312855, 0.24266963]\n body_mean = [0.43832874, 0.3964344, 0.3706214]\n body_std = [0.24784276, 0.23621225, 0.2323653]\n context_norm = [context_mean, context_std]\n body_norm = [body_mean, body_std]\n\n if args.inference_file is not None: \n print ('inference over inference file images')\n yolo_infer(args.inference_file, result_path, model_path, context_norm, body_norm, ind2cat, ind2vad, args)\n if args.video_file is not None:\n print ('inference over test video')\n yolo_video(args.video_file, result_path, model_path, context_norm, body_norm, ind2cat, ind2vad, args)\n"
},
{
"path": "yolo_utils.py",
"content": "import cv2\nimport numpy as np \nimport os \n\nimport torch \nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\ndef to_cpu(tensor):\n return tensor.detach().cpu()\n\ndef xywh2xyxy(x):\n ''' Convert bounding box from [x, y, w, h] to [x1, y1, x2, y2]\n :param x: bounding boxes array\n :return: Converted bounding box array \n '''\n y = x.new(x.shape)\n y[..., 0] = x[..., 0] - x[..., 2] / 2\n y[..., 1] = x[..., 1] - x[..., 3] / 2\n y[..., 2] = x[..., 0] + x[..., 2] / 2\n y[..., 3] = x[..., 1] + x[..., 3] / 2\n return y\n\ndef bbox_iou(box1, box2, x1y1x2y2=True):\n \"\"\"\n Returns the IoU of two bounding boxes\n \"\"\"\n if not x1y1x2y2:\n # Transform from center and width to exact coordinates\n b1_x1, b1_x2 = box1[:, 0] - box1[:, 2] / 2, box1[:, 0] + box1[:, 2] / 2\n b1_y1, b1_y2 = box1[:, 1] - box1[:, 3] / 2, box1[:, 1] + box1[:, 3] / 2\n b2_x1, b2_x2 = box2[:, 0] - box2[:, 2] / 2, box2[:, 0] + box2[:, 2] / 2\n b2_y1, b2_y2 = box2[:, 1] - box2[:, 3] / 2, box2[:, 1] + box2[:, 3] / 2\n else:\n # Get the coordinates of bounding boxes\n b1_x1, b1_y1, b1_x2, b1_y2 = box1[:, 0], box1[:, 1], box1[:, 2], box1[:, 3]\n b2_x1, b2_y1, b2_x2, b2_y2 = box2[:, 0], box2[:, 1], box2[:, 2], box2[:, 3]\n\n # get the corrdinates of the intersection rectangle\n inter_rect_x1 = torch.max(b1_x1, b2_x1)\n inter_rect_y1 = torch.max(b1_y1, b2_y1)\n inter_rect_x2 = torch.min(b1_x2, b2_x2)\n inter_rect_y2 = torch.min(b1_y2, b2_y2)\n # Intersection area\n inter_area = torch.clamp(inter_rect_x2 - inter_rect_x1 + 1, min=0) * torch.clamp(\n inter_rect_y2 - inter_rect_y1 + 1, min=0\n )\n # Union Area\n b1_area = (b1_x2 - b1_x1 + 1) * (b1_y2 - b1_y1 + 1)\n b2_area = (b2_x2 - b2_x1 + 1) * (b2_y2 - b2_y1 + 1)\n\n iou = inter_area / (b1_area + b2_area - inter_area + 1e-16)\n\n return iou\n\ndef rescale_boxes(boxes, current_dim, original_shape):\n \"\"\" Rescales bounding boxes to the original shape \"\"\"\n orig_h, orig_w = original_shape\n # The amount of padding that was added\n pad_x = max(orig_h - orig_w, 0) * (current_dim / max(original_shape))\n pad_y = max(orig_w - orig_h, 0) * (current_dim / max(original_shape))\n # Image height and width after padding is removed\n unpad_h = current_dim - pad_y\n unpad_w = current_dim - pad_x\n # Rescale bounding boxes to dimension of original image\n boxes[:, 0] = ((boxes[:, 0] - pad_x // 2) / unpad_w) * orig_w\n boxes[:, 1] = ((boxes[:, 1] - pad_y // 2) / unpad_h) * orig_h\n boxes[:, 2] = ((boxes[:, 2] - pad_x // 2) / unpad_w) * orig_w\n boxes[:, 3] = ((boxes[:, 3] - pad_y // 2) / unpad_h) * orig_h\n return boxes\n\ndef non_max_suppression(prediction, conf_thres=0.5, nms_thres=0.4):\n \"\"\"\n Removes detections with lower object confidence score than 'conf_thres' and performs\n Non-Maximum Suppression to further filter detections.\n Returns detections with shape:\n (x1, y1, x2, y2, object_conf, class_score, class_pred)\n \"\"\"\n\n # From (center x, center y, width, height) to (x1, y1, x2, y2)\n prediction[..., :4] = xywh2xyxy(prediction[..., :4])\n output = [None for _ in range(len(prediction))]\n for image_i, image_pred in enumerate(prediction):\n # Filter out confidence scores below threshold\n image_pred = image_pred[image_pred[:, 4] >= conf_thres]\n # If none are remaining => process next image\n if not image_pred.size(0):\n continue\n # Object confidence times class confidence\n score = image_pred[:, 4] * image_pred[:, 5:].max(1)[0]\n # Sort by it\n image_pred = image_pred[(-score).argsort()]\n class_confs, class_preds = image_pred[:, 5:].max(1, keepdim=True)\n detections = torch.cat((image_pred[:, :5], class_confs.float(), class_preds.float()), 1)\n # Perform non-maximum suppression\n keep_boxes = []\n while detections.size(0):\n large_overlap = bbox_iou(detections[0, :4].unsqueeze(0), detections[:, :4]) > nms_thres\n label_match = detections[0, -1] == detections[:, -1]\n # Indices of boxes with lower confidence scores, large IOUs and matching labels\n invalid = large_overlap & label_match\n weights = detections[invalid, 4:5]\n # Merge overlapping bboxes by order of confidence\n detections[0, :4] = (weights * detections[invalid, :4]).sum(0) / weights.sum()\n keep_boxes += [detections[0]]\n detections = detections[~invalid]\n if keep_boxes:\n output[image_i] = torch.stack(keep_boxes)\n\n return output\n\ndef parse_model_config(path):\n \"\"\"Parses the yolo-v3 layer configuration file and returns module definitions\"\"\"\n file = open(path, 'r')\n lines = file.read().split('\\n')\n lines = [x for x in lines if x and not x.startswith('#')]\n lines = [x.rstrip().lstrip() for x in lines] # get rid of fringe whitespaces\n module_defs = []\n for line in lines:\n if line.startswith('['): # This marks the start of a new block\n module_defs.append({})\n module_defs[-1]['type'] = line[1:-1].rstrip()\n if module_defs[-1]['type'] == 'convolutional':\n module_defs[-1]['batch_normalize'] = 0\n else:\n key, value = line.split(\"=\")\n value = value.strip()\n module_defs[-1][key.rstrip()] = value.strip()\n\n return module_defs\n\ndef parse_data_config(path):\n \"\"\"Parses the data configuration file\"\"\"\n options = dict()\n options['gpus'] = '0,1,2,3'\n options['num_workers'] = '10'\n with open(path, 'r') as fp:\n lines = fp.readlines()\n for line in lines:\n line = line.strip()\n if line == '' or line.startswith('#'):\n continue\n key, value = line.split('=')\n options[key.strip()] = value.strip()\n return options\n\ndef create_modules(module_defs):\n \"\"\"\n Constructs module list of layer blocks from module configuration in module_defs\n \"\"\"\n hyperparams = module_defs.pop(0)\n output_filters = [int(hyperparams[\"channels\"])]\n module_list = nn.ModuleList()\n for module_i, module_def in enumerate(module_defs):\n modules = nn.Sequential()\n\n if module_def[\"type\"] == \"convolutional\":\n bn = int(module_def[\"batch_normalize\"])\n filters = int(module_def[\"filters\"])\n kernel_size = int(module_def[\"size\"])\n pad = (kernel_size - 1) // 2\n modules.add_module(\n f\"conv_{module_i}\",\n nn.Conv2d(\n in_channels=output_filters[-1],\n out_channels=filters,\n kernel_size=kernel_size,\n stride=int(module_def[\"stride\"]),\n padding=pad,\n bias=not bn,\n ),\n )\n if bn:\n modules.add_module(f\"batch_norm_{module_i}\", nn.BatchNorm2d(filters, momentum=0.9, eps=1e-5))\n if module_def[\"activation\"] == \"leaky\":\n modules.add_module(f\"leaky_{module_i}\", nn.LeakyReLU(0.1))\n\n elif module_def[\"type\"] == \"maxpool\":\n kernel_size = int(module_def[\"size\"])\n stride = int(module_def[\"stride\"])\n if kernel_size == 2 and stride == 1:\n modules.add_module(f\"_debug_padding_{module_i}\", nn.ZeroPad2d((0, 1, 0, 1)))\n maxpool = nn.MaxPool2d(kernel_size=kernel_size, stride=stride, padding=int((kernel_size - 1) // 2))\n modules.add_module(f\"maxpool_{module_i}\", maxpool)\n\n elif module_def[\"type\"] == \"upsample\":\n upsample = Upsample(scale_factor=int(module_def[\"stride\"]), mode=\"nearest\")\n modules.add_module(f\"upsample_{module_i}\", upsample)\n\n elif module_def[\"type\"] == \"route\":\n layers = [int(x) for x in module_def[\"layers\"].split(\",\")]\n filters = sum([output_filters[1:][i] for i in layers])\n modules.add_module(f\"route_{module_i}\", EmptyLayer())\n\n elif module_def[\"type\"] == \"shortcut\":\n filters = output_filters[1:][int(module_def[\"from\"])]\n modules.add_module(f\"shortcut_{module_i}\", EmptyLayer())\n\n elif module_def[\"type\"] == \"yolo\":\n anchor_idxs = [int(x) for x in module_def[\"mask\"].split(\",\")]\n # Extract anchors\n anchors = [int(x) for x in module_def[\"anchors\"].split(\",\")]\n anchors = [(anchors[i], anchors[i + 1]) for i in range(0, len(anchors), 2)]\n anchors = [anchors[i] for i in anchor_idxs]\n num_classes = int(module_def[\"classes\"])\n img_size = int(hyperparams[\"height\"])\n # Define detection layer\n yolo_layer = YOLOLayer(anchors, num_classes, img_size)\n modules.add_module(f\"yolo_{module_i}\", yolo_layer)\n # Register module list and number of output filters\n module_list.append(modules)\n output_filters.append(filters)\n\n return hyperparams, module_list\n\nclass Upsample(nn.Module):\n \"\"\" nn.Upsample is deprecated \"\"\"\n\n def __init__(self, scale_factor, mode=\"nearest\"):\n super(Upsample, self).__init__()\n self.scale_factor = scale_factor\n self.mode = mode\n\n def forward(self, x):\n x = F.interpolate(x, scale_factor=self.scale_factor, mode=self.mode)\n return x\n\nclass EmptyLayer(nn.Module):\n \"\"\"Placeholder for 'route' and 'shortcut' layers\"\"\"\n\n def __init__(self):\n super(EmptyLayer, self).__init__()\n\nclass YOLOLayer(nn.Module):\n \"\"\"Detection layer\"\"\"\n\n def __init__(self, anchors, num_classes, img_dim=416):\n super(YOLOLayer, self).__init__()\n self.anchors = anchors\n self.num_anchors = len(anchors)\n self.num_classes = num_classes\n self.ignore_thres = 0.5\n self.mse_loss = nn.MSELoss()\n self.bce_loss = nn.BCELoss()\n self.obj_scale = 1\n self.noobj_scale = 100\n self.metrics = {}\n self.img_dim = img_dim\n self.grid_size = 0 # grid size\n\n def compute_grid_offsets(self, grid_size, cuda=True):\n self.grid_size = grid_size\n g = self.grid_size\n FloatTensor = torch.cuda.FloatTensor if cuda else torch.FloatTensor\n self.stride = self.img_dim / self.grid_size\n # Calculate offsets for each grid\n self.grid_x = torch.arange(g).repeat(g, 1).view([1, 1, g, g]).type(FloatTensor)\n self.grid_y = torch.arange(g).repeat(g, 1).t().view([1, 1, g, g]).type(FloatTensor)\n self.scaled_anchors = FloatTensor([(a_w / self.stride, a_h / self.stride) for a_w, a_h in self.anchors])\n self.anchor_w = self.scaled_anchors[:, 0:1].view((1, self.num_anchors, 1, 1))\n self.anchor_h = self.scaled_anchors[:, 1:2].view((1, self.num_anchors, 1, 1))\n\n def forward(self, x, targets=None, img_dim=None):\n\n # Tensors for cuda support\n FloatTensor = torch.cuda.FloatTensor if x.is_cuda else torch.FloatTensor\n LongTensor = torch.cuda.LongTensor if x.is_cuda else torch.LongTensor\n ByteTensor = torch.cuda.ByteTensor if x.is_cuda else torch.ByteTensor\n\n self.img_dim = img_dim\n num_samples = x.size(0)\n grid_size = x.size(2)\n\n prediction = (\n x.view(num_samples, self.num_anchors, self.num_classes + 5, grid_size, grid_size)\n .permute(0, 1, 3, 4, 2)\n .contiguous()\n )\n\n # Get outputs\n x = torch.sigmoid(prediction[..., 0]) # Center x\n y = torch.sigmoid(prediction[..., 1]) # Center y\n w = prediction[..., 2] # Width\n h = prediction[..., 3] # Height\n pred_conf = torch.sigmoid(prediction[..., 4]) # Conf\n pred_cls = torch.sigmoid(prediction[..., 5:]) # Cls pred.\n\n # If grid size does not match current we compute new offsets\n if grid_size != self.grid_size:\n self.compute_grid_offsets(grid_size, cuda=x.is_cuda)\n\n # Add offset and scale with anchors\n pred_boxes = FloatTensor(prediction[..., :4].shape)\n pred_boxes[..., 0] = x.data + self.grid_x\n pred_boxes[..., 1] = y.data + self.grid_y\n pred_boxes[..., 2] = torch.exp(w.data) * self.anchor_w\n pred_boxes[..., 3] = torch.exp(h.data) * self.anchor_h\n\n output = torch.cat(\n (\n pred_boxes.view(num_samples, -1, 4) * self.stride,\n pred_conf.view(num_samples, -1, 1),\n pred_cls.view(num_samples, -1, self.num_classes),\n ),\n -1,\n )\n\n if targets is None:\n return output, 0\n else:\n iou_scores, class_mask, obj_mask, noobj_mask, tx, ty, tw, th, tcls, tconf = build_targets(\n pred_boxes=pred_boxes,\n pred_cls=pred_cls,\n target=targets,\n anchors=self.scaled_anchors,\n ignore_thres=self.ignore_thres,\n )\n\n # Loss : Mask outputs to ignore non-existing objects (except with conf. loss)\n loss_x = self.mse_loss(x[obj_mask], tx[obj_mask])\n loss_y = self.mse_loss(y[obj_mask], ty[obj_mask])\n loss_w = self.mse_loss(w[obj_mask], tw[obj_mask])\n loss_h = self.mse_loss(h[obj_mask], th[obj_mask])\n loss_conf_obj = self.bce_loss(pred_conf[obj_mask], tconf[obj_mask])\n loss_conf_noobj = self.bce_loss(pred_conf[noobj_mask], tconf[noobj_mask])\n loss_conf = self.obj_scale * loss_conf_obj + self.noobj_scale * loss_conf_noobj\n loss_cls = self.bce_loss(pred_cls[obj_mask], tcls[obj_mask])\n total_loss = loss_x + loss_y + loss_w + loss_h + loss_conf + loss_cls\n\n # Metrics\n cls_acc = 100 * class_mask[obj_mask].mean()\n conf_obj = pred_conf[obj_mask].mean()\n conf_noobj = pred_conf[noobj_mask].mean()\n conf50 = (pred_conf > 0.5).float()\n iou50 = (iou_scores > 0.5).float()\n iou75 = (iou_scores > 0.75).float()\n detected_mask = conf50 * class_mask * tconf\n precision = torch.sum(iou50 * detected_mask) / (conf50.sum() + 1e-16)\n recall50 = torch.sum(iou50 * detected_mask) / (obj_mask.sum() + 1e-16)\n recall75 = torch.sum(iou75 * detected_mask) / (obj_mask.sum() + 1e-16)\n\n self.metrics = {\n \"loss\": to_cpu(total_loss).item(),\n \"x\": to_cpu(loss_x).item(),\n \"y\": to_cpu(loss_y).item(),\n \"w\": to_cpu(loss_w).item(),\n \"h\": to_cpu(loss_h).item(),\n \"conf\": to_cpu(loss_conf).item(),\n \"cls\": to_cpu(loss_cls).item(),\n \"cls_acc\": to_cpu(cls_acc).item(),\n \"recall50\": to_cpu(recall50).item(),\n \"recall75\": to_cpu(recall75).item(),\n \"precision\": to_cpu(precision).item(),\n \"conf_obj\": to_cpu(conf_obj).item(),\n \"conf_noobj\": to_cpu(conf_noobj).item(),\n \"grid_size\": grid_size,\n }\n\n return output, total_loss\n\nclass Darknet(nn.Module):\n \"\"\"YOLOv3 object detection model\"\"\"\n\n def __init__(self, config_path, img_size=416):\n super(Darknet, self).__init__()\n self.module_defs = parse_model_config(config_path)\n self.hyperparams, self.module_list = create_modules(self.module_defs)\n self.yolo_layers = [layer[0] for layer in self.module_list if hasattr(layer[0], \"metrics\")]\n self.img_size = img_size\n self.seen = 0\n self.header_info = np.array([0, 0, 0, self.seen, 0], dtype=np.int32)\n\n def forward(self, x, targets=None):\n img_dim = x.shape[2]\n loss = 0\n layer_outputs, yolo_outputs = [], []\n for i, (module_def, module) in enumerate(zip(self.module_defs, self.module_list)):\n if module_def[\"type\"] in [\"convolutional\", \"upsample\", \"maxpool\"]:\n x = module(x)\n elif module_def[\"type\"] == \"route\":\n x = torch.cat([layer_outputs[int(layer_i)] for layer_i in module_def[\"layers\"].split(\",\")], 1)\n elif module_def[\"type\"] == \"shortcut\":\n layer_i = int(module_def[\"from\"])\n x = layer_outputs[-1] + layer_outputs[layer_i]\n elif module_def[\"type\"] == \"yolo\":\n x, layer_loss = module[0](x, targets, img_dim)\n loss += layer_loss\n yolo_outputs.append(x)\n layer_outputs.append(x)\n yolo_outputs = to_cpu(torch.cat(yolo_outputs, 1))\n return yolo_outputs if targets is None else (loss, yolo_outputs)\n\n def load_darknet_weights(self, weights_path):\n \"\"\"Parses and loads the weights stored in 'weights_path'\"\"\"\n\n # Open the weights file\n with open(weights_path, \"rb\") as f:\n header = np.fromfile(f, dtype=np.int32, count=5) # First five are header values\n self.header_info = header # Needed to write header when saving weights\n self.seen = header[3] # number of images seen during training\n weights = np.fromfile(f, dtype=np.float32) # The rest are weights\n\n # Establish cutoff for loading backbone weights\n cutoff = None\n if \"darknet53.conv.74\" in weights_path:\n cutoff = 75\n\n ptr = 0\n for i, (module_def, module) in enumerate(zip(self.module_defs, self.module_list)):\n if i == cutoff:\n break\n if module_def[\"type\"] == \"convolutional\":\n conv_layer = module[0]\n if module_def[\"batch_normalize\"]:\n # Load BN bias, weights, running mean and running variance\n bn_layer = module[1]\n num_b = bn_layer.bias.numel() # Number of biases\n # Bias\n bn_b = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.bias)\n bn_layer.bias.data.copy_(bn_b)\n ptr += num_b\n # Weight\n bn_w = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.weight)\n bn_layer.weight.data.copy_(bn_w)\n ptr += num_b\n # Running Mean\n bn_rm = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.running_mean)\n bn_layer.running_mean.data.copy_(bn_rm)\n ptr += num_b\n # Running Var\n bn_rv = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(bn_layer.running_var)\n bn_layer.running_var.data.copy_(bn_rv)\n ptr += num_b\n else:\n # Load conv. bias\n num_b = conv_layer.bias.numel()\n conv_b = torch.from_numpy(weights[ptr : ptr + num_b]).view_as(conv_layer.bias)\n conv_layer.bias.data.copy_(conv_b)\n ptr += num_b\n # Load conv. weights\n num_w = conv_layer.weight.numel()\n conv_w = torch.from_numpy(weights[ptr : ptr + num_w]).view_as(conv_layer.weight)\n conv_layer.weight.data.copy_(conv_w)\n ptr += num_w\n\n def save_darknet_weights(self, path, cutoff=-1):\n \"\"\"\n @:param path - path of the new weights file\n @:param cutoff - save layers between 0 and cutoff (cutoff = -1 -> all are saved)\n \"\"\"\n fp = open(path, \"wb\")\n self.header_info[3] = self.seen\n self.header_info.tofile(fp)\n\n # Iterate through layers\n for i, (module_def, module) in enumerate(zip(self.module_defs[:cutoff], self.module_list[:cutoff])):\n if module_def[\"type\"] == \"convolutional\":\n conv_layer = module[0]\n # If batch norm, load bn first\n if module_def[\"batch_normalize\"]:\n bn_layer = module[1]\n bn_layer.bias.data.cpu().numpy().tofile(fp)\n bn_layer.weight.data.cpu().numpy().tofile(fp)\n bn_layer.running_mean.data.cpu().numpy().tofile(fp)\n bn_layer.running_var.data.cpu().numpy().tofile(fp)\n # Load conv bias\n else:\n conv_layer.bias.data.cpu().numpy().tofile(fp)\n # Load conv weights\n conv_layer.weight.data.cpu().numpy().tofile(fp)\n\n fp.close()\n\ndef prepare_yolo(model_dir):\n ''' Download yolo model files and load the model weights\n :param model_dir: Directory path where to store yolo model weights and yolo model configuration file.\n :return: Yolo model after loading model weights\n '''\n cfg_file = os.path.join(model_dir, 'yolov3.cfg')\n if not os.path.exists(cfg_file):\n download_command = 'wget https://raw.githubusercontent.com/pjreddie/darknet/master/cfg/yolov3.cfg -O ' + cfg_file\n os.system(download_command)\n weight_file = os.path.join(model_dir, 'yolov3.weights')\n if not os.path.exists(weight_file):\n download_command = 'wget https://pjreddie.com/media/files/yolov3.weights -O ' + weight_file\n os.system(download_command)\n \n yolo_model = Darknet(cfg_file, 416)\n yolo_model.load_darknet_weights(weight_file)\n print ('prepared yolo model')\n return yolo_model\n"
}
]