[ { "path": ".gitignore", "content": "*.pyc\n*.swp*\n" }, { "path": "PythonEvaluationTools/vqaEvalDemo.py", "content": "# coding: utf-8\n\nimport sys\ndataDir = '../../VQA'\nsys.path.insert(0, '%s/PythonHelperTools/vqaTools' %(dataDir))\nfrom vqa import VQA\nfrom vqaEvaluation.vqaEval import VQAEval\nimport matplotlib.pyplot as plt\nimport skimage.io as io\nimport json\nimport random\nimport os\n\n# set up file names and paths\nversionType ='v2_' # this should be '' when using VQA v2.0 dataset\ntaskType ='OpenEnded' # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0\ndataType ='mscoco' # 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. \ndataSubType ='train2014'\nannFile ='%s/Annotations/%s%s_%s_annotations.json'%(dataDir, versionType, dataType, dataSubType)\nquesFile ='%s/Questions/%s%s_%s_%s_questions.json'%(dataDir, versionType, taskType, dataType, dataSubType)\nimgDir ='%s/Images/%s/%s/' %(dataDir, dataType, dataSubType)\nresultType ='fake'\nfileTypes = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] \n\n# An example result json file has been provided in './Results' folder. \n\n[resFile, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = ['%s/Results/%s%s_%s_%s_%s_%s.json'%(dataDir, versionType, taskType, dataType, dataSubType, \\\nresultType, fileType) for fileType in fileTypes] \n\n# create vqa object and vqaRes object\nvqa = VQA(annFile, quesFile)\nvqaRes = vqa.loadRes(resFile, quesFile)\n\n# create vqaEval object by taking vqa and vqaRes\nvqaEval = VQAEval(vqa, vqaRes, n=2) #n is precision of accuracy (number of places after decimal), default is 2\n\n# evaluate results\n\"\"\"\nIf you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function\nBy default it uses all the question ids in annotation file\n\"\"\"\nvqaEval.evaluate() \n\n# print accuracies\nprint \"\\n\"\nprint \"Overall Accuracy is: %.02f\\n\" %(vqaEval.accuracy['overall'])\nprint \"Per Question Type Accuracy is the following:\"\nfor quesType in vqaEval.accuracy['perQuestionType']:\n\tprint \"%s : %.02f\" %(quesType, vqaEval.accuracy['perQuestionType'][quesType])\nprint \"\\n\"\nprint \"Per Answer Type Accuracy is the following:\"\nfor ansType in vqaEval.accuracy['perAnswerType']:\n\tprint \"%s : %.02f\" %(ansType, vqaEval.accuracy['perAnswerType'][ansType])\nprint \"\\n\"\n# demo how to use evalQA to retrieve low score result\nevals = [quesId for quesId in vqaEval.evalQA if vqaEval.evalQA[quesId]<35] #35 is per question percentage accuracy\nif len(evals) > 0:\n\tprint 'ground truth answers'\n\trandomEval = random.choice(evals)\n\trandomAnn = vqa.loadQA(randomEval)\n\tvqa.showQA(randomAnn)\n\n\tprint '\\n'\n\tprint 'generated answer (accuracy %.02f)'%(vqaEval.evalQA[randomEval])\n\tann = vqaRes.loadQA(randomEval)[0]\n\tprint \"Answer: %s\\n\" %(ann['answer'])\n\n\timgId = randomAnn[0]['image_id']\n\timgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg'\n\tif os.path.isfile(imgDir + imgFilename):\n\t\tI = io.imread(imgDir + imgFilename)\n\t\tplt.imshow(I)\n\t\tplt.axis('off')\n\t\tplt.show()\n\n# plot accuracy for various question types\nplt.bar(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].values(), align='center')\nplt.xticks(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].keys(), rotation='0',fontsize=10)\nplt.title('Per Question Type Accuracy', fontsize=10)\nplt.xlabel('Question Types', fontsize=10)\nplt.ylabel('Accuracy', fontsize=10)\nplt.show()\n\n# save evaluation results to ./Results folder\njson.dump(vqaEval.accuracy, open(accuracyFile, 'w'))\njson.dump(vqaEval.evalQA, open(evalQAFile, 'w'))\njson.dump(vqaEval.evalQuesType, open(evalQuesTypeFile, 'w'))\njson.dump(vqaEval.evalAnsType, open(evalAnsTypeFile, 'w'))\n\n" }, { "path": "PythonEvaluationTools/vqaEvaluation/__init__.py", "content": "author='aagrawal'\n" }, { "path": "PythonEvaluationTools/vqaEvaluation/vqaEval.py", "content": "# coding=utf-8\n\n__author__='aagrawal'\n\nimport re\n# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link:\n# (https://github.com/tylin/coco-caption/blob/master/pycocoevalcap/eval.py).\nimport sys\n\n\nclass VQAEval:\n\tdef __init__(self, vqa, vqaRes, n=2):\n\t\tself.n \t\t\t = n\n\t\tself.accuracy = {}\n\t\tself.evalQA = {}\n\t\tself.evalQuesType = {}\n\t\tself.evalAnsType = {}\n\t\tself.vqa \t\t = vqa\n\t\tself.vqaRes = vqaRes\n\t\tself.params\t\t = {'question_id': vqa.getQuesIds()}\n\t\tself.contractions = {\"aint\": \"ain't\", \"arent\": \"aren't\", \"cant\": \"can't\", \"couldve\": \"could've\", \"couldnt\": \"couldn't\", \\\n\t\t\t\t\t\t\t \"couldn'tve\": \"couldn't've\", \"couldnt've\": \"couldn't've\", \"didnt\": \"didn't\", \"doesnt\": \"doesn't\", \"dont\": \"don't\", \"hadnt\": \"hadn't\", \\\n\t\t\t\t\t\t\t \"hadnt've\": \"hadn't've\", \"hadn'tve\": \"hadn't've\", \"hasnt\": \"hasn't\", \"havent\": \"haven't\", \"hed\": \"he'd\", \"hed've\": \"he'd've\", \\\n\t\t\t\t\t\t\t \"he'dve\": \"he'd've\", \"hes\": \"he's\", \"howd\": \"how'd\", \"howll\": \"how'll\", \"hows\": \"how's\", \"Id've\": \"I'd've\", \"I'dve\": \"I'd've\", \\\n\t\t\t\t\t\t\t \"Im\": \"I'm\", \"Ive\": \"I've\", \"isnt\": \"isn't\", \"itd\": \"it'd\", \"itd've\": \"it'd've\", \"it'dve\": \"it'd've\", \"itll\": \"it'll\", \"let's\": \"let's\", \\\n\t\t\t\t\t\t\t \"maam\": \"ma'am\", \"mightnt\": \"mightn't\", \"mightnt've\": \"mightn't've\", \"mightn'tve\": \"mightn't've\", \"mightve\": \"might've\", \\\n\t\t\t\t\t\t\t \"mustnt\": \"mustn't\", \"mustve\": \"must've\", \"neednt\": \"needn't\", \"notve\": \"not've\", \"oclock\": \"o'clock\", \"oughtnt\": \"oughtn't\", \\\n\t\t\t\t\t\t\t \"ow's'at\": \"'ow's'at\", \"'ows'at\": \"'ow's'at\", \"'ow'sat\": \"'ow's'at\", \"shant\": \"shan't\", \"shed've\": \"she'd've\", \"she'dve\": \"she'd've\", \\\n\t\t\t\t\t\t\t \"she's\": \"she's\", \"shouldve\": \"should've\", \"shouldnt\": \"shouldn't\", \"shouldnt've\": \"shouldn't've\", \"shouldn'tve\": \"shouldn't've\", \\\n\t\t\t\t\t\t\t \"somebody'd\": \"somebodyd\", \"somebodyd've\": \"somebody'd've\", \"somebody'dve\": \"somebody'd've\", \"somebodyll\": \"somebody'll\", \\\n\t\t\t\t\t\t\t \"somebodys\": \"somebody's\", \"someoned\": \"someone'd\", \"someoned've\": \"someone'd've\", \"someone'dve\": \"someone'd've\", \\\n\t\t\t\t\t\t\t \"someonell\": \"someone'll\", \"someones\": \"someone's\", \"somethingd\": \"something'd\", \"somethingd've\": \"something'd've\", \\\n\t\t\t\t\t\t\t \"something'dve\": \"something'd've\", \"somethingll\": \"something'll\", \"thats\": \"that's\", \"thered\": \"there'd\", \"thered've\": \"there'd've\", \\\n\t\t\t\t\t\t\t \"there'dve\": \"there'd've\", \"therere\": \"there're\", \"theres\": \"there's\", \"theyd\": \"they'd\", \"theyd've\": \"they'd've\", \\\n\t\t\t\t\t\t\t \"they'dve\": \"they'd've\", \"theyll\": \"they'll\", \"theyre\": \"they're\", \"theyve\": \"they've\", \"twas\": \"'twas\", \"wasnt\": \"wasn't\", \\\n\t\t\t\t\t\t\t \"wed've\": \"we'd've\", \"we'dve\": \"we'd've\", \"weve\": \"we've\", \"werent\": \"weren't\", \"whatll\": \"what'll\", \"whatre\": \"what're\", \\\n\t\t\t\t\t\t\t \"whats\": \"what's\", \"whatve\": \"what've\", \"whens\": \"when's\", \"whered\": \"where'd\", \"wheres\": \"where's\", \"whereve\": \"where've\", \\\n\t\t\t\t\t\t\t \"whod\": \"who'd\", \"whod've\": \"who'd've\", \"who'dve\": \"who'd've\", \"wholl\": \"who'll\", \"whos\": \"who's\", \"whove\": \"who've\", \"whyll\": \"why'll\", \\\n\t\t\t\t\t\t\t \"whyre\": \"why're\", \"whys\": \"why's\", \"wont\": \"won't\", \"wouldve\": \"would've\", \"wouldnt\": \"wouldn't\", \"wouldnt've\": \"wouldn't've\", \\\n\t\t\t\t\t\t\t \"wouldn'tve\": \"wouldn't've\", \"yall\": \"y'all\", \"yall'll\": \"y'all'll\", \"y'allll\": \"y'all'll\", \"yall'd've\": \"y'all'd've\", \\\n\t\t\t\t\t\t\t \"y'alld've\": \"y'all'd've\", \"y'all'dve\": \"y'all'd've\", \"youd\": \"you'd\", \"youd've\": \"you'd've\", \"you'dve\": \"you'd've\", \\\n\t\t\t\t\t\t\t \"youll\": \"you'll\", \"youre\": \"you're\", \"youve\": \"you've\"}\n\t\tself.manualMap = { 'none': '0',\n\t\t\t\t\t\t\t 'zero': '0',\n\t\t\t\t\t\t\t 'one': '1',\n\t\t\t\t\t\t\t 'two': '2',\n\t\t\t\t\t\t\t 'three': '3',\n\t\t\t\t\t\t\t 'four': '4',\n\t\t\t\t\t\t\t 'five': '5',\n\t\t\t\t\t\t\t 'six': '6',\n\t\t\t\t\t\t\t 'seven': '7',\n\t\t\t\t\t\t\t 'eight': '8',\n\t\t\t\t\t\t\t 'nine': '9',\n\t\t\t\t\t\t\t 'ten': '10'\n\t\t\t\t\t\t\t}\n\t\tself.articles = ['a',\n\t\t\t\t\t\t\t 'an',\n\t\t\t\t\t\t\t 'the'\n\t\t\t\t\t\t\t]\n\n\n\t\tself.periodStrip = re.compile(\"(?!<=\\d)(\\.)(?!\\d)\")\n\t\tself.commaStrip = re.compile(\"(\\d)(\\,)(\\d)\")\n\t\tself.punct = [';', r\"/\", '[', ']', '\"', '{', '}',\n\t\t\t\t\t\t\t '(', ')', '=', '+', '\\\\', '_', '-',\n\t\t\t\t\t\t\t '>', '<', '@', '`', ',', '?', '!']\n\n\n\tdef evaluate(self, quesIds=None):\n\t\tif quesIds == None:\n\t\t\tquesIds = [quesId for quesId in self.params['question_id']]\n\t\tgts = {}\n\t\tres = {}\n\t\tfor quesId in quesIds:\n\t\t\tgts[quesId] = self.vqa.qa[quesId]\n\t\t\tres[quesId] = self.vqaRes.qa[quesId]\n\n\t\t# =================================================\n\t\t# Compute accuracy\n\t\t# =================================================\n\t\taccQA = []\n\t\taccQuesType = {}\n\t\taccAnsType = {}\n\t\tprint \"computing accuracy\"\n\t\tstep = 0\n\t\tfor quesId in quesIds:\n\t\t\tfor ansDic in gts[quesId]['answers']:\n\t\t\t\tansDic['answer'] = ansDic['answer'].replace('\\n', ' ')\n\t\t\t\tansDic['answer'] = ansDic['answer'].replace('\\t', ' ')\n\t\t\t\tansDic['answer'] = ansDic['answer'].strip()\n\t\t\tresAns = res[quesId]['answer']\n\t\t\tresAns = resAns.replace('\\n', ' ')\n\t\t\tresAns = resAns.replace('\\t', ' ')\n\t\t\tresAns = resAns.strip()\n\t\t\tgtAcc = []\n\t\t\tgtAnswers = [ans['answer'] for ans in gts[quesId]['answers']]\n\n\t\t\tif len(set(gtAnswers)) > 1:\n\t\t\t\tfor ansDic in gts[quesId]['answers']:\n\t\t\t\t\tansDic['answer'] = self.processPunctuation(ansDic['answer'])\n\t\t\t\t\tansDic['answer'] = self.processDigitArticle(ansDic['answer'])\n\t\t\t\tresAns = self.processPunctuation(resAns)\n\t\t\t\tresAns = self.processDigitArticle(resAns)\n\n\t\t\tfor gtAnsDatum in gts[quesId]['answers']:\n\t\t\t\totherGTAns = [item for item in gts[quesId]['answers'] if item!=gtAnsDatum]\n\t\t\t\tmatchingAns = [item for item in otherGTAns if item['answer']==resAns]\n\t\t\t\tacc = min(1, float(len(matchingAns))/3)\n\t\t\t\tgtAcc.append(acc)\n\t\t\tquesType = gts[quesId]['question_type']\n\t\t\tansType = gts[quesId]['answer_type']\n\t\t\tavgGTAcc = float(sum(gtAcc))/len(gtAcc)\n\t\t\taccQA.append(avgGTAcc)\n\t\t\tif quesType not in accQuesType:\n\t\t\t\taccQuesType[quesType] = []\n\t\t\taccQuesType[quesType].append(avgGTAcc)\n\t\t\tif ansType not in accAnsType:\n\t\t\t\taccAnsType[ansType] = []\n\t\t\taccAnsType[ansType].append(avgGTAcc)\n\t\t\tself.setEvalQA(quesId, avgGTAcc)\n\t\t\tself.setEvalQuesType(quesId, quesType, avgGTAcc)\n\t\t\tself.setEvalAnsType(quesId, ansType, avgGTAcc)\n\t\t\tif step%100 == 0:\n\t\t\t\tself.updateProgress(step/float(len(quesIds)))\n\t\t\tstep = step + 1\n\n\t\tself.setAccuracy(accQA, accQuesType, accAnsType)\n\t\tprint \"Done computing accuracy\"\n\n\tdef processPunctuation(self, inText):\n\t\toutText = inText\n\t\tfor p in self.punct:\n\t\t\tif (p + ' ' in inText or ' ' + p in inText) or (re.search(self.commaStrip, inText) != None):\n\t\t\t\toutText = outText.replace(p, '')\n\t\t\telse:\n\t\t\t\toutText = outText.replace(p, ' ')\n\t\toutText = self.periodStrip.sub(\"\",\n\t\t\t\t\t\t\t\t\t outText,\n\t\t\t\t\t\t\t\t\t re.UNICODE)\n\t\treturn outText\n\n\tdef processDigitArticle(self, inText):\n\t\toutText = []\n\t\ttempText = inText.lower().split()\n\t\tfor word in tempText:\n\t\t\tword = self.manualMap.setdefault(word, word)\n\t\t\tif word not in self.articles:\n\t\t\t\toutText.append(word)\n\t\t\telse:\n\t\t\t\tpass\n\t\tfor wordId, word in enumerate(outText):\n\t\t\tif word in self.contractions:\n\t\t\t\toutText[wordId] = self.contractions[word]\n\t\toutText = ' '.join(outText)\n\t\treturn outText\n\n\tdef setAccuracy(self, accQA, accQuesType, accAnsType):\n\t\tself.accuracy['overall'] = round(100*float(sum(accQA))/len(accQA), self.n)\n\t\tself.accuracy['perQuestionType'] = {quesType: round(100*float(sum(accQuesType[quesType]))/len(accQuesType[quesType]), self.n) for quesType in accQuesType}\n\t\tself.accuracy['perAnswerType'] = {ansType: round(100*float(sum(accAnsType[ansType]))/len(accAnsType[ansType]), self.n) for ansType in accAnsType}\n\n\tdef setEvalQA(self, quesId, acc):\n\t\tself.evalQA[quesId] = round(100*acc, self.n)\n\n\tdef setEvalQuesType(self, quesId, quesType, acc):\n\t\tif quesType not in self.evalQuesType:\n\t\t\tself.evalQuesType[quesType] = {}\n\t\tself.evalQuesType[quesType][quesId] = round(100*acc, self.n)\n\n\tdef setEvalAnsType(self, quesId, ansType, acc):\n\t\tif ansType not in self.evalAnsType:\n\t\t\tself.evalAnsType[ansType] = {}\n\t\tself.evalAnsType[ansType][quesId] = round(100*acc, self.n)\n\n\tdef updateProgress(self, progress):\n\t\tbarLength = 20\n\t\tstatus = \"\"\n\t\tif isinstance(progress, int):\n\t\t\tprogress = float(progress)\n\t\tif not isinstance(progress, float):\n\t\t\tprogress = 0\n\t\t\tstatus = \"error: progress var must be float\\r\\n\"\n\t\tif progress < 0:\n\t\t\tprogress = 0\n\t\t\tstatus = \"Halt...\\r\\n\"\n\t\tif progress >= 1:\n\t\t\tprogress = 1\n\t\t\tstatus = \"Done...\\r\\n\"\n\t\tblock = int(round(barLength*progress))\n\t\ttext = \"\\rFinshed Percent: [{0}] {1}% {2}\".format( \"#\"*block + \"-\"*(barLength-block), int(progress*100), status)\n\t\tsys.stdout.write(text)\n\t\tsys.stdout.flush()\n" }, { "path": "PythonHelperTools/vqaDemo.py", "content": "# coding: utf-8\n\nfrom vqaTools.vqa import VQA\nimport random\nimport skimage.io as io\nimport matplotlib.pyplot as plt\nimport os\n\ndataDir\t\t='../../VQA'\nversionType ='v2_' # this should be '' when using VQA v2.0 dataset\ntaskType ='OpenEnded' # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0\ndataType ='mscoco' # 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0.\ndataSubType ='train2014'\nannFile ='%s/Annotations/%s%s_%s_annotations.json'%(dataDir, versionType, dataType, dataSubType)\nquesFile ='%s/Questions/%s%s_%s_%s_questions.json'%(dataDir, versionType, taskType, dataType, dataSubType)\nimgDir \t\t= '%s/Images/%s/%s/' %(dataDir, dataType, dataSubType)\n\n# initialize VQA api for QA annotations\nvqa=VQA(annFile, quesFile)\n\n# load and display QA annotations for given question types\n\"\"\"\nAll possible quesTypes for abstract and mscoco has been provided in respective text files in ../QuestionTypes/ folder.\n\"\"\"\nannIds = vqa.getQuesIds(quesTypes='how many'); \nanns = vqa.loadQA(annIds)\nrandomAnn = random.choice(anns)\nvqa.showQA([randomAnn])\nimgId = randomAnn['image_id']\nimgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg'\nif os.path.isfile(imgDir + imgFilename):\n\tI = io.imread(imgDir + imgFilename)\n\tplt.imshow(I)\n\tplt.axis('off')\n\tplt.show()\n\n# load and display QA annotations for given answer types\n\"\"\"\nansTypes can be one of the following\nyes/no\nnumber\nother\n\"\"\"\nannIds = vqa.getQuesIds(ansTypes='yes/no'); \nanns = vqa.loadQA(annIds)\nrandomAnn = random.choice(anns)\nvqa.showQA([randomAnn])\nimgId = randomAnn['image_id']\nimgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg'\nif os.path.isfile(imgDir + imgFilename):\n\tI = io.imread(imgDir + imgFilename)\n\tplt.imshow(I)\n\tplt.axis('off')\n\tplt.show()\n\n# load and display QA annotations for given images\n\"\"\"\nUsage: vqa.getImgIds(quesIds=[], quesTypes=[], ansTypes=[])\nAbove method can be used to retrieve imageIds for given question Ids or given question types or given answer types.\n\"\"\"\nids = vqa.getImgIds()\nannIds = vqa.getQuesIds(imgIds=random.sample(ids,5)); \nanns = vqa.loadQA(annIds)\nrandomAnn = random.choice(anns)\nvqa.showQA([randomAnn]) \nimgId = randomAnn['image_id']\nimgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg'\nif os.path.isfile(imgDir + imgFilename):\n\tI = io.imread(imgDir + imgFilename)\n\tplt.imshow(I)\n\tplt.axis('off')\n\tplt.show()\n\n" }, { "path": "PythonHelperTools/vqaTools/__init__.py", "content": "__author__ = 'aagrawal'\n" }, { "path": "PythonHelperTools/vqaTools/vqa.py", "content": "__author__ = 'aagrawal'\n__version__ = '0.9'\n\n# Interface for accessing the VQA dataset.\n\n# This code is based on the code written by Tsung-Yi Lin for MSCOCO Python API available at the following link: \n# (https://github.com/pdollar/coco/blob/master/PythonAPI/pycocotools/coco.py).\n\n# The following functions are defined:\n# VQA - VQA class that loads VQA annotation file and prepares data structures.\n# getQuesIds - Get question ids that satisfy given filter conditions.\n# getImgIds - Get image ids that satisfy given filter conditions.\n# loadQA - Load questions and answers with the specified question ids.\n# showQA - Display the specified questions and answers.\n# loadRes - Load result file and create result object.\n\n# Help on each function can be accessed by: \"help(COCO.function)\"\n\nimport json\nimport datetime\nimport copy\n\nclass VQA:\n\tdef __init__(self, annotation_file=None, question_file=None):\n\t\t\"\"\"\n \tConstructor of VQA helper class for reading and visualizing questions and answers.\n :param annotation_file (str): location of VQA annotation file\n :return:\n\t\t\"\"\"\n # load dataset\n\t\tself.dataset = {}\n\t\tself.questions = {}\n\t\tself.qa = {}\n\t\tself.qqa = {}\n\t\tself.imgToQA = {}\n\t\tif not annotation_file == None and not question_file == None:\n\t\t\tprint 'loading VQA annotations and questions into memory...'\n\t\t\ttime_t = datetime.datetime.utcnow()\n\t\t\tdataset = json.load(open(annotation_file, 'r'))\n\t\t\tquestions = json.load(open(question_file, 'r'))\n\t\t\tprint datetime.datetime.utcnow() - time_t\n\t\t\tself.dataset = dataset\n\t\t\tself.questions = questions\n\t\t\tself.createIndex()\n\n\tdef createIndex(self):\n # create index\n\t\tprint 'creating index...'\n \t\timgToQA = {ann['image_id']: [] for ann in self.dataset['annotations']}\n\t\tqa = {ann['question_id']: [] for ann in self.dataset['annotations']}\n\t\tqqa = {ann['question_id']: [] for ann in self.dataset['annotations']}\n \t\tfor ann in self.dataset['annotations']:\n\t\t\timgToQA[ann['image_id']] += [ann]\n\t\t\tqa[ann['question_id']] = ann\n\t\tfor ques in self.questions['questions']:\n \t\t\tqqa[ques['question_id']] = ques\n\t\tprint 'index created!'\n\n \t\t# create class members\n \t\tself.qa = qa\n\t\tself.qqa = qqa\n \t\tself.imgToQA = imgToQA\n\n\tdef info(self):\n\t\t\"\"\"\n\t\tPrint information about the VQA annotation file.\n\t\t:return:\n\t\t\"\"\"\n\t\tfor key, value in self.datset['info'].items():\n\t\t\tprint '%s: %s'%(key, value)\n\n\tdef getQuesIds(self, imgIds=[], quesTypes=[], ansTypes=[]):\n\t\t\"\"\"\n\t\tGet question ids that satisfy given filter conditions. default skips that filter\n\t\t:param \timgIds (int array) : get question ids for given imgs\n\t\t\t\tquesTypes (str array) : get question ids for given question types\n\t\t\t\tansTypes (str array) : get question ids for given answer types\n\t\t:return: ids (int array) : integer array of question ids\n\t\t\"\"\"\n\t\timgIds \t = imgIds if type(imgIds) == list else [imgIds]\n\t\tquesTypes = quesTypes if type(quesTypes) == list else [quesTypes]\n\t\tansTypes = ansTypes if type(ansTypes) == list else [ansTypes]\n\n\t\tif len(imgIds) == len(quesTypes) == len(ansTypes) == 0:\n\t\t\tanns = self.dataset['annotations']\n\t\telse:\n\t\t\tif not len(imgIds) == 0:\n\t\t\t\tanns = sum([self.imgToQA[imgId] for imgId in imgIds if imgId in self.imgToQA],[])\n\t\t\telse:\n \t\t\t\tanns = self.dataset['annotations']\n\t\t\tanns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes]\n \t\t\tanns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes]\n\t\tids = [ann['question_id'] for ann in anns]\n\t\treturn ids\n\n\tdef getImgIds(self, quesIds=[], quesTypes=[], ansTypes=[]):\n\t\t\"\"\"\n\t\tGet image ids that satisfy given filter conditions. default skips that filter\n\t\t:param quesIds (int array) : get image ids for given question ids\n quesTypes (str array) : get image ids for given question types\n ansTypes (str array) : get image ids for given answer types\n\t\t:return: ids (int array) : integer array of image ids\n\t\t\"\"\"\n\t\tquesIds = quesIds if type(quesIds) == list else [quesIds]\n\t\tquesTypes = quesTypes if type(quesTypes) == list else [quesTypes]\n\t\tansTypes = ansTypes if type(ansTypes) == list else [ansTypes]\n\n\t\tif len(quesIds) == len(quesTypes) == len(ansTypes) == 0:\n\t\t\tanns = self.dataset['annotations']\n\t\telse:\n\t\t\tif not len(quesIds) == 0:\n\t\t\t\tanns = sum([self.qa[quesId] for quesId in quesIds if quesId in self.qa],[])\n\t\t\telse:\n\t\t\t\tanns = self.dataset['annotations']\n\t\t\tanns = anns if len(quesTypes) == 0 else [ann for ann in anns if ann['question_type'] in quesTypes]\n\t\t\tanns = anns if len(ansTypes) == 0 else [ann for ann in anns if ann['answer_type'] in ansTypes]\n\t\tids = [ann['image_id'] for ann in anns]\n\t\treturn ids\n\n\tdef loadQA(self, ids=[]):\n\t\t\"\"\"\n\t\tLoad questions and answers with the specified question ids.\n\t\t:param ids (int array) : integer ids specifying question ids\n\t\t:return: qa (object array) : loaded qa objects\n\t\t\"\"\"\n\t\tif type(ids) == list:\n\t\t\treturn [self.qa[id] for id in ids]\n\t\telif type(ids) == int:\n\t\t\treturn [self.qa[ids]]\n\n\tdef showQA(self, anns):\n\t\t\"\"\"\n\t\tDisplay the specified annotations.\n\t\t:param anns (array of object): annotations to display\n\t\t:return: None\n\t\t\"\"\"\n\t\tif len(anns) == 0:\n\t\t\treturn 0\n\t\tfor ann in anns:\n\t\t\tquesId = ann['question_id']\n\t\t\tprint \"Question: %s\" %(self.qqa[quesId]['question'])\n\t\t\tfor ans in ann['answers']:\n\t\t\t\tprint \"Answer %d: %s\" %(ans['answer_id'], ans['answer'])\n\t\t\n\tdef loadRes(self, resFile, quesFile):\n\t\t\"\"\"\n\t\tLoad result file and return a result object.\n\t\t:param resFile (str) : file name of result file\n\t\t:return: res (obj) : result api object\n\t\t\"\"\"\n\t\tres = VQA()\n\t\tres.questions = json.load(open(quesFile))\n\t\tres.dataset['info'] = copy.deepcopy(self.questions['info'])\n\t\tres.dataset['task_type'] = copy.deepcopy(self.questions['task_type'])\n\t\tres.dataset['data_type'] = copy.deepcopy(self.questions['data_type'])\n\t\tres.dataset['data_subtype'] = copy.deepcopy(self.questions['data_subtype'])\n\t\tres.dataset['license'] = copy.deepcopy(self.questions['license'])\n\n\t\tprint 'Loading and preparing results... '\n\t\ttime_t = datetime.datetime.utcnow()\n\t\tanns = json.load(open(resFile))\n\t\tassert type(anns) == list, 'results is not an array of objects'\n\t\tannsQuesIds = [ann['question_id'] for ann in anns]\n\t\tassert set(annsQuesIds) == set(self.getQuesIds()), \\\n\t\t'Results do not correspond to current VQA set. Either the results do not have predictions for all question ids in annotation file or there is atleast one question id that does not belong to the question ids in the annotation file.'\n\t\tfor ann in anns:\n\t\t\tquesId \t\t\t = ann['question_id']\n\t\t\tif res.dataset['task_type'] == 'Multiple Choice':\n\t\t\t\tassert ann['answer'] in self.qqa[quesId]['multiple_choices'], 'predicted answer is not one of the multiple choices'\n\t\t\tqaAnn = self.qa[quesId]\n\t\t\tann['image_id'] = qaAnn['image_id'] \n\t\t\tann['question_type'] = qaAnn['question_type']\n\t\t\tann['answer_type'] = qaAnn['answer_type']\n\t\tprint 'DONE (t=%0.2fs)'%((datetime.datetime.utcnow() - time_t).total_seconds())\n\n\t\tres.dataset['annotations'] = anns\n\t\tres.createIndex()\n\t\treturn res\n" }, { "path": "QuestionTypes/abstract_v002_question_types.txt", "content": "how many\nwhat color is the\nis the\nwhere is the\nwhat\nwhat is\nare the\nwhat is the\nis there a\ndoes the\nis the woman\nis the man\nwhat is on the\nis it\nis the girl\nis the boy\nis the dog\nare they\nwho is\nwhat kind of\nwhat color are the\nwhat is in the\nwhat is the man\nis there\nwhat is the woman\nwhat are the\nwhat is the boy\nare there\nwhat is the girl\nis this\nhow\nwhich\nhow many people are\nis the cat\nwhy is the\nare\nwill the\nwhat type of\nwhat is the dog\ndo\nis she\ndoes\ndo the\nis\nis the baby\nare there any\nis the lady\ncan\nwhat animal is\nwhere are the\nis the sun\nwhat are they\ndid the\nwhat is the cat\nwhat is the lady\nhow many clouds are\nis that\nis the little girl\nis he\nare these\nhow many trees are\nhow many pillows\nare the people\nwhy\nis the young\nhow many windows are\nis this a\nwhat is the little\nis the tv\nhow many animals are\nwho\nhow many pictures\nhow many plants are\nhow many birds are\nwhat color is\nwhat is the baby\nis anyone\nwhat color\nhow many bushes\nis the old man\nnone of the above\n" }, { "path": "QuestionTypes/mscoco_question_types.txt", "content": "how many\nis the\nwhat\nwhat color is the\nwhat is the\nis this\nis this a\nwhat is\nare the\nwhat kind of\nis there a\nwhat type of\nis it\nwhat are the\nwhere is the\nis there\ndoes the\nwhat color are the\nare these\nare there\nwhich\nis\nwhat is the man\nis the man\nare\nhow\ndoes this\nwhat is on the\nwhat does the\nhow many people are\nwhat is in the\nwhat is this\ndo\nwhat are\nare they\nwhat time\nwhat sport is\nare there any\nis he\nwhat color is\nwhy\nwhere are the\nwhat color\nwho is\nwhat animal is\nis the woman\nis this an\ndo you\nhow many people are in\nwhat room is\nhas\nis this person\nwhat is the woman\ncan you\nwhy is the\nis the person\nwhat is the color of the\nwhat is the person\ncould\nwas\nis that a\nwhat number is\nwhat is the name\nwhat brand\nnone of the above\n" }, { "path": "README.md", "content": "Python API and Evaluation Code for v2.0 and v1.0 releases of the VQA dataset.\n===================\n## VQA v2.0 release ##\nThis release consists of\n- Real \n\t- 82,783 MS COCO training images, 40,504 MS COCO validation images and 81,434 MS COCO testing images (images are obtained from [MS COCO website] (http://mscoco.org/dataset/#download))\n\t- 443,757 questions for training, 214,354 questions for validation and 447,793 questions for testing\n\t- 4,437,570 answers for training and 2,143,540 answers for validation (10 per question)\n\nThere is only one type of task\n- Open-ended task\n\n## VQA v1.0 release ##\nThis release consists of\n- Real \n\t- 82,783 MS COCO training images, 40,504 MS COCO validation images and 81,434 MS COCO testing images (images are obtained from [MS COCO website] (http://mscoco.org/dataset/#download))\n\t- 248,349 questions for training, 121,512 questions for validation and 244,302 questions for testing (3 per image)\n\t- 2,483,490 answers for training and 1,215,120 answers for validation (10 per question)\n- Abstract\n\t- 20,000 training images, 10,000 validation images and 20,000 MS COCO testing images\n\t- 60,000 questions for training, 30,000 questions for validation and 60,000 questions for testing (3 per image)\n\t- 600,000 answers for training and 300,000 answers for validation (10 per question)\n\nThere are two types of tasks\n- Open-ended task\n- Multiple-choice task (18 choices per question)\n\n## Requirements ##\n- python 2.7\n- scikit-image (visit [this page](http://scikit-image.org/docs/dev/install.html) for installation)\n- matplotlib (visit [this page](http://matplotlib.org/users/installing.html) for installation)\n\n## Files ##\n./Questions\n- For v2.0, download the question files from the [VQA download page](http://www.visualqa.org/download.html), extract them and place in this folder.\n- For v1.0, both real and abstract, question files can be found on the [VQA v1 download page](http://www.visualqa.org/vqa_v1_download.html).\n- Question files from Beta v0.9 release (123,287 MSCOCO train and val images, 369,861 questions, 3,698,610 answers) can be found below\n\t- [training question files](http://visualqa.org/data/mscoco/prev_rel/Beta_v0.9/Questions_Train_mscoco.zip)\n\t- [validation question files](http://visualqa.org/data/mscoco/prev_rel/Beta_v0.9/Questions_Val_mscoco.zip)\n- Question files from Beta v0.1 release (10k MSCOCO images, 30k questions, 300k answers) can be found [here](http://visualqa.org/data/mscoco/prev_rel/Beta_v0.1/Questions_Train_mscoco.zip).\n\n./Annotations\n- For v2.0, download the annotations files from the [VQA download page](http://www.visualqa.org/download.html), extract them and place in this folder.\n- For v1.0, for both real and abstract, annotation files can be found on the [VQA v1 download page](http://www.visualqa.org/vqa_v1_download.html).\n- Annotation files from Beta v0.9 release (123,287 MSCOCO train and val images, 369,861 questions, 3,698,610 answers) can be found below\n\t- [training annotation files](http://visualqa.org/data/mscoco/prev_rel/Beta_v0.9/Annotations_Train_mscoco.zip)\n\t- [validation annotation files](http://visualqa.org/data/mscoco/prev_rel/Beta_v0.9/Annotations_Val_mscoco.zip)\n- Annotation files from Beta v0.1 release (10k MSCOCO images, 30k questions, 300k answers) can be found [here](http://visualqa.org/data/mscoco/prev_rel/Beta_v0.1/Annotations_Train_mscoco.zip).\n\n./Images\n- For real, create a directory with name mscoco inside this directory. For each of train, val and test, create directories with names train2014, val2014 and test2015 respectively inside mscoco directory, download respective images from [MS COCO website](http://mscoco.org/dataset/#download) and place them in respective folders.\n- For abstract, create a directory with name abstract_v002 inside this directory. For each of train, val and test, create directories with names train2015, val2015 and test2015 respectively inside abstract_v002 directory, download respective images from [VQA download page](http://www.visualqa.org/download.html) and place them in respective folders.\n\n./PythonHelperTools\n- This directory contains the Python API to read and visualize the VQA dataset\n- vqaDemo.py (demo script)\n- vqaTools (API to read and visualize data)\n\n./PythonEvaluationTools\n- This directory contains the Python evaluation code\n- vqaEvalDemo.py (evaluation demo script)\n- vqaEvaluation (evaluation code)\n\n./Results\n- OpenEnded_mscoco_train2014_fake_results.json (an example of a fake results file for v1.0 to run the demo)\n- Visit [VQA evaluation page] (http://visualqa.org/evaluation) for more details.\n\n./QuestionTypes\n- This directory contains the following lists of question types for both real and abstract questions (question types are unchanged from v1.0 to v2.0). In a list, if there are question types of length n+k and length n with the same first n words, then the question type of length n does not include questions that belong to the question type of length n+k.\n- mscoco_question_types.txt\n- abstract_v002_question_types.txt\n\n## References ##\n- [VQA: Visual Question Answering](http://visualqa.org/)\n- [Microsoft COCO](http://mscoco.org/)\n\n## Developers ##\n- Aishwarya Agrawal (Virginia Tech)\n- Code for API is based on [MSCOCO API code](https://github.com/pdollar/coco).\n- The format of the code for evaluation is based on [MSCOCO evaluation code](https://github.com/tylin/coco-caption).\n" }, { "path": "license.txt", "content": "Copyright (c) 2014, Aishwarya Agrawal\nAll rights reserved.\n\nRedistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions are met: \n\n1. Redistributions of source code must retain the above copyright notice, this\n list of conditions and the following disclaimer. \n2. Redistributions in binary form must reproduce the above copyright notice,\n this list of conditions and the following disclaimer in the documentation\n and/or other materials provided with the distribution. \n\nTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\"\nAND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\nWARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\nDISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE\nFOR\nANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\nLOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\nON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\nSOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n\nThe views and conclusions contained in the software and documentation are\nthose\nof the authors and should not be interpreted as representing official\npolicies, \neither expressed or implied, of the FreeBSD Project.\n" } ]