\"\n\t\t\t\t\t\t\telif type(row[colid]).__name__ != 'NoneType':\n\t\t\t\t\t\t\t\tmessage = row[colid].encode(\"utf-8\")\n\t\t\t\t\t\t\ttry:\n\t\t\t\t\t\t\t\tmessage = message.replace('\\n', ' ') # Python 2\n\t\t\t\t\t\t\texcept:\n\t\t\t\t\t\t\t\tmessage = message.decode().replace('\\n', ' ') # Python 3\n\t\t\t\t\t\t\toutput += \"| {message:{fill}{align}{width}}\".format(message=message[:size[colid]], fill=\" \", align='<', width=size[colid])\n\t\t\t\t\t\toutput += \"|\\n\"\n\t\t\t\t\toutput += \"-\" * self.settings['output_width'] + \"\\n\"\n\t\treturn output\n\n\tdef print_text_bottom_row(self):\n\t\t\"\"\"Print the last bottom row of a txt output\"\"\"\n\t\treturn \"\\n\"\n\n\tdef print_csv_top_row(self, columns):\n\t\t\"\"\"Print the first row of the csv table (and then print_csv_row will be used)\"\"\"\n\t\toutput = \"\"\n\t\tif isinstance(columns, list):\n\t\t\toutput = \",\".join(columns) + \"\\n\"\n\t\treturn output\n\n\tdef print_csv_row(self, results):\n\t\t\"\"\"Print a row of the table (previously print_text_top_row was used and finally print_text_bottom_row will be used used)\"\"\"\n\t\toutput = \"\"\n\t\tif isinstance(results, list):\n\t\t\tfor result in results:\n\t\t\t\tfor row in result:\n\t\t\t\t\tfor colid in range(0, len(row)):\n\t\t\t\t\t\tif type(row[colid]).__name__ in ['int', 'NoneType']:\n\t\t\t\t\t\t\tmessage = str(row[colid])\n\t\t\t\t\t\telse:\n\t\t\t\t\t\t\tmessage = (row[colid]).encode(\"utf-8\")\n\t\t\t\t\t\tif colid != 0:\n\t\t\t\t\t\t\toutput += \",\"\n\t\t\t\t\t\ttry:\n\t\t\t\t\t\t\toutput += message # Python 2\n\t\t\t\t\t\texcept:\n\t\t\t\t\t\t\toutput += message.decode() # Python 3\n\t\t\t\t\toutput += \"\\n\"\n\t\treturn output\n\n\tdef print_xml_row(self, title, column, results):\n\t\t\"\"\"Print a row of the table (previously print_text_top_row was used and finally print_text_bottom_row will be used used)\"\"\"\n\t\toutput = \"\"\n\t\tif isinstance(title, str) and isinstance(column, list) and isinstance(results, list):\n\t\t\toutput = \"\\t<\" + \"\".join(ch for ch in title if ch.isalnum()) + \">\\n\"\n\t\t\tfor result in results:\n\t\t\t\tfor row in result:\n\t\t\t\t\tcolumn_id = 0\n\t\t\t\t\tfor item in row:\n\t\t\t\t\t\toutput += \"\\t\\t<\" + str(compat.escape(column[column_id])) + \">\" + str(compat.escape(item)) + \"\\n\"\n\t\t\t\t\t\tcolumn_id += 1\n\t\t\t\t\toutput += \"\\n\"\n\t\t\toutput += \"\\t\\n\"\n\t\treturn output\n\n\tdef print_html_top_row(self, title, columns):\n\t\t\"\"\"Print the first row of the HTML table (and then print_html_row will be used)\"\"\"\n\t\toutput = \"\"\n\t\tif isinstance(title, str) and isinstance(columns, list):\n\t\t\toutput = \"\"\"\n\t\t\t\n\t\t\t\t\"\"\"\n\t\t\tfor column in columns:\n\t\t\t\toutput += \"| \" + str(compat.escape(column)) + \" | \"\n\t\t\toutput += \"
\\n\"\n\t\treturn output\n\n\tdef print_html_row(self, results):\n\t\t\"\"\"Print a row of the table (previously print_html_top_row was used and finally print_html_bottom_row will be used used)\"\"\"\n\t\toutput = \"\"\n\t\tif isinstance(results, list):\n\t\t\tcont = 1\n\t\t\tfor result in results:\n\t\t\t\tif cont % 2 == 0:\n\t\t\t\t\ttrclass = \" class='gray'\"\n\t\t\t\telse:\n\t\t\t\t\ttrclass = \"\"\n\t\t\t\tfor row in result:\n\t\t\t\t\toutput += \" \"\n\t\t\t\t\tfor item in row:\n\t\t\t\t\t\toutput += \"\" + str(compat.escape(str(item)).encode('ascii', 'xmlcharrefreplace')) + \" | \"\n\t\t\t\t\toutput += \"\\n\"\n\t\t\t\tcont += 1\n\t\treturn output\n\n\tdef print_html_bottom_row(self, title):\n\t\t\"\"\"Print the first row of the HTML table (and then print_html_row will be used)\"\"\"\n\t\toutput = \"
\\n\"\n\t\tif isinstance(title, str) and title.find(\"Analyze\") != -1 and self.toggle_table:\n\t\t\toutput += \"\\n\"\n\t\treturn output\n\n\tdef set_toggle_table(self, toggle):\n\t\t\"\"\"Set a boolean flag to activate/deactivate if a table will be shown in HTML\"\"\"\n\t\tself.toggle_table = bool(toggle)\n\n\tdef pre_general(self, output):\n\t\t\"\"\"Print any previous code or perform tasks required before printing any table\"\"\"\n\t\tcontents = \"\"\n\n\t\tif output == \"xml\":\n\t\t\tcontents = \"\\n\"\n\t\telif output == \"html\":\n\t\t\tcontents = \"\"\"\n\t\n\t\t\n\t\t\tFuzzer Results for \"\"\" + str(compat.escape(self.settings['db_file'])) + \"\"\"\n\t\t\t\n\t\t\t\n\t\t\t\n\t\t\n\t\t\"\"\"\n\t\tif \"output_file\" in self.settings:\n\t\t\tself.write_file(self.settings['output_file'], 'w+', contents)\n\t\telse:\n\t\t\tprint(contents)\n\n\tdef post_general(self, output):\n\t\t\"\"\"Print any post code required before wrapping up\"\"\"\n\t\tcontents = \"\"\n\n\t\tif output == \"xml\":\n\t\t\tcontents = \"\"\n\t\telif output == \"html\":\n\t\t\tcontents = \" \\n\"\n\n\t\tif \"output_file\" in self.settings:\n\t\t\tself.write_file(self.settings['output_file'], 'a+', contents)\n\t\telse:\n\t\t\tprint(contents)\n\n\tdef general(self, output, title, columns, rows):\n\t\t\"\"\"Main function to dump stuff: from here, you can export in different formats (txt, csv, xml, html) to the screen or files\"\"\"\n\t\tif not rows:\n\t\t\treturn\n\t\tcontents = \"\"\n\t\ttitle = title + \" (\" + str(len(rows)) + \" rows)\"\n\n\t\tif output is None:\n\t\t\treturn\n\t\telif output == \"txt\":\n\t\t\tcontents = self.print_text_top_row(title, columns)\n\t\t\tcontents += self.print_text_row(columns, rows)\n\t\t\tcontents += self.print_text_bottom_row()\n\t\telif output == \"csv\":\n\t\t\tcontents = self.print_csv_top_row(columns)\n\t\t\tcontents += self.print_csv_row(rows)\n\t\telif output == \"xml\":\n\t\t\tcontents += self.print_xml_row(title, columns, rows)\n\t\telif output == \"html\":\n\t\t\tcontents += self.print_html_top_row(title, columns)\n\t\t\tcontents += self.print_html_row(rows)\n\t\t\tcontents += self.print_html_bottom_row(title)\n\t\telse:\n\t\t\tself.settings['logger'].error(\"Incorrect output selected\")\n\n\t\tif output in [\"txt\", \"csv\", \"xml\", \"html\"] and contents:\n\t\t\tif \"output_file\" in self.settings and self.settings['output_file'] is not None:\n\t\t\t\tself.write_file(self.settings['output_file'], 'a+', contents)\n\t\t\telse:\n\t\t\t\tprint(contents)\n\n\tdef write_file(self, output_file, mode, content):\n\t\t\"\"\"Write the content into a file\"\"\"\n\t\tif content:\n\t\t\ttry:\n\t\t\t\ttarget = open(output_file, mode)\n\t\t\t\ttarget.write(content)\n\t\t\t\ttarget.close()\n\t\t\texcept:\n\t\t\t\tself.settings['logger'].error(\"Could not write in file '%s'.\", output_file)\n"
},
{
"path": "classes/execute.py",
"content": "#\n# Copyright (C) 2018 Fernando Arnaboldi\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n#\nimport os\nimport signal\nimport subprocess\nimport threading\nimport time\nimport compat\n\n\nclass Execute(object):\n\t\"\"\"Thread being executed by Fuzzer\"\"\"\n\tdef __init__(self, settings, piece, testcase):\n\t\tself.kill_status = None\n\t\tself.settings = settings\n\t\tself.results = {}\n\t\tself.t = threading.Thread(target=self.run_subprocess, args=(piece, testcase))\n\t\tself.t.start()\n\t\tself.deleteme = testcase['data']\n\n\tdef join(self):\n\t\t\"\"\"Join the results to the thread\"\"\"\n\t\ttry:\n\t\t\tself.t.join()\n\t\texcept:\n\t\t\tpass\n\n\tdef get_output(self):\n\t\t\"\"\"Delete the file as part of getting the output\"\"\"\n\t\tif self.deleteme and os.path.isfile(self.deleteme[0]['datafile'][1]):\n\t\t\tos.remove(self.deleteme[0]['datafile'][1])\n\t\treturn self.results\n\n\tdef kill_process(self, process):\n\t\t\"\"\"After the defined timeout, try to kill the process\"\"\"\n\t\tself.kill_status = self.settings['kill_status']['requested']\n\t\tif process.poll() is None: # don't send the signal unless it seems it is necessary\n\t\t\ttry:\n\t\t\t\t# Unix\n\t\t\t\tos.killpg(os.getpgid(process.pid), signal.SIGTERM)\n\t\t\t\t# Windows/Unix\n\t\t\t\t# process.kill()\n\t\t\t\tself.kill_status = self.settings['kill_status']['killed']\n\t\t\texcept OSError: # ignore\n\t\t\t\tself.kill_status = self.settings['kill_status']['not_killed']\n\t\tself.settings['logger'].debug(\"Killed process status: %s\" % str(self.kill_status))\n\n\tdef run_subprocess(self, piece, testcase):\n\t\t\"\"\"Obtain the stdout and stderr when executing a piece of software using subprocess\"\"\"\n\t\tself.settings['logger'].debug(\"Input received: \" + str(testcase))\n\t\tstdout = stderr = elapsed = returncode = \"\"\n\t\tself.kill_status = self.settings['kill_status']['not_killed']\n\t\tstart_test = time.time()\n\t\tif \"execute\" in piece:\n\t\t\ttry:\n\t\t\t\tif 'stdin' in testcase:\n\t\t\t\t\t# Unix\n\t\t\t\t\tp = subprocess.Popen(testcase['execute'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid)\n\t\t\t\t\t# Windows/Unix\n\t\t\t\t\t# p = subprocess.Popen(testcase['execute'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\t\t\t\telse:\n\t\t\t\t\t# Unix\n\t\t\t\t\tp = subprocess.Popen(testcase['execute'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=os.setsid)\n\t\t\t\t\t# Windows/Unix\n\t\t\t\t\t# p = subprocess.Popen(testcase['execute'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n\t\t\t\tt = threading.Timer(self.settings['timeout'], self.kill_process, [p])\n\t\t\t\tt.start()\n\t\t\t\tif 'stdin' in testcase:\n\t\t\t\t\tstdout, stderr = p.communicate(input=testcase['stdin'])\n\t\t\t\telse:\n\t\t\t\t\tstdout, stderr = p.communicate()\n\t\t\t\tt.cancel()\n\t\t\t\treturncode = p.returncode\n\t\t\t\tstdout = compat.unicode(stdout.strip(), errors='ignore')\n\t\t\t\tstderr = compat.unicode(stderr.strip(), errors='ignore')\n\t\t\t\tstdout, stderr = self.analyze_results(stdout, stderr)\n\t\t\texcept OSError:\n\t\t\t\tstderr = \"Exception: OSErrorException\"\n\t\t\texcept KeyboardInterrupt:\n\t\t\t\tstderr = \"Exception: KeyboardInterruptException\"\n\t\t\texcept Exception as e:\n\t\t\t\tstderr = \"Exception: \" + str(e)\n\t\telapsed = str(round(time.time() - start_test, 4))\n\t\tself.results = {\"softwareid\": piece['softwareid'], \"testcaseid\": testcase['testcaseid'], \"stdout\": stdout, \"stderr\": stderr, \"network\": None, \"returncode\": returncode, \"elapsed\": elapsed, \"kill_status\": self.kill_status}\n\t\tself.settings['logger'].debug(\"Output produced: \" + str(self.results))\n\n\tdef analyze_results(self, stdout, stderr):\n\t\t\"\"\"Save full results for certain specific special strings\"\"\"\n\t\tif 'soft_bypass' in self.settings:\n\t\t\tfull = False\n\t\t\tif any([x in stdout for x in self.settings['soft_bypass']]):\n\t\t\t\tfull = True\n\t\t\telif any([x in stderr for x in self.settings['soft_bypass']]):\n\t\t\t\tfull = True\n\t\t\tif not full:\n\t\t\t\tstdout = stdout[:self.settings['soft_limit']]\n\t\t\t\tstderr = stderr[:self.settings['soft_limit']]\n\t\tif 'hard_limit' in self.settings:\n\t\t\tstdout = stdout[:self.settings['hard_limit']]\n\t\t\tstderr = stderr[:self.settings['hard_limit']]\n\t\tif 'hard_limit_lines' in self.settings:\n\t\t\tstdout = stdout.split(\"\\n\")[0]\n\t\treturn stdout, stderr\n"
},
{
"path": "classes/fuzzer.py",
"content": "#\n# Copyright (C) 2018 Fernando Arnaboldi\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n#\nimport os\nimport random\nimport string\nimport subprocess\nimport sys\nimport tempfile\nimport compat\nfrom distutils.spawn import find_executable\nfrom .execute import Execute\n\n\nclass Fuzzer(object):\n\t\"\"\"Executes fuzzing threads\"\"\"\n\tdef __init__(self, settings, ids):\n\t\tself.settings = settings\n\t\tself.ids = ids\n\n\tdef chdir_tmp(self):\n\t\t\"\"\"Change to the temporary directory\"\"\"\n\t\tstatus = False\n\t\ttry:\n\t\t\tos.chdir(self.settings['tmp_dir'])\t\t# it is safer to operate somewhere else\n\t\t\tstatus = True\n\t\texcept Exception as e:\n\t\t\tself.settings['logger'].error(\"It wasn't possible to change to the ram disk directory (%s). Instructions to mount it: %s\\nError: %s\" % (self.settings['tmp_dir'], self.settings['tmp_dir_howto'], e))\n\t\treturn status\n\n\tdef fuzz(self, tests):\n\t\t\"\"\"Executes something in all the different pieces of software\"\"\"\n\t\tprocess = []\t\t# info to be return and saved in the database\n\t\t# go through each test\n\t\tfor test in tests:\n\t\t\tfor piece in self.settings['software']:\n\t\t\t\tinput = self.get_input(piece, test)\n\t\t\t\ttry:\n\t\t\t\t\tprocess.append(Execute(self.settings, piece, input))\n\t\t\t\texcept Exception:\n\t\t\t\t\tself.settings['logger'].critical(\"Error when trying to append a new process, try using less parallel threads. Just in case, check also if there are too many processes running in the background.\")\n\t\t\t\t\tsys.exit()\n\t\tfor x in range(0, len(process)):\n\t\t\tprocess[x].join()\n\t\tfor x in range(0, len(process)):\n\t\t\tprocess[x] = process[x].get_output()\n\t\t# save the network results\n\t\tif self.ids:\n\t\t\tfor x in range(0, len(self.ids)):\n\t\t\t\tfor z in range(0, len(process)):\n\t\t\t\t\tif process[z]['testcaseid'] == self.ids[x][0] and process[z]['softwareid'] == self.ids[x][1]:\n\t\t\t\t\t\tprocess[z]['network'] = self.ids[x][2]\n\t\t\t\t\t\tif self.ids[x][3] != None:\n\t\t\t\t\t\t\tprocess[z]['stdout'] = self.ids[x][3]\n\t\t\t\t\t\tif self.ids[x][4] != None:\n\t\t\t\t\t\t\tprocess[z]['elapsed'] = self.ids[x][4]\n\t\t\t\t\t\tif self.ids[x][5] != None:\n\t\t\t\t\t\t\tprocess[z]['stderr'] = self.ids[x][5]\n\t\t\t\t\t\tbreak\n\t\t\tself.ids = []\n\t\tself.settings['logger'].debug(\"Process: %s\" % str(process))\n\t\treturn process\n\n\tdef get_input(self, piece, test):\n\t\t\"\"\"Based on how the type, suffix and fuzzdata that were defined in the piece of software,\n\t\tcreate a valid input file, url or as part of the CLI for the test\"\"\"\n\t\tinput = {}\n\t\tinput['testcaseid'] = test[0]\n\t\tinput['execute'] = []\n\t\tinput['data'] = []\n\t\t# default values\n\t\tdata = \"\"\n\t\ttypeid = 0\n\t\tfor arg in piece['execute']:\n\t\t\tif arg.startswith(\"-fuzzdata=\"):\n\t\t\t\trandomstring = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase) for _ in range(10))\n\t\t\t\tdata = compat.unicode(arg[len(\"-fuzzdata=\"):])\n\t\t\t\tdata = data.replace(\"[[test]]\", test[1])\n\t\t\t\tdata = data.replace(\"canaryhost\", self.settings['canaryhost'])\n\t\t\t\tdata = data.replace(\"[[softwareid]]\", str(piece['softwareid']))\n\t\t\t\tdata = data.replace(\"[[randomstring]]\", randomstring)\n\t\t\t\tdata = data.replace(\"[[testcaseid]]\", str(input['testcaseid']))\n\t\t\t\tinput_type = piece['type'][typeid].lower()\n\t\t\t\tif input_type in ['file', 'url']:\n\t\t\t\t\tif 'suffix' not in piece:\n\t\t\t\t\t\tpiece['suffix'] = []\n\t\t\t\t\t\tfor suffixid in xrange(0, len(piece['type'])):\n\t\t\t\t\t\t\tpiece['suffix'].append(\"\")\n\t\t\t\t\tif 'filename' in piece and piece['filename'][0]:\n\t\t\t\t\t\tfileid = os.open(piece['filename'][typeid], os.O_RDWR|os.O_CREAT)\n\t\t\t\t\t\tdatafile = []\n\t\t\t\t\t\tdatafile.append(fileid)\n\t\t\t\t\t\tdatafile.append(piece['filename'][typeid])\n\t\t\t\t\telse:\n\t\t\t\t\t\tdatafile = tempfile.mkstemp(suffix=piece['suffix'][typeid], prefix=self.settings['tmp_prefix'] + str(test[0]) + \"_\", dir=self.settings['tmp_dir'])\n\t\t\t\t\tinput['data'].append({\"data\": data, \"datafile\": datafile})\n\t\t\t\t\tif input_type == \"file\":\n\t\t\t\t\t\tinput['execute'].append(datafile[1])\n\t\t\t\t\telif input_type == \"url\":\n\t\t\t\t\t\tinput['execute'].append(\"http://\" + self.settings['canaryhost'] + \"/\" + os.path.basename(datafile[1]))\n\t\t\t\telif input_type == 'stdin':\n\t\t\t\t\tinput['stdin'] = data\n\t\t\t\telse:\n\t\t\t\t\tinput['execute'].append(data) # cli\n\t\t\t\ttypeid += 1\n\t\t\telse:\n\t\t\t\tinput['execute'].append(arg)\n\t\tfor id in xrange(0, len(input['data'])):\n\t\t\tfor id2 in xrange(0, len(input['data'])):\n\t\t\t\tinput['data'][id]['data'] = input['data'][id]['data'].replace(\"[[file\" + str(id2) + \"]]\", os.path.basename(input['data'][id2]['datafile'][1]))\n\t\t\t\tif 'canaryhost' in self.settings:\n\t\t\t\t\tinput['data'][id]['data'] = input['data'][id]['data'].replace(\"[[url\" + str(id2) + \"]]\", \"http://\" + self.settings['canaryhost'] + \"/\" + os.path.basename(input['data'][id2]['datafile'][1]))\n\t\t\tos.write(input['data'][id]['datafile'][0], input['data'][id]['data'].encode('utf8'))\n\t\t\tos.close(input['data'][id]['datafile'][0])\n\t\treturn input\n\n\tdef generate_tests(self, latest_id, limit):\n\t\t\"\"\"Generate random tests using functions as an input and values as random entry points\"\"\"\n\t\tif 'generate_tests' not in self.settings:\n\t\t\tself.settings[\"logger\"].error(\"Generate test option not defined\")\n\t\telif self.settings['generate_tests'] > 5 or self.settings['generate_tests'] < 0:\n\t\t\tself.settings[\"logger\"].error(\"Option for random tests not available\")\n\t\telif not isinstance(latest_id, int):\n\t\t\tself.settings[\"logger\"].error(\"The latest id should be an int\")\n\t\telif not isinstance(limit, int):\n\t\t\tself.settings[\"logger\"].error(\"The limit should be an int\")\n\t\telse:\n\t\t\tvalues = self.settings['db'].get_values()\n\t\t\tif not values:\n\t\t\t\tself.settings[\"logger\"].error(\"No values detected, you require at least 1 value in the table 'value'. For example: ./xdiff_dbaction.py -d %s -t value -i canaryfile\", self.settings['db_file'])\n\t\t\telse:\n\t\t\t\tfunctions = self.settings['db'].get_functions()\n\t\t\t\tif not functions:\n\t\t\t\t\tself.settings[\"logger\"].error(\"No functions detected, you require at least 1 value in the table 'function'. For example: ./xdiff_dbaction.py -d %s -t function -i [[test]]\", self.settings['db_file'])\n\t\t\t\telse:\n\t\t\t\t\tself.settings['logger'].info(\"Testcases being generated\")\n\t\t\t\t\tcount = 0\n\t\t\t\t\twhile count < (limit * self.settings['generate_multiplier']): # add more tests than necessary\n\t\t\t\t\t\tfor value in values:\n\t\t\t\t\t\t\tstdout = [] # where the new random values will be stored\n\t\t\t\t\t\t\tif self.settings['generate_tests'] in [0, 1, 2, 3]: # radamsa\n\t\t\t\t\t\t\t\tif not find_executable(\"radamsa\"):\n\t\t\t\t\t\t\t\t\tself.settings[\"logger\"].error(\"Radamsa not found within PATH\")\n\t\t\t\t\t\t\t\t\tsys.exit()\n\t\t\t\t\t\t\t\tinput_value = tempfile.mkstemp(suffix=\"File\", prefix=self.settings['tmp_prefix'] + \"mutate_\", dir=self.settings['tmp_dir'])\n\t\t\t\t\t\t\t\tif self.settings['generate_tests'] in [0, 2]: # add a newline to speed up the generation process\n\t\t\t\t\t\t\t\t\tos.write(input_value[0], value[0] + \"\\n\")\n\t\t\t\t\t\t\t\t\trepeat = 1\n\t\t\t\t\t\t\t\t\tinput_count = limit\n\t\t\t\t\t\t\t\telse:\n\t\t\t\t\t\t\t\t\tos.write(input_value[0], value[0])\n\t\t\t\t\t\t\t\t\trepeat = limit\n\t\t\t\t\t\t\t\t\tinput_count = 1\n\t\t\t\t\t\t\t\tos.close(input_value[0])\n\t\t\t\t\t\t\t\tfor x in range(0, repeat):\n\t\t\t\t\t\t\t\t\tstdout.append(self.execute_shell(\"radamsa -n \" + str(input_count) + \" \" + input_value[1]))\n\t\t\t\t\t\t\t\tos.unlink(input_value[1])\n\t\t\t\t\t\t\tif self.settings['generate_tests'] in [0, 1, 4, 5]: # zzuf\n\t\t\t\t\t\t\t\tif not find_executable(\"zzuf\"):\n\t\t\t\t\t\t\t\t\tself.settings[\"logger\"].error(\"Zzuf not found within PATH\")\n\t\t\t\t\t\t\t\t\tsys.exit()\n\t\t\t\t\t\t\t\tinput_value = tempfile.mkstemp(suffix=\"File\", prefix=self.settings['tmp_prefix'] + \"mutate_\", dir=self.settings['tmp_dir'])\n\t\t\t\t\t\t\t\tif self.settings['generate_tests'] in [0, 4]: # add a newline to speed up the generation process\n\t\t\t\t\t\t\t\t\tos.write(input_value[0], \"\\n\".join([value[0]] * limit))\n\t\t\t\t\t\t\t\t\trepeat = 1\n\t\t\t\t\t\t\t\telse:\n\t\t\t\t\t\t\t\t\tos.write(input_value[0], value[0])\n\t\t\t\t\t\t\t\t\trepeat = limit\n\t\t\t\t\t\t\t\tos.close(input_value[0])\n\t\t\t\t\t\t\t\tfor x in range(0, repeat):\n\t\t\t\t\t\t\t\t\tstdout.append(self.execute_shell(\"zzuf -r\" + str(random.uniform(0.01, 0.03)) + \" -s\" + str(latest_id + repeat + x) + \" <\" + input_value[1])) # zzuf -s 1.\n#\nimport ctypes\nimport os.path\nimport shutil\nimport socket\nimport subprocess\nimport sys\n\n\ntry:\n\tfrom urllib2 import urlopen # python 2\n\tfrom urllib2 import HTTPError\n\tfrom urllib2 import URLError\nexcept ImportError:\n\tfrom urllib.request import urlopen # python 3\n\tfrom urllib.error import HTTPError\n\tfrom urllib.error import URLError\n\n\nclass Monitor(object):\n\t\"\"\"Checks that everything is looking good before the fuzzer stats, and while the fuzzer operates\"\"\"\n\tdef __init__(self, settings):\n\t\t\"\"\"Execute all the checks within this class to verify that canarys have been properly set up in the testcases\"\"\"\n\t\tself.settings = settings\n\n\tdef check_once(self):\n\t\t\"\"\"Check only once\"\"\"\n\t\tself.check_canary_references(self.settings['canaryfile'])\n\t\tself.check_canary_references(\"canaryhost\")\n\t\tself.check_canary_web(self.settings['canaryhost'], self.settings['canaryfile'], self.settings['canaryfileremote'])\n\t\tself.check_canary_command(self.settings['canaryexec'], self.settings['canaryexectoken'])\n\t\tself.check_ulimit()\n\t\tself.check()\n\t\treturn None\n\n\tdef check(self):\n\t\t\"\"\"Check on each loop the canary file and the free space\"\"\"\n\t\tself.remove_stuff()\n\t\tstatus = self.check_canary_file(self.settings['tmp_dir'] + self.settings['canaryfile'], self.settings['canaryfiletoken'])\n\t\tstatus += self.check_free_space()\n\t\treturn status\n\n\tdef remove_stuff(self):\n\t\t\"\"\"Remove files that may affect the behaviour\"\"\"\n\t\t# delete specific files\n\t\tif sys.platform == \"linux2\":\n\t\t\ttry:\n\t\t\t\tos.remove(os.getenv(\"HOME\") + '.hhvm.hhbc') # hhvm may fill up the disk with temp stuff\n\t\t\texcept:\n\t\t\t\tpass\n\t\t# delete all tmp_dir files\n\t\tfor root, dirs, files in os.walk(self.settings['tmp_dir']):\n\t\t\tfor f in files:\n\t\t\t\ttry:\n\t\t\t\t\tif os.path.isfile(os.path.join(root, f)):\n\t\t\t\t\t\tos.unlink(os.path.join(root, f))\n\t\t\t\texcept:\n\t\t\t\t\tpass\n\t\t\tfor d in dirs:\n\t\t\t\ttry:\n\t\t\t\t\tif os.path.isdir(os.path.join(root, d)):\n\t\t\t\t\t\tshutil.rmtree(os.path.join(root, d))\n\t\t\t\texcept:\n\t\t\t\t\tpass\n\n\tdef check_canary_file(self, filename, token):\n\t\t\"\"\"Check if the file exists and its contents are equal to the token\"\"\"\n\t\tstatus = None\n\t\tif not isinstance(filename, str):\n\t\t\tself.settings['logger'].error(\"Filename is not a string\")\n\t\telif not isinstance(token, str):\n\t\t\tself.settings['logger'].error(\"Token is not a string\")\n\t\telse:\n\t\t\tif os.path.isfile(filename):\n\t\t\t\ttry:\n\t\t\t\t\ttoken_file = open(filename, 'r')\n\t\t\t\texcept:\n\t\t\t\t\tself.settings['logger'].debug(\"CanaryFile could not be open, changing its permissions\")\n\t\t\t\t\tos.chmod(filename, 0o644)\n\t\t\t\t\ttoken_file = open(filename, 'r')\n\t\t\t\ttmptoken = token_file.read().strip()\n\t\t\t\ttoken_file.close()\n\t\t\t\tif tmptoken == token:\n\t\t\t\t\treturn 1\n\t\t\t\telse:\n\t\t\t\t\tself.settings['logger'].debug(\"CanaryFile token differs, creating a new one\")\n\t\t\telse:\n\t\t\t\tself.settings['logger'].debug(\"CanaryFile %s not found, creating a new one\", str(filename))\n\t\t\t\tstatus = self.create_canary_file(filename, token)\n\t\treturn status\n\n\tdef create_canary_file(self, filename, token):\n\t\t\"\"\"Create a text file with a certain token\"\"\"\n\t\tstatus = None\n\t\tif not isinstance(filename, str):\n\t\t\tself.settings['logger'].error(\"Filename is not a string\")\n\t\telif not isinstance(token, str):\n\t\t\tself.settings['logger'].error(\"Token is not a string\")\n\t\telse:\n\t\t\tcanary_file = open(filename, 'w')\n\t\t\tcanary_file.write(token)\n\t\t\tcanary_file.close()\n\t\t\tself.settings['logger'].debug(\"CanaryFile created\")\n\t\t\tstatus = True\n\t\treturn status\n\n\tdef check_canary_web(self, hostname, filename, token):\n\t\t\"\"\"Check if the hostname exists, that is possible to retrieve the filename and the contents are equal to the token\"\"\"\n\t\tstatus = None\n\t\tif not isinstance(hostname, str):\n\t\t\tself.settings['logger'].error(\"Hostname is not a string\")\n\t\telif not isinstance(filename, str):\n\t\t\tself.settings['logger'].error(\"Filename is not a string\")\n\t\telif not isinstance(token, str):\n\t\t\tself.settings['logger'].error(\"Token is not a string\")\n\t\telse:\n\t\t\turl = \"http://\" + hostname + \"/\" + filename + \"?monitor\"\n\t\t\ttry:\n\t\t\t\tresponse = urlopen(\"http://\" + hostname + \"/\" + filename + \"?monitor\", timeout=5)\n\t\t\t\tdata = response.read().strip()\n\t\t\t\tif data == token:\n\t\t\t\t\tstatus = True\n\t\t\t\telse:\n\t\t\t\t\tself.settings['logger'].warning(\"CanaryWeb token mismatch: expected %s and received %s\", token, data)\n\t\t\t\t\tstatus = False\n\t\t\texcept socket.error:\n\t\t\t\tself.settings['logger'].warning(\"CanaryWeb Hostname %s not found\", str(hostname))\n\t\t\t\tstatus = False\n\t\t\texcept HTTPError:\n\t\t\t\tself.settings['logger'].warning(\"CanaryWeb Filename %s not found: %s\", str(filename), url)\n\t\t\t\tstatus = False\n\t\t\texcept URLError:\n\t\t\t\tself.settings['logger'].warning(\"CanaryWeb may not work, network is unreachable\")\n\t\t\t\tstatus = False\n\t\treturn status\n\n\tdef check_canary_command(self, command, token):\n\t\t\"\"\"Check that the command can be executed and returns the expected token\"\"\"\n\t\tstdout = None\n\t\tfound = None\n\t\ttry:\n\t\t\tstdout, stderr = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()\n\t\texcept Exception as e:\n\t\t\tself.settings['logger'].warning(\"CanaryCommand %s not found: %s\", str(command), str(e))\n\n\t\tif stdout:\n\t\t\tfound = True\n\t\t\tif token not in stdout.strip():\n\t\t\t\tself.settings['logger'].warning(\"CanaryCommand token (%s) differs: '%s'\", token, str(stdout.strip()))\n\t\t\t\tfound = False\n\t\treturn found\n\n\tdef check_canary_references(self, reference):\n\t\t\"\"\"Check if the reference is on any of the testcases of the database\"\"\"\n\t\tfound = 1\n\t\tif self.settings['db'].count_reference(reference) == 0:\n\t\t\tself.settings['logger'].warning(\"CanaryReferences were not found in the db for the string: %s\", str(reference))\n\t\t\tfound = 0\n\t\treturn found\n\n\tdef check_free_space(self):\n\t\t\"\"\"Check if the there are more than Xmb free\"\"\"\n\t\tif sys.platform == \"win32\":\n\t\t\tfree_bytes = ctypes.c_ulong(0)\n\t\t\tctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(\".\"), None, None, ctypes.pointer(free_bytes))\n\t\t\tfree_mb = free_bytes.value / 1024 / 1024\n\t\telse:\n\t\t\tstat = os.statvfs('.')\n\t\t\tfree_mb = stat.f_bfree * stat.f_frsize / 1024 / 1024\n\t\tif free_mb <= self.settings['lowerlimit']:\n\t\t\tself.settings['logger'].critical(\"There is not enough space on the device. The current free disk space in gigabytes is: %s\", str(stat.f_bfree * stat.f_frsize / 1024 / 1024))\n\t\t\tsys.exit()\n\t\treturn 1\n\n\tdef check_ulimit(self):\n\t\t\"\"\"Check that the command can be executed and returns the expected token\"\"\"\n\t\tif sys.platform != \"win32\":\n\t\t\tminimum = 1024\n\t\t\ttry:\n\t\t\t\tstdout, stderr = subprocess.Popen([\"ulimit\", \"-n\"], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()\n\t\t\texcept:\n\t\t\t\tself.settings['logger'].debug(\"ulimit check did not work\")\n\t\t\t\treturn 0\n\n\t\t\tif int(stdout.strip()) < minimum:\n\t\t\t\tself.settings['logger'].critical(\"ulimit is too low (%s), you must raise ulimit (`ulimit -n %s`)\", str(stdout.strip()), str(minimum))\n\t\t\t\tsys.exit()\n\t\treturn 1\n"
},
{
"path": "classes/queue.py",
"content": "#\n# Copyright (C) 2018 Fernando Arnaboldi\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n#\nfrom __future__ import absolute_import\nfrom .fuzzer import Fuzzer\nfrom .webserver import WebServer\n\n\nclass Queue(Fuzzer, WebServer):\n\t\"\"\"Used to share information between executions and the webserver\"\"\"\n\tdef __init__(self, settings):\n\t\tself.ids = []\n\t\tFuzzer.__init__(self, settings, self.ids)\n\t\tWebServer.__init__(self, settings)\n"
},
{
"path": "classes/settings.py",
"content": "#\n# Copyright (C) 2018 Fernando Arnaboldi\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n#\nfrom __future__ import print_function\nfrom __future__ import absolute_import\nimport getpass\nimport logging\nimport os\nimport random\nimport sys\nfrom xdiff_dbaction import Dbaction\nfrom .queue import Queue\nfrom .dbsqlite import DbSqlite\nfrom .monitor import Monitor\n\n\ndef define_software(settings):\n\t\"\"\"The software gets loaded in a dictionary\"\"\"\n\tsoftware = []\n\tif \"software\" in settings and settings['software'] and \"fuzz_category\" in settings and settings['fuzz_category']:\n\t\tCategory = None\n\t\tif os.path.isfile(settings['software']):\n\t\t\tsoftware_file = open(settings['software'], \"r\")\n\t\t\tfor line in software_file:\n\t\t\t\tline = line.strip()\n\t\t\t\tif line[:1] != \"#\": # parse lines that are not comments\n\t\t\t\t\tif line[:1] == \"[\" and line[len(line) - 1:len(line)] == \"]\": # is this a category?\n\t\t\t\t\t\tCategory = line[1:len(line) - 1]\n\t\t\t\t\t\tType = None\n\t\t\t\t\t\tSuffix = None\n\t\t\t\t\t\tFilename = None\n\t\t\t\t\t\tOS = []\n\t\t\t\t\tif Category == settings['fuzz_category']:\n\t\t\t\t\t\tif line[:2] == \"OS\" or line[:4] == \"Type\" or line[:6] == \"Suffix\" or line[:8] == \"Filename\":\n\t\t\t\t\t\t\texec(line)\n\t\t\t\t\t\t\tif OS is not None and sys.platform not in OS:\n\t\t\t\t\t\t\t\tOS = None\n\t\t\t\t\t\telse:\n\t\t\t\t\t\t\tif line.find('=') != -1 and OS is not None:\n\t\t\t\t\t\t\t\tif Type is None:\n\t\t\t\t\t\t\t\t\tType = [\"CLI\"]\n\t\t\t\t\t\t\t\tif Suffix is None:\n\t\t\t\t\t\t\t\t\tSuffix = [\"\"]\n\t\t\t\t\t\t\t\tif Filename is None:\n\t\t\t\t\t\t\t\t\tFilename = [\"\"]\n\t\t\t\t\t\t\t\titem = {}\n\t\t\t\t\t\t\t\titem['category'] = Category\n\t\t\t\t\t\t\t\titem['type'] = Type\n\t\t\t\t\t\t\t\titem['suffix'] = Suffix\n\t\t\t\t\t\t\t\titem['filename'] = Filename\n\t\t\t\t\t\t\t\titem['name'] = line[:line.find('=')].strip()\n\t\t\t\t\t\t\t\tif 'valgrind' in settings and settings['valgrind']:\n\t\t\t\t\t\t\t\t\titem['execute'] = eval('[\"valgrind\", \"-q\", ' + line[line.find('=') + 1:].strip()[1:])\n\t\t\t\t\t\t\t\telse:\n\t\t\t\t\t\t\t\t\titem['execute'] = eval(line[line.find('=') + 1:].strip())\n\t\t\t\t\t\t\t\titem['softwareid'] = settings['db'].get_software_id(item)\n\t\t\t\t\t\t\t\tif item['softwareid']:\n\t\t\t\t\t\t\t\t\tsettings['logger'].debug(\"Software found: %s\", str(item))\n\t\t\t\t\t\t\t\t\tsoftware.append(item)\n\t\t\tsoftware_file.close()\n\t\telse:\n\t\t\tsettings['logger'].error(\"The settings file %s does not exist\", os.path.abspath(settings['software']))\n\treturn software\n\n\ndef set_logger(settings):\n\t\"\"\"Insantiate the logging functionality\"\"\"\n\tlogging.basicConfig(filename='fuzz.log', level=logging.INFO, format='%(asctime)s %(levelname)s %(module)s: %(message)s', datefmt='%Y-%m-%d %H.%M.%S')\n\tconsole = logging.StreamHandler()\n\tconsole.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(module)s: %(message)s'))\n\tlogger = logging.getLogger('fuzzer')\n\tlogger.addHandler(console)\n\tif 'loglevel' in settings and settings['loglevel'] == 'debug':\n\t\tlogger.setLevel(logging.DEBUG)\n\telif 'loglevel' in settings and settings['loglevel'] == 'critical':\n\t\tlogger.setLevel(logging.CRITICAL)\n\treturn logger\n\n\ndef load_settings(settings):\n\t\"\"\"Define global settings\"\"\"\n\tsettings['logger'] = set_logger(settings)\n\t# Run\n\tsettings['version'] = \"1.2.0 (HITB Edition)\"\n\tsettings['soft_limit'] = 250 # maximum limit for the output of stdout & stderr\n\tsettings['soft_bypass'] = [\"canarytoken\", getpass.getuser(), \"root\", \"/usr\", \"/bin\", \"PATH\", \"== \"] # exceptions for the soft_limit setting\n\tsettings['hard_limit'] = 1024 # maximum hard limit, regardless of the soft_limit & soft_bypass\n\t# settings['hard_limit_lines'] = 1 # maximum line limit in the output\n\tsettings['tmp_prefix'] = \"chkF_\" # prefix for temporary files created\n\n\tif sys.platform in [\"darwin\"]:\n\t\tsettings['tmp_dir'] = \"/Volumes/ramdisk/\"\n\t\tsettings['tmp_dir_howto'] = \"diskutil erasevolume HFS+ 'ramdisk' `hdiutil attach -nomount ram://838860`\"\n\telif sys.platform == \"win32\":\n\t\tsettings['tmp_dir'] = \"X:\\\\\"\n\t\tsettings['tmp_dir_howto'] = \"imdisk -a -s 512M -m X: -p \\\"/fs:ntfs /q/y\\\"; notepad \\\"C:\\Windows\\System32\\canaryfile.bat\\\": @echo off; echo canarytokencommand\"\n\telif sys.platform == \"linux2\" or sys.platform == \"freebsd11\":\n\t\tsettings['tmp_dir'] = \"/mnt/ramdisk/\"\n\t\tsettings['tmp_dir_howto'] = \"mkdir /mnt/ramdisk; mount -t tmpfs -o size=512m tmpfs /mnt/ramdisk; echo \\\"tmpfs /mnt/ramdisk tmpfs nodev,nosuid,noexec,nodiratime,size=512M 0 0\\\" >> /etc/fstab\"\n\tsettings['webserver_port'] = random.randrange(10000, 65535) # dynamic web server port: crashes in the same port may interfere\n\t# settings['webserver_port'] = 8000 # sometimes you just need a fixed value\n\tif \"db_file\" not in settings:\n\t\tsettings[\"db_file\"] = None\n\tsettings['db'] = DbSqlite(settings, settings['db_file'])\n\tif settings['db'].db_connection:\n\t\tsettings['kill_status'] = {\"not_killed\": settings['db'].get_constant_value(\"kill_status\", \"not killed\"), \"requested\": settings['db'].get_constant_value(\"kill_status\", \"requested\"), \"killed\": settings['db'].get_constant_value(\"kill_status\", \"killed\"), \"not_found\": settings['db'].get_constant_value(\"kill_status\", \"not found\")}\n\tif \"db_tests\" not in settings:\n\t\tsettings['db_tests'] = 100 # save the results in the database every X tests\n\tif \"software\" not in settings:\n\t\tsettings['software'] = os.path.abspath(\"software.ini\") # software definitions\n\tif \"timeout\" not in settings:\n\t\tsettings['timeout'] = 10 # default timeout for threads in seconds\n\n\tsettings['software'] = define_software(settings) # load the software and find potential inconsistencies\n\tsettings['queue'] = Queue(settings) # prepare the fuzzer and the webserver to interact\n\tsettings['monitor'] = Monitor(settings) # instantiate the monitor object\n\tsettings['dbaction'] = Dbaction(settings) # instantiate the dbaction object\n\n\t# Fuzzer\n\tif \"generate_multiplier\" not in settings:\n\t\tsettings['generate_multiplier'] = 100 # multiply the testcase limit by this number to generate new test cases\n\n\t# Monitor\n\tsettings['lowerlimit'] = 200 # minimum free space in megabytes\n\tsettings['canaryfile'] = \"canaryfile\"\n\tsettings['canaryfiletoken'] = \"canarytokenfilelocal\" # contents of settings['canaryfile']\n\tsettings['canaryexec'] = \"canaryfile\"\n\tsettings['canaryexectoken'] = \"canarytokencommand\" # contents of settings['canaryexec']\n\tsettings['canaryhost'] = \"127.0.0.1:\" + str(settings['webserver_port'])\n\tsettings['canaryfileremote'] = \"canarytokenfileremote\"\n\n\t# Analyze\n\tsettings['output_width'] = 130\n\tsettings['testcase_limit'] = 200 # a low number will help with RAM comsumption when performing queries against big databases\n\tif \"output_type\" not in settings:\n\t\tsettings[\"output_type\"] = \"html\" # default output type\n\tsettings[\"print_risk\"] = False # print the risk?\n\tif \"minimum_risk\" not in settings:\n\t\tsettings[\"minimum_risk\"] = 0 # defaul minimum risk\n\tsettings[\"max_results\"] = 999999999 # ridiculous high number to get all the occurrences of a function\n\tif settings['db_file']:\n\t\tsettings['output_file'] = settings['db_file'] + \".\" + settings['output_type']\n\tsettings['error_disclosure'] = [\"Exception\", \"stack trace\", \"core dump\", \"egmentation fault\", \"Traceback\"]\n\tsettings['soft_bypass'].extend(settings['error_disclosure'])\n\n\treturn settings\n"
},
{
"path": "classes/webserver.py",
"content": "#\n# Copyright (C) 2018 Fernando Arnaboldi\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n#\nimport threading\nimport os.path\nimport compat\n\ntry: # Python 2\n\tfrom SimpleHTTPServer import SimpleHTTPRequestHandler\n\timport BaseHTTPServer\n\timport urlparse\nexcept ImportError: # Python 3\n\tfrom http.server import SimpleHTTPRequestHandler\n\tfrom http.server import BaseHTTPRequestHandler, HTTPServer\n\tfrom urllib.parse import urlparse\n\n\nclass BaseHandler(SimpleHTTPRequestHandler):\n\t\"\"\"Changes a few things from SimpleHTTPServer to handle requests\"\"\"\n\tmy_class = None # type:BaseHandler\n\n\tdef log_message(self, format, *args):\n\t\t\"\"\"Avoid SimpleHTTPServer logs\"\"\"\n\t\tpass\n\n\tdef do_GET(self):\n\t\t\"\"\"Handle GET requests to parse parameters and save the responses to the corresponding ids\"\"\"\n\t\t# self.my_class.settings['logger'].debug(\"URL: %s Query: %s\", str(url), str(query))\n\t\tdata = compat.unicode(\"GET \" + str(self.path) + \"\\n\" + str(self.headers), errors='ignore')\n\t\tself.do_REQUEST(data)\n\n\tdef do_POST(self):\n\t\t\"\"\"Handle GET requests to parse parameters and save the responses to the corresponding ids\"\"\"\n\t\t# self.my_class.settings['logger'].debug(\"URL: %s Query: %s\", str(url), str(query))\n\t\tdata = compat.unicode(\"POST \" + str(self.path) + \"\\n\" + str(self.headers), errors='ignore')\n\t\tself.do_REQUEST(data)\n\n\tdef do_REQUEST(self, data):\n\t\t\"\"\"Handle GET and POST requests to parse parameters and save the responses to the corresponding ids\"\"\"\n\t\turl = urlparse.urlparse(self.path)\n\t\tquery = url.query.split('&')\n\t\tself.my_class.settings['logger'].debug(\"%s\", data)\n\t\tif len(query) > 1:\n\t\t\t# with tag0 we can identify the testcaseid\n\t\t\ttag0 = query[0].split(\"=\")\n\t\t\t# with tag1 we can identify the softwareid\n\t\t\ttag1 = query[1].split(\"=\")\n\t\t\tif tag0[0] == \"tag0\" and tag1[0] == \"tag1\":\n\t\t\t\ttestcaseid = None\n\t\t\t\tsoftwareid = None\n\t\t\t\ttry:\n\t\t\t\t\ttestcaseid = int(tag0[1])\n\t\t\t\texcept Exception as e:\n\t\t\t\t\tself.my_class.settings['logger'].warning(\"Tag0 received, but is not a number: %s\",e)\n\t\t\t\ttry:\n\t\t\t\t\tsoftwareid = int(tag1[1])\n\t\t\t\texcept Exception as e:\n\t\t\t\t\tself.my_class.settings['logger'].warning(\"Tag1 received, but is not a number: %s\",e)\n\t\t\t\t# if we found a testcaseid and a software id, we can correlate it to the results\n\t\t\t\tif testcaseid and softwareid:\n\t\t\t\t\t# we don't want dupes, check if the request hasn't been issued before\n\t\t\t\t\tflag = False\n\t\t\t\t\tfor x in range(0, len(self.my_class.ids)):\n\t\t\t\t\t\tif self.my_class.ids[x][0] == testcaseid and self.my_class.ids[x][1] == softwareid and self.my_class.ids[x][2] == data:\n\t\t\t\t\t\t\tflag = True\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\tif not flag:\n\t\t\t\t\t\t# can we extract the stdout and elapsed data from the url?\n\t\t\t\t\t\tstdout = None\n\t\t\t\t\t\telapsed = None\n\t\t\t\t\t\tstderr = None\n\t\t\t\t\t\tfor parameter in query:\n\t\t\t\t\t\t\tparameter = parameter.split('=')\n\t\t\t\t\t\t\tif len(parameter) == 2:\n\t\t\t\t\t\t\t\tif parameter[0] == 'stdout':\n\t\t\t\t\t\t\t\t\tstdout = parameter[1]\n\t\t\t\t\t\t\t\telif parameter[0] == 'elapsed':\n\t\t\t\t\t\t\t\t\telapsed = parameter[1]\n\t\t\t\t\t\t\t\telif parameter[0] == 'stderr':\n\t\t\t\t\t\t\t\t\tstderr = parameter[1]\n\t\t\t\t\t\tself.my_class.ids.append([testcaseid, softwareid, data, stdout, elapsed, stderr])\n\n\t\tself.send_response(200)\n\t\tself.send_header(\"Content-type\", \"text/html\")\n\t\tself.end_headers()\n\t\tgetfile = url[2][1:].split('?')[0]\n\t\tif url.path == \"/canaryfile\":\n\t\t\tself.wfile.write(self.my_class.settings['canaryfileremote'])\n\t\telif os.path.isfile(getfile):\n\t\t\tcontent = open(getfile, \"r\")\n\t\t\tself.wfile.write(content.read())\n\n\nclass WebServer(object):\n\t\"\"\"Used to parse HTTP connections\"\"\"\n\tdef __init__(self, settings):\n\t\tself.settings = settings\n\t\tself.server = None\n\n\tdef start_web_server(self):\n\t\t\"\"\"Web server: load simplehttpserver as a thread and continue execution\"\"\"\n\t\tBaseHandler.my_class = self\n\t\tself.server = BaseHTTPServer.HTTPServer((\"127.0.0.1\", self.settings['webserver_port']), BaseHandler)\n\t\tthread = threading.Thread(target=self.server.serve_forever)\n\t\tthread.daemon = True\n\t\tself.settings['logger'].debug(\"Loading web server using port %s\" % str(self.settings['webserver_port']))\n\t\ttry:\n\t\t\tthread.start()\n\t\texcept KeyboardInterrupt:\n\t\t\tself.stop_web_server()\n\n\tdef stop_web_server(self):\n\t\t\"\"\"Web server shutdown when closing the fuzzer\"\"\"\n\t\tif self.server:\n\t\t\tself.settings['logger'].debug(\"Shutting down Web Server...\")\n\t\t\tself.server.shutdown()\n"
},
{
"path": "docs/1.-Install.md",
"content": "Follwing are the instructions on how to execute XDiFF in:\n* [Linux](#Linux)\n* [OSX](#OSX)\n* [Freebsd](#Freebsd)\n* [Windows](#Windows)\n\n---\n## Linux (Ubuntu/Debian)\n1. Install some utilities as root:\n```\napt update; apt -y install python2.7 gcc make git sqlite3 wget\n```\n2. Download the latest copy of XDiFF:\n```\ngit clone https://github.com/IOActive/XDiFF.git; cd XDiFF\n``` \n3. Install some input fuzzers (minimum 1gb of RAM required) as root:\n```\ngit clone https://github.com/aoh/radamsa.git; cd radamsa; make OFLAGS=-O1; make install; cd ..; rm -r radamsa/\nwget https://github.com/samhocevar/zzuf/releases/download/v0.15/zzuf-0.15.tar.bz2; tar -xf zzuf-0.15.tar.bz2; cd zzuf-0.15/; ./configure; make; make install; cd ..; rm -r zzuf-0.15.tar.bz2 zzuf-0.15/\n```\n4. Create a ramdisk where files will be created as root:\n```\nmkdir /mnt/ramdisk; mount -t tmpfs -o size=512m tmpfs /mnt/ramdisk; echo \"tmpfs /mnt/ramdisk tmpfs nodev,nosuid,noexec,nodiratime,size=512M 0 0\" >> /etc/fstab\n```\n5. Point the host *canaryhost* to *localhost* as root:\n```\necho \"127.0.0.1 canaryhost\"|tee -a /etc/hosts\n```\n6. Create the *canarycommand*:\n```\necho '#!/bin/sh'>/usr/local/bin/canaryfile.bat; echo 'echo canarytokencommand'>>/usr/local/bin/canaryfile.bat; chmod +x /usr/local/bin/canaryfile.bat; cp /usr/local/bin/canaryfile.bat /usr/local/bin/canaryfile\n```\n---\n## OSX\n1. Install some utilities. The following utilies are installed using brew, if you don't have it you can install it by executing ```/usr/bin/ruby -e \"$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)\"```:\n```\nbrew install git wget\n```\n2. Download the latest copy of XDiFF:\n```\ngit clone https://github.com/IOActive/XDiFF.git; cd XDiFF\n``` \n3. Install some input fuzzers (minimum 1gb of RAM required):\n```\ngit clone https://github.com/aoh/radamsa.git; cd radamsa; make OFLAGS=-O1; sudo cp bin/radamsa /usr/local/bin/; cd ..\nwget https://github.com/samhocevar/zzuf/releases/download/v0.15/zzuf-0.15.tar.bz2; tar -xf zzuf-0.15.tar.bz2; cd zzuf-0.15/; ./configure; make; make install; cd ..; rm -r zzuf-0.15.tar.bz2 zzuf-0.15/\n```\n4. Create a ramdisk where files will be created:\n```\ndiskutil erasevolume HFS+ 'ramdisk' `hdiutil attach -nomount ram://838860`\n```\n5. Point the host *canaryhost* to *localhost*:\n```\necho \"127.0.0.1 canaryhost\"|sudo tee -a /etc/hosts\n```\n6. Create the *canarycommand*:\n```\necho '#!/bin/sh'>/usr/local/bin/canaryfile.bat; echo 'echo canarytokencommand'>>/usr/local/bin/canaryfile.bat; chmod +x /usr/local/bin/canaryfile.bat; cp /usr/local/bin/canaryfile.bat /usr/local/bin/canaryfile\n```\n7. Raise the ulimit\n```\nulimit -n 1024 \n```\n---\n## Freebsd\n1. Install some utilities:\n```\npkg install git wget py27-sqlite3\n```\n2. Download the latest copy of XDiFF:\n```\ngit clone https://github.com/IOActive/XDiFF.git; cd XDiFF\n``` \n3. Install some input fuzzers (minimum 1gb of RAM required):\n```\ngit clone https://github.com/aoh/radamsa.git; cd radamsa; make OFLAGS=-O1; sudo make install; cd ..; rm -r radamsa/\n```\nPending: Zzuf compile options\n\n4. Create a ramdisk where files will be created:\n```\nsudo mkdir /mnt/ramdisk; sudo mount -t tmpfs -o size=512m tmpfs /mnt/ramdisk; sudo echo \"tmpfs /mnt/ramdisk tmpfs nodev,nosuid,noexec,nodiratime,size=512M 0 0\" >> /etc/fstab\n```\n5. Point the host *canaryhost* to *localhost*:\n```\necho \"127.0.0.1 canaryhost\" | sudo tee -a /etc/hosts\n```\n6. Create the *canarycommand*:\n```\necho '#\\!/bin/sh' > /usr/local/bin/canaryfile.bat ; echo 'echo canarytokencommand' >> /usr/local/bin/canaryfile.bat ; chmod +x /usr/local/bin/canaryfile.bat ; cp /usr/local/bin/canaryfile.bat /usr/local/bin/canaryfile\n```\n---\n## Windows\n\n1. Download and install some utilities:\n```\nPython 2.7: https://www.python.org/ftp/python/2.7.14/python-2.7.14.amd64.msi\n\nIMDisk: https://sourceforge.net/projects/imdisk-toolkit/files/latest/download\n```\n2. Download the latest copy of XDiFF:\n```\nhttps://github.com/IOActive/XDiFF/archive/master.zip\n``` \n3. Download some input fuzzers. For Radamsa, download and put within your PATH the .dll and the .exe:\n```\nhttps://github.com/vah13/radamsa/releases\n```\n4. Create a ramdisk where files will be created:\n```\nimdisk -a -s 512M -m X: -p \\\"/fs:ntfs /q/y\\\"\n```\nThen, format the ram disk once the Windows pop up appears\n\n5. Point the host *canaryhost* to *localhost*. Right click on startup -> Command Prompt (Admin):\n```\necho 127.0.0.1 canaryhost >> C:\\Windows\\System32\\drivers\\etc\\hosts\n```\n6. Create the *canarycommand*. Right click on startup -> Command Prompt (Admin):\n```\necho @echo off > C:\\Windows\\System32\\canaryfile.bat & echo.echo canarytokencommand >> C:\\Windows\\System32\\canaryfile.bat\n```\n---\n# What's next?\n\nYou want to define [the input](https://github.com/IOActive/XDiFF/wiki/2.-The-input)\n"
},
{
"path": "docs/2.-The-input.md",
"content": "# Why do I want to use a database?\nA database allows you to compare the results of how the software was executed when using different inputs, versions, implementations or operating systems. All the test cases to be evaluated are contained in one place and any issues found across multiple scenarios will be detected. Not only there is value on exploiting the vulnerabilities with the higher risk, but also the ones that affect multiple pieces of software at the same time. The performance and capabilities of SQLite for the fuzzer were proven to be better than MySQL and Redis.\n\n## How's the database structure?\nThe initial analysis of the database was constructed around how to fuzz programming languages. They allow you to create any piece of software, so they will have access to all the functionalities. With this in mind, this is the basic look of a plain SQLite database used by XDiFF:\n\n\n# sqlite3 dbs/plain.sqlite \nSQLite version 3.11.0 2016-02-15 17:29:24\nEnter \".help\" for usage hints.\nsqlite> .tables\nfunction fuzz_software fuzz_testcase_result\nfuzz_constants fuzz_testcase value \n
\n\nThere are two tables where you may want to manually insert or edit some values:\n* **value**: contains the items that will replace the ```[[test]]``` values in *function*. If you don't have a 'function', you can use the values in here with input fuzzers.\n\n* **function**: contains what you want to fuzz. There is a special keyword ```[[test]]``` that gets replaced by the values contained in **value**. For example, if you would like to fuzz the print() function, you would normally want to have in here ```print([[test]])```.\n\nThe tables that start with 'fuzz_' are generated by XDiFF:\n* **fuzz_testcase**: contains the combination of *function* and *value*\n\n* **fuzz_software**: contains the software defined in *software.ini*\n\n* **fuzz_testcase_result**: contains the result of executing the software defined in *fuzz_software* with the input defined in *fuzz_testcase*\n\n* **fuzz_constants**: contains internal constant values used by the fuzzer\n\n## Grab a sample database\nLet's grab a copy of the plain.sqlite database:\n```\ncp dbs/plain.sqlite shells.sqlite\n```\n\n## Insert testcases\n\nData can be inserted in the database using a ***sqlite3*** parser or using the ***xdiff_dbaction.py*** script. In case your test case/s are in a file, you may want to insert it directly into the database like this for example:\n\necho \"insert into value values (readfile('sample_file'))\"|sqlite3 shells.sqlite \n\n\n## Insert combinations of functions/values\n\nIf you have a certain function (or portion of code) that you want to fuzz with certain values, you can insert first the functions into the database:\n\n./xdiff_dbaction.py -d shells.sqlite -t function -i \"foo([[test]])\"\n
\n\nInsert the values that you want to use to fuzz the piece of code within the function table:\n\n./xdiff_dbaction.py -d shells.sqlite -t value -i \"bar\"\n
\n\nThen you can generate the permutations:\n\n./xdiff_dbaction.py -d shells.sqlite -g 1\n2017-11-20 22:06:24,901 INFO dbaction: Values: 1 - Functions: 1\n2017-11-20 22:06:24,901 INFO dbaction: Testcases generated: 1\n2017-11-20 22:06:24,902 INFO dbaction: Time required: 0.0 seconds\n
\n\nYou can later confirm how the information everything looks like:\n\n./xdiff_dbaction.py -d shells.sqlite -t fuzz_testcase -p\n----------------------------------------------------------------------------------------------------------\n| fuzz_testcase (1 rows) |\n----------------------------------------------------------------------------------------------------------\n| id | testcase |\n----------------------------------------------------------------------------------------------------------\n| 1 | foo(bar) |\n----------------------------------------------------------------------------------------------------------\n
\n\n## Extending the detection\n\nPart of the install process required to create a command named ```canaryfile``` (and ```canaryfile.bat```). When this file gets executed, it produces a specific output that can be later analyzed. Basically, you want the string ```canaryfile``` as part of your values.\n\nMoreover, if the software may open network connections, you also want to define the ```canaryhost``` as part of the potential values to be used. The connections will be detected locally and be included as part of the output to be analyzed.\n\n# What's next?\n\nYou want to define [the software](https://github.com/IOActive/XDiFF/wiki/3.-The-software)\n"
},
{
"path": "docs/3.-The-software.md",
"content": "In here you will find information about how to define pieces of software in the file *software.ini*.\n\nThis defines pieces of data in three columns:\n1. The first column defines the software category between brackets. Lets suppose that you want to fuzz command shells, so we can name the software category ***shells***.\n```javascript\n[shells]\n```\n2. The second column has four predefined possibilities:\n\n 2.1. **Type**: how the information is going to be read by the programs. By default if you don't specify anything is going to be ```CLI```, which means that the input to be fuzzed is grabbed from the command line. Another possibility is ```File```, which means that the contents of what's going to be fuzzed will be written into a file first. Moreover, whenever you're fuzzing files, you may want to specify what is the suffix of that file (please see below in 2.3). Finally, one last possibility for the input is ```Stdin```, as you would use it when piping information to another program.\n\n 2.2. **OS**: it could either be ***darwin***, ***linux2***, ***freebsd11*** or ***win32***\n\n 2.3. **Suffix**: the suffix used for files when the input type is set to ```File```. We can easily fuzz command shells without files and suffixes, but to illustrate the point let's use them:\n \n Type = [\"File\"]\n OS = [\"darwin\", \"linux2\", \"freebsd11\"]\n Suffix = [\".sh\"]
\n 2.4. **Filename**: if the software to be fuzzed reads information from a certain static filename, you can define it in here. Don't forget to run the fuzzer with only 1 thread when using this.\n\n3. The third column defines the pieces of software to be fuzzed. If you want to fuzz mp3 files using mpg321 and mpg123, you can do it like this:\n\n \n Bash = [\"bash\", \"-c\", \"-fuzzdata=echo $(([[test]]))\"]\n Ksh = [\"ksh\", \"-c\", \"-fuzzdata=echo $(([[test]]))\"]
\n First we set the name of the software to be fuzzed (***bash***, ***dash***, or ***ksh***). Then, we defined in an array the command and options to be executed. There is a special option named *-fuzzdata=* that indicates the fuzzer that the next piece of information is where we will be placed our fuzzed test case. The *[[test]]* will be replaced by a temporary file name containing a weird mp3 to fuzz the software on this example.\n\n### Putting all the pieces together\nThis is how you could define the software category ***shells*** to be fuzzed using the ***CLI***:\n```\n# Sample fuzzing of shells\n[shells]\n OS = [\"darwin\", \"linux2\", \"freebsd11\"]\n Bash = [\"bash\", \"-c\", \"-fuzzdata=echo $(([[test]]))\"]\n Ksh = [\"ksh\", \"-c\", \"-fuzzdata=echo $(([[test]]))\"]\n```\n---\n# What's next?\n\nYou want to [run the fuzzer](https://github.com/IOActive/XDiFF/wiki/4.-The-fuzzer)\n"
},
{
"path": "docs/4.-The-fuzzer.md",
"content": "## Fuzzing\nThe most basic execution requires defining which category and which database will be used:\n```\n./xdiff_run.py -c shells -d shells.sqlite\n```\nThe output should look like this:\n\n\nIt includes a lot of debugging information, and the most important parts are marked. At the top is the execution, and at the bottom is the beginning of the execution along with the rate (you want this number to be as high as possible).\n\n## Fuzzing using the input fuzzers\n\nIf you want to generate new test cases based on the currently defined test cases, you can use the input fuzzers that were installed as part of the install process.\n```\n./xdiff_run.py -c shells -d shells.sqlite -z 0\n```\nNow the output should indicate now and then when new inputs are being generated\n\n\n## Additional fuzzing options:\n\nThere are three additional important optional settings to be mentioned:\n\n- [*-D*]: Print debugging information\n- [*-t 100*]: The amount of threads to be executed in parallel.\n- [*-T 10*]: The timeout per thread\n- [*-v*]: Use valgrind to execute the software to be fuzzed.\n\nThe combination of threads and the timeout is something to be defined per category. Fuzzing a shell requires no time, while compiling and fuzzing a java program takes much more time. Pay attention at the output produced to see if the software is being properly executed (or is getting mostly killed because the timeout is too low).\n\n---\n# What's next?\n\nYou want to analyze [the output](https://github.com/IOActive/XDiFF/wiki/5.-The-output)\n"
},
{
"path": "docs/5.-The-output.md",
"content": "## Analyzing the output\nThe most basic form of analyzing the output is running:\n```\n./xdiff_analyze.py -d shells.sqlite\n```\nA normal analysis output looks like this:\n\n\n### HTML\nThe previous execution creates by default an HTML file named ```shells.sqlite.html``` that for this session looks like this on a web browser:\n\n\n\n### Text\n\nAnother possibility is to output the analysis as text when using the ```-t txt``` option:\n\n\n## The analytic functions\nThere are multiple analytic functions that can expose information from the database. The default function that gets executed is ```report```, which include 15 functions. Following is the whole list of function, and the ones in bold are already included as part of the ```report```:\n\n- **```analyze_canary_file```**: Find canary filenames in the stdout or stderr, even though canary files were not part of the payload\n- **```analyze_canary_token_code```**: Find canary tokens of code executed in the stdout or in the stderr\n- **```analyze_canary_token_command```**: Find canary tokens of commands in the stdout or stderr\n- **```analyze_canary_token_file```**: Find canary tokens of files in the stdout or in the stderr\n- ```analyze_elapsed```: Analize which was the total time required for each piece of software\n- ```analyze_error_disclosure```: Analyze errors disclosed in the output taken from settings['error_disclosure']\n- ```analyze_file_disclosure_without_path```: Find the tmp_prefix in the stdout or stderr without the full path\n- ```analyze_file_disclosure```: Find the tmp_prefix in the stdout or in the stderr\n- ```analyze_killed_differences```: Find when one piece of software was killed AND another one was not killed for the same input\n- **```analyze_output_messages```**: Analize which were the different output messages for each piece of software\n- ```analyze_path_disclosure_without_file```: Find the tmp_dir in the stdout or stderr, even though the testcase did not have a temporary file\n- ```analyze_path_disclosure_stdout```: Find the tmp_dir in the stdout\n- ```analyze_path_disclosure_stderr```: Find the tmp_dir in the stderr\n- **```analyze_remote_connection```**: Find remote connections made\n- **```analyze_return_code_differences```**: Find when different return codes are received for the same input\n- **```analyze_return_code_same_software_differences```**: Find when different return codes are received for the same software using different input forms\n- **```analyze_return_code```**: Get the different return codes for each piece of software\n- ```analyze_same_software```: Find when the same software produces different results when using different inputs \n- ```analyze_same_stdout```: Finds different testcases that produce the same standard output\n- ```analyze_specific_return_code```: Find specific return codes\n- **```analyze_stdout```**: Find when different pieces of software produces different results \n- ```analyze_top_elapsed_killed```: Find which killed tests cases required more time\n- ```analyze_top_elapsed_not_killed```: Find which not killed tests cases required more time\n- **```analyze_username_disclosure```**: Find when a specific username is disclosed in the stdout or in the stderr\n- **```analyze_valgrind```**: Find Valgrind references in case it was used\n- ```list_killed_results```: Print the killed fuzzing results\n- **```list_results```**: Print the fuzzing results: valuable to see how the software worked with the testcases defined, without using any constrains\n- **```list_software```**: Print the list of [active] software used with testcases from the database\n- ```list_summary```: Print an quantitative information summary using all the analytic functions from this class\n\n### Working with the analytic functions\nDepending on what type of software you're fuzzing, it may be convenient to enable or disable certain functions. The best way is to modify the ```xdiff_analyze.py``` script to expose the information that we need. \n\nFor other scenarios, you may just want to expose the output of a single function. Let's suppose that you only care about the analytic function ```analyze_return_code``` to see how code behaves:\n\n./xdiff_analyze.py -d shells.sqlite -m analyze_return_code -o txt\n
\n\nThe previous command produces the following output:\n```\n----------------------------------------------------------------------------------------\n| Analyze Different Return Codes per Software - analyze_return_code (5 rows) |\n----------------------------------------------------------------------------------------\n| Software | Type | OS | Return Code | Amount |\n----------------------------------------------------------------------------------------\n| Bash | CLI | darwin | 1 | 499 |\n----------------------------------------------------------------------------------------\n| Bash | CLI | darwin | 2 | 76 |\n----------------------------------------------------------------------------------------\n| Ksh | CLI | darwin | 0 | 73 |\n----------------------------------------------------------------------------------------\n| Ksh | CLI | darwin | 1 | 495 |\n----------------------------------------------------------------------------------------\n| Ksh | CLI | darwin | 3 | 7 |\n----------------------------------------------------------------------------------------\n```\n"
},
{
"path": "docs/Changelog.md",
"content": "# Changelog\nChanges are listed in time order: newer changes are at the top, older changes are at the bottom.\n\n## Version: [1.2.0](https://github.com/IOActive/XDiFF/releases/tag/1.2)\n- Changed main function names in the root directory\n- Improved code, documentation, and (most of) the code is now tested. Tons of bugfixes.\n- Added new analysis for error disclosure (analyze_error_disclosure) and path disclosure (analyze_path_disclosure_stderr)\n- Added new compatibility class (classes.compat) to support Python 3\n- Added risk value to the different analytic functions. Print functions based on their rating: ./xdiff_analyze.py -d db.sqlite -r 0/1/2/3\n- Improved analysis of network connections to test browsers connections\n- software.ini: added support to test non random filenames. Set on the second column: Filename = /etc/myfixedfilename\n- Added -d for debug output\n- Added new parameters in the settings.py class\n \n#### Contributors:\n- farnaboldi\n\n## Version: [1.1.1](https://github.com/IOActive/XDiFF/releases/tag/1.1.1) (beta)\n- Added support for Python 3 [[2]](https://github.com/IOActive/XDiFF/pull/2)\n\n#### Contributors:\n- cclauss\n\n## Version: [1.1.0](https://github.com/IOActive/XDiFF/releases/tag/1.1.0)\n- First public release for Blackhat Europe 2017\n\n#### Contributors:\n- farnaboldi\n"
},
{
"path": "xdiff_analyze.py",
"content": "#!/usr/bin/env python\n#\n# Copyright (C) 2018 Fernando Arnaboldi\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\nfrom __future__ import print_function\nimport datetime\nimport getopt\nimport getpass\nimport inspect\nimport os\n# import profile # uncomment here for benchmarking and at the bottom\nimport re\nimport sys\nimport time\nimport classes.settings\nfrom classes.dump import Dump\n\ntry:\n\treload # Python 2\nexcept NameError: # Python 3\n\tfrom importlib import reload\n\n\nclass Analyze(object):\n\t\"\"\"Analyzes the fuzzing information for abnormal behaviors\"\"\"\n\tdef __init__(self, settings):\n\t\treload(sys)\n\t\ttry:\n\t\t\tsys.setdefaultencoding('utf8')\n\t\texcept:\n\t\t\tpass # Python3\n\t\tself.settings = settings\n\t\tself.settings['tmp_dir'] = \"ramdisk\" # by using this, it will work on multiple directories (ie, /Volumes/ramdisk, /mnt/ramdisk, etc)\n\t\tself.dump = Dump(self.settings)\n\t\tself.count_results = None\n\n\tdef check_minimum_risk(self, function_risk, title):\n\t\t\"\"\"Check if the function has the minum risk required\"\"\"\n\t\tcheck = False\n\t\tif self.settings['print_risk']:\n\t\t\tprint(\"Function: %s, Risk: %s, Title: %s\" % (inspect.stack()[1][3], function_risk, title[:title.find(\" - \")]))\n\t\telif function_risk >= self.settings['minimum_risk']:\n\t\t\tcheck = True\n\t\treturn check\n\n\tdef dump_results(self, method, toplimit, extra):\n\t\t\"\"\"Prints the output of an internal method\"\"\"\n\t\tsuccess = False\n\t\tmethod_to_call = None\n\t\tif self.settings['output_type'] not in [\"txt\", \"csv\", \"xml\", \"html\"]:\n\t\t\tself.settings['logger'].error(\"Incorrect output type selected. Valid outputs: txt, csv, xml, html.\")\n\t\telse:\n\t\t\tif method not in ['dump_results']:\n\t\t\t\ttry:\n\t\t\t\t\tmethod_to_call = getattr(self, method)\n\t\t\t\texcept Exception as e:\n\t\t\t\t\tself.settings['logger'].error(\"Error when executing the method %s: %s\", method, e)\n\n\t\tif method_to_call:\n\t\t\tif method != \"report\":\n\t\t\t\tself.settings[\"minimum_risk\"] = 0 # set the minimum risk to 0\n\t\t\t\tself.dump.set_toggle_table(False)\n\t\t\tstart_time = time.time()\n\t\t\tself.settings['logger'].info(\"Dumping: database %s - method %s - output %s\" % (self.settings['db_file'], method, self.settings['output_type']))\n\t\t\tself.dump.pre_general(self.settings['output_type'])\n\n\t\t\tif extra:\n\t\t\t\ttry:\n\t\t\t\t\tmethod_to_call(self.settings['output_type'], toplimit, extra)\n\t\t\t\t\tsuccess = True\n\t\t\t\texcept Exception as e:\n\t\t\t\t\tself.settings['logger'].error(\"Error executing the method '%s' with parameter '%s': %s\", method, extra, e)\n\t\t\telse:\n\t\t\t\ttry:\n\t\t\t\t\tmethod_to_call(self.settings['output_type'], toplimit)\n\t\t\t\t\tsuccess = True\n\t\t\t\texcept Exception as e:\n\t\t\t\t\tself.settings['logger'].error(\"Error executing the method '%s': %s\", method, e)\n\t\t\tif success:\n\t\t\t\tself.dump.post_general(self.settings['output_type'])\n\t\t\t\tsize = \"\"\n\t\t\t\tif 'output_file' in self.settings and os.path.isfile(self.settings['output_file']):\n\t\t\t\t\tsize = \", output file: \" + self.settings['output_file'] + \" (\" + str(int(os.stat(self.settings['output_file']).st_size / 1024)) + \" kb)\"\n\t\t\t\telif 'output_file' in self.settings:\n\t\t\t\t\tsize = \". No information to be written into the output file.\"\n\t\t\t\tfinish_time = time.time() - start_time\n\t\t\t\tself.settings['logger'].info(\"Time elapsed %s seconds%s\" % (str(int(finish_time)), size))\n\t\treturn success\n\n\tdef report(self, output, toplimit):\n\t\t\"\"\"Print several functions in the form of a report (useful for HTML)\"\"\"\n\t\t# self.settings['db'].set_software([\"9\", \"10\"])\n\n\t\t# self.list_summary(output, toplimit) # informational\n\t\tself.list_software(output, self.settings[\"max_results\"])\n\n\t\tself.analyze_elapsed(output, toplimit) # informational\n\t\tself.list_results(output, toplimit)\n\n\t\tself.analyze_top_elapsed_killed(output, toplimit) # informational\n\t\tself.analyze_top_elapsed_not_killed(output, toplimit) # informational\n\n\t\tself.analyze_valgrind(output, toplimit)\n\t\tself.analyze_username_disclosure(output, toplimit, username=\"root\")\n\t\tif getpass.getuser() != \"root\": # do not repeat the information if the root user was the one already used for the execution\n\t\t\tself.analyze_username_disclosure(output, toplimit, username=getpass.getuser())\n\t\tself.analyze_canary_token_file(output, toplimit)\n\t\tself.analyze_canary_token_code(output, toplimit)\n\t\tself.analyze_remote_connection(output, toplimit)\n\t\tself.analyze_canary_token_command(output, toplimit)\n\t\tself.analyze_canary_file(output, toplimit)\n\n\t\tself.analyze_killed_differences(output, toplimit) # informational\n\n\t\tself.analyze_return_code(output, toplimit)\n\t\tself.analyze_specific_return_code(output, toplimit)\n\t\tself.analyze_return_code_differences(output, toplimit)\n\t\tself.analyze_return_code_same_software_differences(output, toplimit)\n\n\t\tself.analyze_output_messages(output, toplimit, 'stderr')\n\t\tself.analyze_output_messages(output, toplimit, 'stdout')\n\t\tself.analyze_error_disclosure(output, toplimit)\n\t\tself.analyze_same_software(output, toplimit) # low_risk\n\t\tself.analyze_stdout(output, toplimit)\n\t\tself.analyze_same_stdout(output, toplimit) # low_risk\n\n\t\tself.analyze_file_disclosure(output, toplimit) # low_risk\n\t\tself.analyze_file_disclosure_without_path(output, toplimit) # low_risk\n\t\tself.analyze_path_disclosure_stdout(output, toplimit) # low_risk\n\t\tself.analyze_path_disclosure_stderr(output, toplimit) # low_risk\n\t\tself.analyze_path_disclosure_without_file(output, toplimit) # low_risk\n\n\tdef list_summary(self, output, toplimit):\n\t\t\"\"\"Print an quantitative information summary using all the analytic functions from this class\"\"\"\n\t\ttitle = \"Summary for \" + self.settings['db_file']\n\t\tcolumns = [\"Information\", \"Amount\"]\n\t\tfunction_risk = 0\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\t\trows = []\n\n\t\tresults = len(self.list_software(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Pieces of Software\", str(results)]])\n\t\tif self.count_results is None:\n\t\t\tself.count_results = self.settings['db'].count_results(0, None)\n\t\trows.append([[\"Amount of Testcases\", str(self.count_results)]])\n\t\trows.append([[\"Output Top Limit\", str(toplimit)]])\n\n\t\tresults = len(self.analyze_valgrind(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Valgrind References Found\", str(results)]])\n\t\tresults = len(self.analyze_username_disclosure(None, self.settings[\"max_results\"], \"root\"))\n\t\trows.append([[\"Username 'root' Disclosure\", str(results)]])\n\t\tresults = len(self.analyze_username_disclosure(None, self.settings[\"max_results\"], getpass.getuser()))\n\t\trows.append([[\"Username '\" + getpass.getuser() + \"' Disclosure\", str(results)]])\n\t\tresults = len(self.analyze_canary_token_file(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Canary Token File Found\", str(results)]])\n\t\tresults = len(self.analyze_canary_token_code(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Canary Token Code Found\", str(results)]])\n\t\tresults = len(self.analyze_canary_token_command(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Canary Token Command Found\", str(results)]])\n\t\tresults = len(self.analyze_canary_file(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Canary File Found\", str(results)]])\n\t\tresults = len(self.analyze_top_elapsed_killed(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Testcases Killed\", str(results)]])\n\t\tresults = len(self.analyze_top_elapsed_not_killed(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Testcases not Killed\", str(results)]])\n\t\tresults = len(self.analyze_killed_differences(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Software Killed and Not Killed\", str(results)]])\n\t\tresults = len(self.analyze_return_code(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Return Code\", str(results)]])\n\t\tresults = len(self.analyze_return_code_differences(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Return Code Differences\", str(results)]])\n\t\tresults = len(self.analyze_return_code_same_software_differences(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Return Code Same Software Differences\", str(results)]])\n\t\tresults = len(self.analyze_same_software(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Same Software having a Different Output\", str(results)]])\n\t\tresults = len(self.analyze_stdout(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Stdout for Different Results\", str(results)]])\n\t\tresults = len(self.analyze_output_messages(None, self.settings[\"max_results\"], 'stderr'))\n\t\trows.append([[\"Different Stderr Messages\", str(results)]])\n\t\tresults = len(self.analyze_output_messages(None, self.settings[\"max_results\"], 'stdout'))\n\t\trows.append([[\"Different Stdout Messages\", str(results)]])\n\t\tresults = len(self.analyze_error_disclosure(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Analyze Error Messages for exceptions\", str(results)]])\n\t\tresults = len(self.analyze_same_stdout(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Testcases that Produce the Same Stdout\", str(results)]])\n\t\tresults = len(self.analyze_file_disclosure(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Temp File Disclosure\", str(results)]])\n\t\tresults = len(self.analyze_file_disclosure_without_path(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Temp File Disclosure (without path)\", str(results)]])\n\t\tresults = len(self.analyze_path_disclosure_stdout(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Path Disclosure Stdout\", str(results)]])\n\t\tresults = len(self.analyze_path_disclosure_stderr(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Path Disclosure Stderr\", str(results)]])\n\t\tresults = len(self.analyze_path_disclosure_without_file(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Path Disclosure (without temp file)\", str(results)]])\n\t\tresults = len(self.analyze_remote_connection(None, self.settings[\"max_results\"]))\n\t\trows.append([[\"Remote Connections\", str(results)]])\n\t\tresults = self.analyze_elapsed(None, self.settings[\"max_results\"])\n\t\tresults = datetime.timedelta(seconds=round(results, 0))\n\t\trows.append([[\"Total Time Elapsed\", str(results)]])\n\n\t\tself.dump.general(output, title, columns, rows)\n\n\tdef list_software(self, output, toplimit):\n\t\t\"\"\"Print the list of [active] software used with testcases from the database\"\"\"\n\t\ttitle = \"List Software Tested - list_software \"\n\t\tcolumns = [\"ID\", \"Software\", \"Type\", \"OS\"]\n\t\tfunction_risk = 0\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\t\trows = []\n\t\tresults = self.settings['db'].list_software()\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([result])\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef list_results(self, output, toplimit):\n\t\t\"\"\"Print the fuzzing results: valuable to see how the software worked with the testcases defined, without using any constrains\"\"\"\n\t\tlowerlimit = 0\n\t\ttitle = \"Analyze the Testcase Results from \" + str(int(lowerlimit)) + \" to \" + str(lowerlimit + toplimit) + \" - list_results\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\", \"Kill\"]\n\t\tfunction_risk = 0\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\ttestcase = None\n\t\ttmpoutput = []\n\t\tresults = self.settings['db'].list_results(lowerlimit, toplimit * len(self.list_software(None, self.settings[\"max_results\"])))\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\tif testcase is None:\n\t\t\t\ttestcase = result[0]\n\t\t\tif testcase != result[0]:\n\t\t\t\ttestcase = result[0]\n\t\t\t\trows.append(tmpoutput)\n\t\t\t\ttmpoutput = []\n\t\t\ttmpoutput.append((result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5], result[6]))\n\t\tif len(rows) < toplimit and tmpoutput:\n\t\t\trows.append(tmpoutput)\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_valgrind(self, output, toplimit):\n\t\t\"\"\"Find Valgrind references in case it was used\"\"\"\n\t\ttitle = \"Analyze Valgrind Output - analyze_valgrind\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\", \"Return Code\"]\n\t\tfunction_risk = 2\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_string_disclosure(\"== \",)\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\tif result[5][:10].count('=') == 4: # Valgrind outputs can be detected because they have 4 equal signs in the first 10 characters\n\t\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5], result[6])])\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef list_killed_results(self, output, toplimit):\n\t\t\"\"\"Print the killed fuzzing results\"\"\"\n\t\ttitle = \"Analyze the Killed Testcase Results - list_killed_results\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\", \"Kill\"]\n\t\tfunction_risk = 2\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\ttestcase = None\n\t\ttmpoutput = []\n\t\tresults = self.settings['db'].list_killed_results()\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\tif testcase is None:\n\t\t\t\ttestcase = result[0]\n\t\t\tif testcase != result[0]:\n\t\t\t\ttestcase = result[0]\n\t\t\t\trows.append(tmpoutput)\n\t\t\t\ttmpoutput = []\n\t\t\ttmpoutput.append((result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4][:500], result[5][:500], result[6]))\n\t\tif len(rows) < toplimit and tmpoutput:\n\t\t\trows.append(tmpoutput)\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_return_code(self, output, toplimit):\n\t\t\"\"\"Get the different return codes for each piece of software\"\"\"\n\t\ttitle = \"Analyze Different Return Codes per Software - analyze_return_code\"\n\t\tcolumns = [\"Software\", \"Type\", \"OS\", \"Return Code\", \"Amount\"]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].list_return_code_per_software()\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([(result[0], result[1], result[2], result[3], result[4])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_specific_return_code(self, output, toplimit):\n\t\t\"\"\"Find specific return codes\"\"\"\n\t\treturncodes = [\"-6\", \"-9\", \"-11\", \"-15\"]\n\t\ttitle = \"Analyze Specific Return Codes: \" + \",\".join(returncodes) + \" - analyze_specific_return_code\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Returncode\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 2\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_specific_return_code(returncodes)\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5], result[6])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_return_code_same_software_differences(self, output, toplimit):\n\t\t\"\"\"Find when different return codes are received for the same software using different input forms\"\"\"\n\t\ttitle = \"Analyze Return Code Same Software Differences - analyze_return_code_same_software_differences\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"Return Code\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 2\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\t# First check if there is more than one type of input per software, and save the IDs\n\t\tsoftware_ids = []\n\t\tsoftware_name = \"\"\n\t\tresults = self.settings['db'].list_software()\n\t\tfor result in results:\n\t\t\tif software_name == result[1]:\n\t\t\t\tsoftware_ids.append(str(result[0]))\n\t\t\telse:\n\t\t\t\tsoftware_name = result[1]\n\n\t\trows = []\n\t\tif software_ids:\n\t\t\toriginal_ids = self.settings['db'].get_software()\n\t\t\tself.settings['db'].set_software(software_ids) # restrict the ids\n\t\t\tsoftware = \"\"\n\t\t\tsoftware_returncode = \"\"\n\t\t\ttestcase = \"\"\n\t\t\toutputtmp = []\n\t\t\tresults = self.settings['db'].analyze_return_code_differences()\n\t\t\tfor result in results:\n\t\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\t\tbreak\n\t\t\t\tif testcase == result[0] and software == result[1] and software_returncode != result[3]:\n\t\t\t\t\toutputtmp.append([result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5]])\n\t\t\t\telse:\n\t\t\t\t\tif len(outputtmp) > 1:\n\t\t\t\t\t\trows.append(outputtmp)\n\t\t\t\t\toutputtmp = []\n\t\t\t\t\toutputtmp.append([result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5]])\n\t\t\t\ttestcase = result[0]\n\t\t\t\tsoftware = result[1]\n\t\t\t\tsoftware_returncode = result[3]\n\t\t\tself.settings['db'].set_software(original_ids)\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_return_code_differences(self, output, toplimit):\n\t\t\"\"\"Find when different return codes are received for the same input\"\"\"\n\t\ttitle = \"Analyze Return Code Differences - analyze_return_code_differences\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"Return Code\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 2\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tsoftware_returncode = \"\"\n\t\ttestcase = \"\"\n\t\toutputtmp = []\n\t\tresults = self.settings['db'].analyze_return_code_differences()\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\tif testcase == result[0] and software_returncode != result[3]:\n\t\t\t\toutputtmp.append([result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5]])\n\t\t\telse:\n\t\t\t\tif len(outputtmp) > 1:\n\t\t\t\t\trows.append(outputtmp)\n\t\t\t\toutputtmp = []\n\t\t\t\toutputtmp.append([result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5]])\n\t\t\ttestcase = result[0]\n\t\t\tsoftware_returncode = result[3]\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_username_disclosure(self, output, toplimit, username=None):\n\t\t\"\"\"Find when a specific username is disclosed in the stdout or in the stderr\"\"\"\n\t\ttitle = \"Analyze Username Disclosure: \" + username + \" - analyze_username_disclosure\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif username is None:\n\t\t\tprint(\"Error: extra parameter username has not been defined\")\n\t\t\thelp()\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_string_disclosure(username, excludeme=self.settings['tmp_prefix'])\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_error_disclosure(self, output, toplimit):\n\t\t\"\"\"Find canary filenames in the stdout or stderr, even though canary files were not part of the payload\"\"\"\n\t\ttitle = \"Analyze Presence of Exceptions - analyze_error_disclosure\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tfor error in self.settings['error_disclosure']:\n\t\t\tresults = self.settings['db'].analyze_string_disclosure(error)\n\t\t\tfor result in results:\n\t\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\t\tbreak\n\t\t\t\tif result[0].find('canaryfile') == -1:\n\t\t\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_canary_file(self, output, toplimit):\n\t\t\"\"\"Find canary filenames in the stdout or stderr, even though canary files were not part of the payload\"\"\"\n\t\ttitle = \"Analyze Presence of Canary Files - analyze_canary_file\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 3\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_canary_file()\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\tif result[0].find('canaryfile') == -1:\n\t\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_canary_token_file(self, output, toplimit):\n\t\t\"\"\"Find canary tokens of files in the stdout or in the stderr\"\"\"\n\t\ttitle = \"Analyze Presence of Canary Tokens File Local - analyze_canary_token_file\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 3\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_string_disclosure(\"canarytokenfile\")\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_canary_token_code(self, output, toplimit):\n\t\t\"\"\"Find canary tokens of code executed in the stdout or in the stderr\"\"\"\n\t\ttitle = \"Analyze Presence of Canary Tokens Code - analyze_canary_token_code\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 3\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_string_disclosure(\"canarytokencode\")\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_canary_token_command(self, output, toplimit):\n\t\t\"\"\"Find canary tokens of commands in the stdout or stderr\"\"\"\n\t\ttitle = \"Analyze Presence of Canary Tokens Command - analyze_canary_token_command\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 3\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_string_disclosure(\"canarytokencommand\")\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_remote_connection(self, output, toplimit):\n\t\t\"\"\"Find remote connections made\"\"\"\n\t\ttitle = \"Analyze Remote Connections - analyze_remote_connection\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\", \"Network\"]\n\t\tfunction_risk = 3\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\ttestcase = \"\"\n\t\toutputtmp = []\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_remote_connection()\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\tif testcase != result[0] and outputtmp:\n\t\t\t\ttestcase = result[0]\n\t\t\t\trows.append(outputtmp)\n\t\t\t\toutputtmp = []\n\t\t\toutputtmp.append((result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5], result[6]))\n\n\t\tif outputtmp:\n\t\t\trows.append(outputtmp)\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_top_elapsed_killed(self, output, toplimit):\n\t\t\"\"\"Find which killed tests cases required more time\"\"\"\n\t\ttitle = \"Analyze Top Time Elapsed (and eventually killed) - analyze_top_elapsed_killed\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Elapsed\"]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_top_elapsed(True)\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_top_elapsed_not_killed(self, output, toplimit):\n\t\t\"\"\"Find which not killed tests cases required more time\"\"\"\n\t\ttitle = \"Analyze Top Time Elapsed (but not killed) - analyze_top_elapsed_not_killed\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Elapsed\"]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_top_elapsed(False)\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_killed_differences(self, output, toplimit):\n\t\t\"\"\"Find when one piece of software was killed AND another one was not killed for the same input\"\"\"\n\t\ttitle = \"Analyze Killed Software vs Not Killed Software - analyze_killed_differences\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Kill\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 2\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\ttestcase = kill_status = None\n\t\toutputtmp = []\n\t\ttry:\n\t\t\tresults = self.settings['db'].analyze_killed_differences()\n\t\texcept:\n\t\t\tprint(\"Error when requesting the killed differences\")\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\n\t\t\tif testcase is None or testcase != result[0]:\n\t\t\t\ttestcase = result[0]\n\t\t\t\tkill_status = result[4]\n\n\t\t\tif testcase == result[0] and kill_status != result[4]:\n\t\t\t\toutputtmp.append([result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5], result[6]])\n\t\t\telse:\n\t\t\t\tif len(outputtmp) > 1:\n\t\t\t\t\trows.append(outputtmp)\n\t\t\t\toutputtmp = []\n\t\t\t\toutputtmp.append([result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5], result[6]])\n\t\t\ttestcase = result[0]\n\t\t\tkill_status = result[4]\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_same_software(self, output, toplimit):\n\t\t\"\"\"Find when the same software produces different results when using different inputs (ie, Node CLI vs Node File Input)\"\"\"\n\t\ttitle = \"Analyze Same Software having a Different Output - analyze_same_software\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"Stdout\"]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\t# First check if there is more than one type of input per software, and save the IDs\n\t\tsoftware_ids = []\n\t\tsoftware_name = \"\"\n\t\tresults = self.settings['db'].list_software()\n\t\tfor result in results:\n\t\t\tif software_name == result[1]:\n\t\t\t\tsoftware_ids.append(str(result[0]))\n\t\t\telse:\n\t\t\t\tsoftware_name = result[1]\n\n\t\trows = []\n\t\tif software_ids:\n\t\t\toriginal_ids = self.settings['db'].get_software()\n\t\t\tself.settings['db'].set_software(software_ids) # restrict the ids\n\t\t\tsoftware = \"\"\n\t\t\tsoftware_stdout = \"\"\n\t\t\ttestcase = \"\"\n\t\t\toutputtmp = []\n\t\t\tresults = self.settings['db'].analyze_same_software()\n\t\t\tfor result in results:\n\t\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\t\tbreak\n\t\t\t\tif testcase == result[0] and software == result[1] and software_stdout != result[3]:\n\t\t\t\t\toutputtmp.append([result[0][:self.settings['testcase_limit']], result[1], result[2], result[3]])\n\t\t\t\telse:\n\t\t\t\t\tif len(outputtmp) > 1:\n\t\t\t\t\t\trows.append(outputtmp)\n\t\t\t\t\toutputtmp = []\n\t\t\t\t\toutputtmp.append([result[0][:self.settings['testcase_limit']], result[1], result[2], result[3]])\n\t\t\t\ttestcase = result[0]\n\t\t\t\tsoftware = result[1]\n\t\t\t\tsoftware_stdout = result[3]\n\t\t\tif len(outputtmp) > 1:\n\t\t\t\trows.append(outputtmp)\n\t\t\tself.dump.general(output, title, columns, rows)\n\t\t\tself.settings['db'].set_software(original_ids)\n\t\treturn rows\n\n\tdef analyze_stdout(self, output, toplimit):\n\t\t\"\"\"Find when different pieces of software produces different results (basic differential testing)\"\"\"\n\t\ttitle = \"Analyze Stdout for Different Results (Basic Differential Testing) - analyze_stdout\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"ID\"]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\ttestcase = \"\"\n\t\tstdout = \"\"\n\t\ttobeprinted = False\n\t\toutputtmp = []\n\t\trows = []\n\n\t\tlowerlimit = 0\n\t\tupperlimit = 100000\n\t\twhile True:\n\t\t\tresults = self.settings['db'].analyze_stdout(lowerlimit, upperlimit)\n\t\t\tif not results:\n\t\t\t\tbreak\n\t\t\tlowerlimit += 100000\n\t\t\tupperlimit += 100000\n\t\t\tfor result in results:\n\t\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\t\tbreak\n\t\t\t\tif testcase != result[0]:\n\t\t\t\t\ttestcase = result[0]\n\t\t\t\t\tstdout = result[3]\n\t\t\t\t\tif outputtmp and tobeprinted:\n\t\t\t\t\t\trows.append(outputtmp)\n\t\t\t\t\ttobeprinted = False\n\t\t\t\t\toutputtmp = []\n\t\t\t\toutputtmp.append([result[0][:self.settings['testcase_limit']], result[1], result[2], result[5], result[3], result[6]])\n\t\t\t\tif stdout != result[3]:\n\t\t\t\t\ttobeprinted = True\n\t\tif outputtmp and tobeprinted and len(rows) < toplimit:\n\t\t\trows.append(outputtmp)\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_same_stdout(self, output, toplimit):\n\t\t\"\"\"Finds different testcases that produce the same standard output, but ignore the testcases where ALL the pieces of software match\"\"\"\n\t\ttitle = \"Analyze Testcases that Produce the Same Stdout - analyze_same_stdout\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\"]\n\t\tfunction_risk = 0\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\ttestcase = \"\"\n\t\toutputtmp = []\n\t\trows = []\n\t\tcountsoftware = self.settings['db'].count_software()\n\t\tresults = self.settings['db'].analyze_same_stdout()\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\tif testcase != result[4]:\n\t\t\t\tif outputtmp and len(outputtmp) != countsoftware:\n\t\t\t\t\trows.append(outputtmp)\n\t\t\t\toutputtmp = []\n\t\t\t\ttestcase = result[4]\n\t\t\tif not results or results[len(results) - 1][0] != result[0] or results[len(outputtmp) - 1][1] != result[1]:\n\t\t\t\toutputtmp.append([result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4]])\n\t\t#if outputtmp and len(outputtmp) != countsoftware and len(rows) < toplimit:\n\t\t#\trows.append(outputtmp)\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_file_disclosure(self, output, toplimit):\n\t\t\"\"\"Find the tmp_prefix in the stdout or in the stderr\"\"\"\n\t\ttitle = \"Analyze Temp File Disclosure (\" + self.settings['tmp_prefix'] + \") - analyze_file_disclosure\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_string_disclosure(self.settings['tmp_prefix'])\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_file_disclosure_without_path(self, output, toplimit):\n\t\t\"\"\"Find the tmp_prefix in the stdout or stderr without the full path\"\"\"\n\t\ttitle = \"Analyze Temp File Disclosure (\" + self.settings['tmp_prefix'] + \") Without Path (\" + self.settings['tmp_dir'] + \") - analyze_file_disclosure_without_path\"\n\t\tcolumns = [\"Test\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_string_disclosure(self.settings['tmp_prefix'])\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\tif result[3].find(self.settings['tmp_dir']) == -1 and result[4].find(self.settings['tmp_dir']) == -1:\n\t\t\t\trows.append([(result[0], result[1], result[2], result[3], result[4], result[5])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_path_disclosure_stdout(self, output, toplimit):\n\t\t\"\"\"Find the tmp_dir in the stdout or stderr\"\"\"\n\t\ttitle = \"Analyze Path Disclosure Stdout (\" + self.settings['tmp_dir'] + \") - analyze_path_disclosure_stdout\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_string_disclosure(self.settings['tmp_dir'], where='stdout')\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_path_disclosure_stderr(self, output, toplimit):\n\t\t\"\"\"Find the tmp_dir in the stdout or stderr\"\"\"\n\t\ttitle = \"Analyze Path Disclosure Stderr (\" + self.settings['tmp_dir'] + \") - analyze_path_disclosure_stderr\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_string_disclosure(self.settings['tmp_dir'], where='stderr')\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5])])\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_path_disclosure_without_file(self, output, toplimit):\n\t\t\"\"\"Find the tmp_dir in the stdout or stderr, even though the testcase did not have a temporary file\"\"\"\n\t\ttitle = \"Analyze Path Disclosure (\" + self.settings['tmp_dir'] + \") Without Temp File (\" + self.settings['tmp_prefix'] + \") - analyze_path_disclosure_without_file\"\n\t\tcolumns = [\"Testcase\", \"Software\", \"Type\", \"OS\", \"Stdout\", \"Stderr\"]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\tsoftware_ids = []\n\t\tresults = self.settings['db'].get_software_type('CLI')\n\t\tfor result in results:\n\t\t\tsoftware_ids.append(str(result[0]))\n\n\t\trows = []\n\t\tif software_ids:\n\t\t\toriginal_ids = self.settings['db'].get_software()\n\t\t\tself.settings['db'].set_software(software_ids) # restrict the ids\n\t\t\tresults = self.settings['db'].analyze_string_disclosure(self.settings['tmp_dir'])\n\t\t\tself.settings['db'].set_software(original_ids) # set the ids to the original value\n\t\t\tfor result in results:\n\t\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\t\tbreak\n\t\t\t\tif result[3].find(self.settings['tmp_prefix']) == -1 and result[4].find(self.settings['tmp_prefix']) == -1:\n\t\t\t\t\trows.append([(result[0][:self.settings['testcase_limit']], result[1], result[2], result[3], result[4], result[5])])\n\t\t\tself.dump.general(output, title, columns, rows)\n\t\t\tself.settings['db'].set_software(original_ids)\n\t\treturn rows\n\n\tdef analyze_output_messages(self, output, toplimit, messages='stderr'):\n\t\t\"\"\"Analize which were the different output messages for each piece of software\"\"\"\n\t\ttitle = \"Analyze Different \" + messages[0].upper() + messages[1:] + \" Output Messages - analyze_output_messages\"\n\t\tcolumns = [\"Software\", \"Type\", \"OS\", \"Return Code\", messages[0].upper() + messages[1:]]\n\t\tfunction_risk = 1\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\trows = []\n\t\tresults = self.settings['db'].analyze_output_messages(messages)\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\n\t\t\toutput_parsed = result[5]\n\t\t\tif len(result[0]) > 5:\n\t\t\t\toutput_parsed = output_parsed.replace(result[0], \"TESTCASE\") # if possible, remove the testcase from output\n\t\t\t\toutput_parsed = output_parsed.replace(str(result[0].encode(\"utf-8\")), \"TESTCASE\") # if possible, remove the testcase from output\n\t\t\tif output_parsed.find(self.settings['tmp_prefix']) != -1:\n\t\t\t\tregex = re.compile('[\\S]*' + self.settings['tmp_prefix'] + '[\\S]*')\n\t\t\t\tregex_iter = re.finditer(regex, output_parsed)\n\t\t\t\tfor match in regex_iter:\n\t\t\t\t\toutput_parsed = output_parsed.replace(match.group(0), \"TMPFILE\")\n\t\t\ttest = [result[1], result[2], result[3], result[4], output_parsed]\n\n\t\t\tflag = False\n\t\t\tfor row in rows:\n\t\t\t\tif [test] == row:\n\t\t\t\t\tflag = True\n\t\t\t\t\tbreak\n\t\t\tif not flag:\n\t\t\t\trows.append([test])\n\t\trows = sorted(rows)\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn rows\n\n\tdef analyze_elapsed(self, output, toplimit):\n\t\t\"\"\"Analize which was the total time required for each piece of software\"\"\"\n\t\ttitle = \"Analyze Elapsed Time - analyze_elapsed\"\n\t\tcolumns = [\"Software\", \"Type\", \"OS\", \"Elapsed\", \"Average per Testcase\"]\n\t\tfunction_risk = 0\n\t\tif not self.check_minimum_risk(function_risk, title):\n\t\t\treturn False\n\n\t\tif output:\n\t\t\tself.settings['logger'].info(title)\n\n\t\ttotal = 0\n\t\trows = []\n\t\tif self.count_results is None:\n\t\t\tself.count_results = self.settings['db'].count_results(0, None)\n\t\tresults = self.settings['db'].analyze_elapsed()\n\t\tfor result in results:\n\t\t\tif toplimit is not None and len(rows) >= toplimit:\n\t\t\t\tbreak\n\t\t\trows.append([[result[0], result[1], result[2], str(datetime.timedelta(seconds=int(result[3]))), str(round(result[3] / self.count_results, 5))]])\n\t\t\ttotal += result[3]\n\n\t\tself.dump.general(output, title, columns, rows)\n\t\treturn total\n\n\ndef help(err=\"\"):\n\t\"\"\"Print a help screen and exit\"\"\"\n\tif err:\n\t\tprint(\"Error: %s\\n\" % err)\n\tprint(\"Syntax: \")\n\tprint(os.path.basename(__file__) + \" -d db.sqlite Choose the database\")\n\tprint(\"\\t\\t [-D] Debug information\")\n\tprint(\"\\t\\t [-m methodName] Method: report (default), analyze_stdout, analyze_specific_return_code, etc\")\n\tprint(\"\\t\\t [-e extra_parameter] Extra parameter used when specifying a for certain methodName (ie, analyze_username_disclosure)\")\n\tprint(\"\\t\\t [-o html] Output: html (default), txt or csv.\")\n\tprint(\"\\t\\t [-l 20] Top limit results (default: 20)\")\n\tprint(\"\\t\\t [-r 3] Minimum risk (0:informational, 1:low, 2:medium, 3:high (default)\")\n\tsys.exit()\n\n\ndef main():\n\t\"\"\"Analyze potential vulnerabilities on a database fuzzing session\"\"\"\n\ttry:\n\t\topts, args = getopt.getopt(sys.argv[1:], \"hd:De:m:o:pl:r:\", [\"help\", \"database=\", \"extra=\", \"method=\", \"output=\", \"limit=\", \"risk=\"])\n\texcept getopt.GetoptError as err:\n\t\thelp(err)\n\n\tsettings = {}\n\tmethod = \"report\" # default method name\n\ttoplimit = 20 # default top limit\n\textra = None\n\tfor o, a in opts:\n\t\tif o in (\"-d\", \"--database\"):\n\t\t\tif os.path.isfile(a):\n\t\t\t\tsettings['db_file'] = a\n\t\t\telse:\n\t\t\t\thelp(\"Database should be a valid file.\")\n\t\telif o in (\"-D\"):\n\t\t\tsettings['loglevel'] = 'debug'\n\t\telif o in (\"-e\", \"--extra\"):\n\t\t\textra = a\n\t\telif o in (\"-h\", \"--help\"):\n\t\t\thelp()\n\t\telif o in (\"-l\", \"--limit\"):\n\t\t\ttry:\n\t\t\t\ttoplimit = int(a)\n\t\t\texcept ValueError:\n\t\t\t\thelp(\"Top limit should be an integer.\")\n\t\telif o in (\"-m\", \"--method\"):\n\t\t\tmethod = a\n\t\telif o in (\"-o\", \"--output\"):\n\t\t\tsettings[\"output_type\"] = a\n\t\telif o in (\"-p\"):\n\t\t\tsettings[\"print_risk\"] = True\n\t\telif o in (\"-r\", \"--risk\"):\n\t\t\ttry:\n\t\t\t\tsettings[\"minimum_risk\"] = int(a)\n\t\t\texcept ValueError:\n\t\t\t\thelp(\"Risk should be an integer.\")\n\n\tif 'db_file' not in settings:\n\t\thelp(\"The database was not specified.\")\n\telif 'db_file' not in settings and 'print_risk' not in settings:\n\t\thelp(\"The database was not specified and the only functionality without a database -p was not selected. \")\n\tsettings = classes.settings.load_settings(settings)\n\tif settings['db'].db_connection:\n\t\tanalyze = Analyze(settings)\n\t\tanalyze.dump_results(method, toplimit, extra)\n\n\nif __name__ == \"__main__\":\n\tmain()\n\t# profile.run('analyze.dump_results(method, toplimit)')\n"
},
{
"path": "xdiff_dbaction.py",
"content": "#!/usr/bin/env python\n#\n# Copyright (C) 2018 Fernando Arnaboldi\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n#\nfrom __future__ import print_function\nimport getopt\nimport itertools\nimport os.path\nimport sys\nimport time\nimport classes.settings\nimport classes.compat\nfrom classes.dump import Dump\nfrom classes.dbsqlite import DbSqlite\n\n\nclass Dbaction(object):\n\t\"\"\"Do stuff with the fuzzer's databases: copy databases, print tables, insert stuff and generate testcases\"\"\"\n\tdef __init__(self, settings):\n\t\tself.settings = settings\n\t\tif 'max_permutation' not in self.settings:\n\t\t\tself.settings['max_permutation'] = 5\n\t\tif 'generate_type' not in self.settings:\n\t\t\tself.settings['generate_type'] = 2\n\n\tdef print_table(self, fromdb, table, output_type):\n\t\t\"\"\"Print all the conents of a table\"\"\"\n\t\tif table is None:\n\t\t\tself.settings['logger'].error(\"You must select a table.\")\n\t\telse:\n\t\t\tself.settings['output_file'] = None\n\t\t\tself.settings['db'] = DbSqlite(self.settings, fromdb)\n\t\t\tcolumns = self.settings['db'].get_columns(table)\n\t\t\trows = self.settings['db'].get_rows(table)\n\t\t\tif columns:\n\t\t\t\tdump = Dump(self.settings)\n\t\t\t\tdump.general(output_type, table, columns, [rows])\n\t\t\telse:\n\t\t\t\tself.print_valid_tables(table)\n\n\tdef insert_table(self, fromdb, table, separator, insert):\n\t\t\"\"\"Insert a row into a table\"\"\"\n\t\tif table is None:\n\t\t\tself.settings['logger'].error(\"You must select a table.\")\n\t\telse:\n\t\t\tif not insert:\n\t\t\t\tself.settings['logger'].error(\"There are no values to be inserted\")\n\t\t\telse:\n\t\t\t\tself.settings['db'] = DbSqlite(self.settings, fromdb)\n\t\t\t\tcolumns = self.settings['db'].get_columns(table)\n\t\t\t\tif columns:\n\t\t\t\t\t# If the user supplied one value less than the one required and the first column is called id, just ignore that column..\n\t\t\t\t\tif len(columns) == (len(insert.split(separator)) + 1) and columns[0] == 'id':\n\t\t\t\t\t\tdel columns[0]\n\t\t\t\t\tif len(columns) != len(insert.split(separator)):\n\t\t\t\t\t\tprint(\"The table '\" + table + \"' has \" + str(len(columns)) + \" columns: \" + str(columns) + \". However, you want to insert \" + str(len(insert.split(separator))) + \" value/s: \" + str(insert.split(separator)) + \". It doesn't work like that.\")\n\t\t\t\t\telse:\n\t\t\t\t\t\tself.settings['db'].insert_row(table, columns, insert.split(separator))\n\t\t\t\telse:\n\t\t\t\t\tself.print_valid_tables(table)\n\n\tdef print_valid_tables(self, table=None):\n\t\t\"\"\"Provide information on what are the valid tables\"\"\"\n\t\tif table:\n\t\t\tself.settings['logger'].error(\"Error: table '%s' not found\" % table)\n\t\telse:\n\t\t\tif self.output_type:\n\t\t\t\tprint(\"Valid table names:\")\n\t\t\t\tprint(\"- fuzz_testcase: contains the inputs to be sent to the software. You can define an input in 'function' and potential values in 'value' and generate the combinations on this table.\")\n\t\t\t\tprint(\"- function: contains what you want to fuzz. The special keyword [[test]] gets replaced by the values contained in the table 'value'. Ie, if you want to fuzz the 'print()'' function, you want to write in here 'print([[test]])'.\")\n\t\t\t\tprint(\"- value: contains the items that will replace the [[test]] values in the 'function' table\")\n\t\t\t\tprint(\"\")\n\t\t\t\tprint(\"Valid tables generated by XDiFF:\")\n\t\t\t\tprint(\"- fuzz_software: contains the software defined in software.ini\")\n\t\t\t\tprint(\"- fuzz_testcase_result: contains the result of executing the software defined in 'fuzz_software' with the input defined in 'fuzz_testcase'\")\n\t\t\t\tprint(\"- fuzz_constants: contains internal constant values used by the fuzzer\")\n\n\tdef permute(self, functions, values):\n\t\t\"\"\"Perform a permutation between the two lists received (functions & values)\"\"\"\n\t\ttotal = 0\n\t\tif not functions:\n\t\t\tself.settings['logger'].error(\"There are no functions to permute\")\n\t\telif not values:\n\t\t\tself.settings['logger'].error(\"There are no values to permute\")\n\t\telse:\n\t\t\t# Prioritize the lower count injections\n\t\t\tfor count in range(0, self.settings['max_permutation'] + 1):\n\t\t\t\t# Give a heads up of how many testcases will be generated\n\t\t\t\tsubtotal = 0\n\t\t\t\tcountfunctions = functions\n\t\t\t\tfor function in countfunctions:\n\t\t\t\t\tif isinstance(function, tuple):\n\t\t\t\t\t\tif len(function) == 1:\n\t\t\t\t\t\t\tfunction = function[0] # when it is generated by random testcases (classes/fuzzer.py)\n\t\t\t\t\t\telif len(function) == 2:\n\t\t\t\t\t\t\tfunction = function[1] # when it is read from the database\n\t\t\t\t\tif function is not None and count == function.count(\"[[test]]\"):\n\t\t\t\t\t\tsubtotal += 1\n\t\t\t\tself.settings['logger'].debug(\"Testcases generation: %s entry points, %s testcases to be generated.\" % (str(count), str(subtotal)))\n\t\t\t\t# Generate the testcases\n\t\t\t\tfor function in functions:\n\t\t\t\t\tif len(function) == 1:\n\t\t\t\t\t\tfunction = function[0] # when it is generated by random testcases (classes/fuzzer.py)\n\t\t\t\t\telif len(function) == 2:\n\t\t\t\t\t\tfunction = function[1] # when it is read from the database\n\t\t\t\t\tif function is not None and count == function.count(\"[[test]]\"):\n\t\t\t\t\t\ttestcases, total = self.permute_values(values, function, total)\n\t\t\t\t\t\tself.settings['db'].set_testcase(testcases)\n\t\treturn total\n\n\tdef permute_values(self, values, function, total):\n\t\t\"\"\"Perform a permutation between the values and the functions received based on the generate_type received\"\"\"\n\t\ttestcases = []\n\t\tfunction_tuple = function\n\n\t\t# There are no values, only functions:\n\t\tif not values:\n\t\t\ttestcases.append((classes.compat.unicode(function_tuple),))\n\t\telse:\n\t\t\tif self.settings['generate_type'] == 1:\n\t\t\t\t# Permute\n\t\t\t\tfor valuetuple in itertools.product(values, repeat=function_tuple.count(\"[[test]]\")):\n\t\t\t\t\ttotal += 1\n\t\t\t\t\tfor value in valuetuple:\n\t\t\t\t\t\t# unicode values are tuples\n\t\t\t\t\t\tif isinstance(valuetuple, tuple):\n\t\t\t\t\t\t\tvalue = value[0]\n\t\t\t\t\t\tvalue = value.replace('[[id]]', str(total))\n\t\t\t\t\t\tfunction_tuple = function_tuple.replace(\"[[test]]\", value, 1)\n\t\t\t\t\ttestcases.append((classes.compat.unicode(function_tuple),))\n\t\t\t\t\tfunction_tuple = function # reset to the original value\n\t\t\telif self.settings['generate_type'] == 2:\n\t\t\t\t# Do not permute, just replace\n\t\t\t\tfor value in values:\n\t\t\t\t\tif isinstance(value, tuple):\n\t\t\t\t\t\tvalue = value[0]\n\t\t\t\t\ttotal += 1\n\t\t\t\t\tvalue = value.replace('[[id]]', str(total))\n\t\t\t\t\tfunction_tuple = function_tuple.replace('[[test]]', value)\n\t\t\t\t\ttestcases.append((classes.compat.unicode(function_tuple),))\n\t\t\t\t\tfunction_tuple = function # reset to the original value\n\t\t\telif self.settings['generate_type'] == 3:\n\t\t\t\t# Do not permute, replace but also include testcases with less parameters\n\t\t\t\tif (function.count(\"[[test]]\")) > 1:\n\t\t\t\t\tfor tests in range(1, function.count(\"[[test]]\") + 1):\n\t\t\t\t\t\tfor value in values:\n\t\t\t\t\t\t\tif isinstance(value, tuple):\n\t\t\t\t\t\t\t\tvalue = value[0]\n\t\t\t\t\t\t\ttotal += 1\n\t\t\t\t\t\t\tvalue = value.replace('[[id]]', str(total))\n\t\t\t\t\t\t\tfunction_tuple = function_tuple.replace('[[test]]', value)\n\t\t\t\t\t\t\ttestcases.append((classes.compat.unicode(function_tuple),))\n\t\t\t\t\t\t\tfunction_tuple = function # reset to the original value\n\t\t\t\t\t\tfunction_tuple = function = function.replace(',[[test]]', '', 1)\n\t\t\telse:\n\t\t\t\tprint(\"Error: the permutation type does not exist\")\n\t\t\t\tsys.exit()\n\n\t\treturn testcases, total\n\n\tdef generate(self, fromdb):\n\t\t\"\"\"Generate the testcases with a permutation of values and functions\"\"\"\n\t\tstart_time = time.time()\n\t\tself.settings['db'] = DbSqlite(self.settings, fromdb)\n\t\tif self.settings['db'].db_connection:\n\t\t\tself.settings['db'].create_table()\n\t\t\tvalues = self.settings['db'].get_values()\n\t\t\tfunctions = self.settings['db'].get_functions()\n\t\t\tself.settings['logger'].info(\"Values: %s - Functions: %s\" % (str(len(values)), str(len(functions))))\n\t\t\ttotal = self.permute(functions, values)\n\n\t\t\tself.settings['db'].commit()\n\t\t\tfinish_time = time.time() - start_time\n\t\t\tself.settings['logger'].info(\"Testcases generated: %s\" % str(total))\n\t\t\tself.settings['logger'].info(\"Time required: %s seconds\" % str(round(finish_time, 2)))\n\n\tdef migrate(self, fromdb, todb):\n\t\t\"\"\"Migrates tables from one database ('dbfrom') to another database ('dbto')\"\"\"\n\t\tstart_time = time.time()\n\t\tself.settings['dbfrom'] = DbSqlite(self.settings, fromdb)\n\t\tself.settings['dbto'] = DbSqlite(self.settings, todb)\n\n\t\tif self.settings['dbfrom'].db_connection and self.settings['dbto'].db_connection:\n\t\t\tself.settings['dbto'].create_table()\n\n\t\t\tvalues = self.settings['dbfrom'].get_values()\n\t\t\tself.settings['dbto'].set_values(values)\n\n\t\t\tfunctions = self.settings['dbfrom'].get_functions()\n\t\t\tself.settings['dbto'].set_functions(functions)\n\n\t\t\tself.settings['dbto'].commit()\n\n\t\t\tfinish_time = time.time() - start_time\n\t\t\tself.settings['logger'].info(\"Finished, time elapsed %s seconds\" % str(finish_time)[:5])\n\n\ndef help(err=None):\n\t\"\"\"Print a help screen and exit\"\"\"\n\tif err:\n\t\tprint(\"Error: %s\\n\" % str(err))\n\tprint(\"Syntax: \")\n\tprint(os.path.basename(__file__) + \" -d db.sqlite -D fuzz.db Migrate values and functions to another database\")\n\tprint(\"\\t\\t -d fuzz.db -g 1 [-m 5] Generate testcases permuting values and functions (set to maximum 5 input test cases)\")\n\tprint(\"\\t\\t -d fuzz.db -g 2 [-m 5] Generate testcases replacing values in functions (set to max..)\")\n\tprint(\"\\t\\t -d fuzz.db -g 3 [-m 5] Generate testcases replacing values in functions including testcases with less parameters (set to max..)\")\n\tprint(\"\\t\\t -d fuzz.db -t table -p Print a database table: fuzz_software, fuzz_testcase, value, function)\")\n\tprint(\"\\t\\t -d fuzz.db -t table [-s,] -i \\\"foo\\\" Insert foo into table (optional field separator -s uses a comma)\")\n\tsys.exit()\n\n\ndef main():\n\t\"\"\"Perform multiple database actions\"\"\"\n\ttry:\n\t\topts, args = getopt.getopt(sys.argv[1:], \"hd:D:g:i:m:ps:t:\", [\"help\", \"database=\", \"Database=\", \"generate=\", \"insert=\", \"maximum=\", \"print\", \"separator=\", \"table=\"])\n\texcept getopt.GetoptError as err:\n\t\thelp(err)\n\n\tsettings = {}\n\tsettings['output_type'] = 'txt'\n\tfromdb = None\n\ttodb = None\n\ttable = None\n\taction = None\n\tseparator = \",\"\n\n\tfor o, a in opts:\n\t\tif o in (\"-h\", \"--help\"):\n\t\t\thelp()\n\t\telif o in (\"-d\", \"--database\"):\n\t\t\tfromdb = a\n\t\t\tif os.path.isfile(fromdb):\n\t\t\t\tsettings['db_file'] = fromdb\n\t\t\telse:\n\t\t\t\thelp(\"The database selected '%s' is not a valid file.\" % a)\n\t\telif o in (\"-D\", \"--Database\"):\n\t\t\ttodb = a\n\t\t\taction = \"migrate\"\n\t\t\tbreak\n\t\telif o in (\"-g\", \"--generate\"):\n\t\t\taction = \"generate\"\n\t\t\ttry:\n\t\t\t\tsettings['generate_type'] = int(a)\n\t\t\texcept:\n\t\t\t\thelp(\"The generate parameter should be a number\")\n\t\telif o in (\"-i\", \"--insert\"):\n\t\t\taction = \"insert\"\n\t\t\tinsert = classes.compat.unicode(str(a), errors='ignore')\n\t\telif o in (\"-m\", \"--maximum\"):\n\t\t\ttry:\n\t\t\t\tsettings['max_permutation'] = int(a)\n\t\t\texcept ValueError:\n\t\t\t\thelp(\"The max permutation parameter should be a number\")\n\t\telif o in (\"-p\", \"--print\"):\n\t\t\taction = \"print\"\n\t\telif o in (\"-s\", \"--separator\"):\n\t\t\tseparator = a\n\t\telif o in (\"-t\", \"--table\"):\n\t\t\ttable = a\n\n\tif not fromdb:\n\t\thelp(\"The database was not specified.\")\n\n\tsettings = classes.settings.load_settings(settings)\n\tdbaction = Dbaction(settings)\n\tif action == \"migrate\":\n\t\tdbaction.migrate(fromdb, todb)\n\telif action == \"generate\":\n\t\tif todb is not None:\n\t\t\tfromdb = todb\n\t\tdbaction.generate(fromdb)\n\telif action == \"print\":\n\t\tdbaction.print_table(fromdb, table, settings['output_type'])\n\telif action == \"insert\":\n\t\tdbaction.insert_table(fromdb, table, separator, insert)\n\telse:\n\t\thelp(\"You must select an action: migrate, generate, print or insert.\")\n\n\nif __name__ == \"__main__\":\n\tmain()\n"
},
{
"path": "xdiff_run.py",
"content": "#!/usr/bin/env python\n#\n# Copyright (C) 2018 Fernando Arnaboldi\n#\n# This program is free software: you can redistribute it and/or modify\n# it under the terms of the GNU General Public License as published by\n# the Free Software Foundation, either version 3 of the License, or\n# (at your option) any later version.\n#\n# This program is distributed in the hope that it will be useful,\n# but WITHOUT ANY WARRANTY; without even the implied warranty of\n# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n# GNU General Public License for more details.\n#\n# You should have received a copy of the GNU General Public License\n# along with this program. If not, see .\n#\nfrom __future__ import print_function\nimport getopt\nimport os\nimport signal\nimport sys\nimport time\nimport classes.settings\n\n\ndef dfuzz(settings):\n\t\"\"\"Fuzz something based on he settings received\"\"\"\n\tif 'fuzz_category' not in settings:\n\t\thelp(\"The category was not specified.\")\n\n\tsettings = classes.settings.load_settings(settings)\t # load the fuzzer settings\n\tif not settings:\n\t\treturn False\n\n\tif not settings['software']:\n\t\thelp(\"There is no software associated to the category selected\")\n\n\tif not settings['queue'].chdir_tmp():\n\t\treturn False\n\n\tbanner = \"Starting Fuzzer v%s\" % str(settings['version'])\n\tsettings['logger'].info(len(banner) * \"-\")\n\tsettings['logger'].info(banner)\n\tsettings['logger'].info(len(banner) * \"-\")\n\tfor key in sorted(settings.iterkeys()):\n\t\tsettings['logger'].debug(\"Setting %s: %s\" % (key, str(settings[key])))\n\n\tsettings['queue'].start_web_server() # load the webserver\n\tsettings['monitor'].check_once() # check before start if the canaries are in place\n\tsettings['db'].optimize()\n\ttotal_testcases = settings['db'].count_testcases()\n\tcurrent_test = settings['db'].get_latest_id(settings['software'])\n\tsettings['logger'].info(\"Setting testcases: %s/%s\" % (str(current_test), str(total_testcases)))\n\n\telapsed_time = 0\n\ttest_count = 0\n\twhile True:\n\t\tstart_time = time.time()\n\t\ttests = settings['db'].get_test(current_test, settings['db_tests'])\n\t\tif not tests:\n\t\t\tsettings['logger'].info(\"Terminated: no more testcases\")\n\t\t\tbreak\n\t\tdbinput = settings['queue'].fuzz(tests)\n\t\tsettings['monitor'].check() # check free space before saving results\n\t\tsaved, size = settings['db'].set_results(dbinput)\n\t\tfinish_time = (time.time() - start_time)\n\t\telapsed_time += finish_time # Total time elapsed testing\n\t\tremaining_tests = total_testcases - (current_test + settings['db_tests']) # Tests left\n\t\ttest_count += settings['db_tests']\n\t\trate = test_count / elapsed_time # Rate per second\n\t\ttime_left = remaining_tests / rate / 60 # How many hours are left ?\n\t\tsettings['logger'].info(\"Tests \" + str(current_test) + \"-\" + str(current_test + settings['db_tests']) + \" - Set \" + str(saved) + \" (\" + str(int(size / 1024)) + \" kb) - Took \" + str(int(finish_time)) + \"s - Avg Rate \" + str(int(rate) * len(settings['software'])) + \" - ETC \" + str(int(time_left)) + \"'\")\n\t\tcurrent_test += settings['db_tests']\n\t\t# break # uncomment if you want to run just one cycle of the fuzzer for debugging purposes\n\tsettings['queue'].stop_web_server()\n\n\ndef help(err=\"\"):\n\t\"\"\"Print a help screen and exit\"\"\"\n\tif err:\n\t\tprint(\"Error: %s\\n\" % err)\n\tprint(\"XDiFF Syntax: \")\n\tprint(os.path.basename(__file__) + \" -d db.sqlite Choose the database\")\n\tprint(\"\\t -c Python Software category to be fuzzed\")\n\tprint(\"\\t [-D] Print debugging information\")\n\tprint(\"\\t [-r 0] Random inputs: radamsa & zzuf without newlines (faster)\")\n\tprint(\"\\t [-r 1] Random inputs: radamsa & zzuf with newlines (slower)\")\n\tprint(\"\\t [-r 2] Random inputs: radamsa without newlines (faster)\")\n\tprint(\"\\t [-r 3] Random inputs: radamsa with newlines (slower)\")\n\tprint(\"\\t [-r 4] Random inputs: zzuf without newlines (faster)\")\n\tprint(\"\\t [-r 5] Random inputs: zzuf with newlines (slower)\")\n\tprint(\"\\t [-s software.ini] Configuration file for software to be fuzzed\")\n\tprint(\"\\t [-t 100] Threads executed in parallel\")\n\tprint(\"\\t [-T 10] Timeout per thread\")\n\tprint(\"\\t [-v] Use valgrind\")\n\tsys.exit()\n\n\ndef main():\n\t\"\"\"Fuzz something FFS!\"\"\"\n\tdef signal_handler(signal, frame):\n\t\t\"\"\"Catch SIGINT and do some cleaning before termination\"\"\"\n\t\tsettings['monitor'].remove_stuff()\n\t\tsettings['queue'].stop_web_server()\n\t\tsettings['logger'].info(\"Program terminated\")\n\t\tsys.exit(1)\n\tsignal.signal(signal.SIGINT, signal_handler)\n\n\ttry:\n\t\topts, args = getopt.getopt(sys.argv[1:], \"hc:d:Dr:s:t:T:v\", [\"help\", \"category=\", \"database=\", \"random=\", \"software=\", \"threads=\", \"timeout=\", \"valgrind\"])\n\texcept getopt.GetoptError as err:\n\t\thelp(err)\n\tsettings = {}\n\tfor o, a in opts:\n\t\tif o in (\"-h\", \"--help\"):\n\t\t\thelp()\n\t\telif o in (\"-c\", \"--category\"):\n\t\t\tsettings['fuzz_category'] = a\n\t\telif o in (\"-d\", \"--database\"):\n\t\t\tsettings['db_file'] = os.path.abspath(a)\n\t\telif o in (\"-D\"):\n\t\t\tsettings['loglevel'] = 'debug'\n\t\telif o in (\"-r\", \"--random\"):\n\t\t\tsettings['generate_tests'] = int(a)\n\t\telif o in (\"-s\", \"--software\"):\n\t\t\tsettings['software'] = os.path.abspath(a)\n\t\telif o in (\"-t\", \"--threads\"):\n\t\t\tsettings['db_tests'] = int(a)\n\t\telif o in (\"-T\", \"--timeout\"):\n\t\t\tsettings['timeout'] = int(a)\n\t\telif o in (\"-v\", \"--valgrind\"):\n\t\t\tsettings['valgrind'] = True\n\n\tif \"db_file\" not in settings or \"fuzz_category\" not in settings:\n\t\thelp(\"The -d and -c parameters are mandatory\")\n\telse:\n\t\tdfuzz(settings)\n\n\nif __name__ == \"__main__\":\n\tmain()\n"
}
]