[
  {
    "path": "README.md",
    "content": "MalShare Toolkit\n==============\n\n\nSet of tools for interacting with MalShare API\nTools:\nmalshare_digest \n--------------\n*Generate a CSV file of daily MD5 list*\n\n\tusage: malshare_digest [-h] -o OUTFILE\n\tmalshare_digest: error: argument -o/--outfile is required\n\n\nwget_malshare\n--------------\n*Download sample*\n\n\tusage: wget_malshare [-h] [-k APIKEY] -d DOWNLOAD [-xVXCAGE]\n\twget_malshare: error: argument -d/--download is required\n\nwget_malshare_daily\n--------------\n*Download all samples from the day prior*\n\n        usage: wget_malshare_daily [-h] [-k APIKEY] -d DOWNLOAD [-x VXCAGE]\n        wget_malshare_daily: error: argument -d/--download is required\n\n\n\t\t \nDon't forget to set your API Key for wget_malshare && wget_malshare_daily\n"
  },
  {
    "path": "malshare_api",
    "content": "#!/usr/bin/env python\n# Copyright (C) 2013 - 2016 Malshare Developers.\n# Multitool for MalShare API\n\nimport os\nimport re\nimport json\nimport argparse\nimport requests\n\nBASE_HTTP_PATH = \"http://malshare.com/\"\nAPI_PATHS = {\n\t\t\t\t\"MD5LIST\" : \"api.php?api_key=%s&action=getlistraw\",\n\t\t\t\t\"SOURCES\" : \"api.php?api_key=%s&action=getsourcesraw\",\n\n\t\t\t\t\"DOWNLOAD\" : \"api.php?api_key=%s&action=getfile&hash=%s\",\n\t\t\t\t\"DETAILS\" : \"api.php?api_key=%s&action=details&hash=%s\",\n\n\t\t\t\t\"TYPE\" : \"api.php?api_key=%s&action=type&type=%s\",\n\t\t\t}\n\napi_key = \"\"\n\ndef main():\n\tlargs = parse_args()\n\n\tif largs['details']:\n\t\turi = API_PATHS['DETAILS'] % (api_key, largs['details'])\n\t\tr = api_call(uri)\n\t\tif r is not None:\n\t\t\tdetails = r.json()\n\t\t\tprint json.dumps(details, indent=4, sort_keys=True)\n\n\n\telif largs['download']:\n\t\turi = API_PATHS['DOWNLOAD'] % (api_key, largs['download'])\n\t\tr = api_call(uri)\n\t\tif r is not None:\n\t\t\ttry:\n\t\t\t\twith open(str(largs['download']) + \".malshare\", 'wb') as f:\n\t\t\t\t\tf.write( r.content )\n\t\t\texcept Exception, e:\n\t\t\t\tprint \"[X] Problem saving file\"\n\t\t\t\tprint \"[E] %s\" % e\n\n\telif largs['type']:\n\t\turi = API_PATHS['TYPE'] % (api_key, largs['type'])\n                r = api_call(uri)\n                if r is not None:\n\t\t\tfor rhash in set(r.json()):\n\t\t\t\tprint rhash\t\n\n\telif largs['listmd5']:\n\t\turi = API_PATHS['MD5LIST'] % (api_key)\n                r = api_call(uri)\n\t\tprint r.text.strip()\n\n\n\telif largs['listsources']:\n\t\turi = API_PATHS['SOURCES'] % (api_key)\n                r = api_call(uri)\n\t\tprint r.text.strip()\n\ndef api_call(rpath):\n\tglobal api_key\n\ttry:\n\t\tuser_agent = {'User-Agent': 'MalShare API Tool v/0.1 beta'}\n\t\tr = requests.get(BASE_HTTP_PATH + rpath, headers=user_agent)\n\t\t\t\n\t\tif r.status_code == 200:\n\t\t\tif standard_error_check(r.content):\n\t\t\t\treturn r\n\t\telse:\n\t\t\tif standard_error_check(r.content):\n\t\t\t\tprint \"[X] API Call Failed\"\n\t\t\t\treturn None\n\t\t\telse:\n\t\t\t\treturn None\n\t\t\t\n\n\n\texcept Exception, e:\n\t\tprint \"[X] API Call Failed: %s\" % e\n\t\treturn None\n\n\ndef standard_error_check(rtext):\n\tif (rtext == \"Sample not found\"):\n\t\tprint \"[X] Sample not Found\"\n\t\treturn False\n\n\tif (rtext == \"ERROR! => Account not activated\"):\n\t\tprint \"[X] Bad API Key\"\n\t\treturn False\t\n\n        if (rtext == \"Invalid Hash\"):\n                print \"[X] Invalid Hash\"\n                return False\n\n        if ( \"Sample not found by hash\" in rtext ):\n                print \"[X] Hash not found\"\n                return False\n\n\treturn True\n\n\n\ndef parse_args():\n\tglobal api_key\n\tparser = argparse.ArgumentParser()\n\tparser.add_argument(\"-m\", \"--listmd5\", help=\"Pull MD5 List\", required=False, action='store_true')\n\tparser.add_argument(\"-s\", \"--listsources\", help=\"Pull MD5 List\", required=False, action='store_true')\n\n\tparser.add_argument(\"-d\", \"--download\", help=\"Download File by Hash\", required=False)\n\tparser.add_argument(\"-l\", \"--details\", help=\"List File Details\", required=False)\n\tparser.add_argument(\"-t\", \"--type\", help=\"Search For Daily files by Type\", required=False)\n\n\tparser.add_argument(\"-a\", \"--apikey\", help=\"Set API key for session\", required=False)\n\n\n\targs = parser.parse_args()\n\tif stored_api_check() == False:\n\t\tif args.apikey:\n\t\t\tapi_key = args.apikey\n\n\treturn vars(args)\n\n# Read ~/.malshare and read the first line.  This file only needs the API string in it.\ndef stored_api_check():\n\tglobal api_key\n\ttry:\n\t\tif ( os.path.exists(os.path.expanduser('~') + '/.malshare' ) ): \n\t\t\twith open( os.path.expanduser('~') + '/.malshare' ) as handle_api_file:\n\t\t\t\tapi_key = func_parse_api_key(handle_api_file.readlines())\n\t\t\treturn True\n\t\telif (  os.path.exists('.malshare' ) ): \n\t\t\twith open( '.malshare' ) as handle_api_file:\n\t\t\t\tapi_key = func_parse_api_key(handle_api_file.readlines())\n\t\treturn True\n\texcept IOError:\n\t\tpass\n\treturn False\n\n# Parse the API key and exit if the API key contains any non [A-Za-z0-9]+\ndef func_parse_api_key(lst_tmp_key):\n\tstr_tmp_key = \"\".join(lst_tmp_key).rstrip()\n\tif re.match(\"^[A-Za-z0-9]+$\", str_tmp_key): \n\t\treturn str_tmp_key\n\nif __name__ == \"__main__\":\n\tmain()\n\n"
  },
  {
    "path": "malshare_digest",
    "content": "#! /usr/bin/env python\n# Copyright (C) 2013 Malshare Developers.\n# Written by Blevene github.com/Blevene \n\n# Open and catalog list of samples observed for a given date\n# on Malshare.com, location: http://www.malshare.com/daily/malshare.current.txt\n# To Do:\n# [x] Create output file (csv or txt?)\n# [x] Store each md5 as a csv with 'hash' : 'date' mapping\n\nimport urllib2\nimport argparse\nimport csv\nfrom sys import argv\nfrom datetime import datetime, date\n\ndef main():\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"-o\", \"--outfile\", help=\"Pull the most recent Malshare digest\", required=True)\n    args = parser.parse_args()\n    \n    mal_digest = urllib2.urlopen(url='http://www.malshare.com/daily/malshare.current.txt')\n    mal_digest = list(mal_digest)\n    pull_time = [str(date.today())] * len(mal_digest) \n    strip_list = [x.strip('\\n') for x in mal_digest]\n    dictionary = dict(zip(strip_list, pull_time))\n    outfile_name = args.outfile + '.csv'\n    writer = csv.writer(open( outfile_name , 'a'))\n    for key, value in dictionary.items():\n        if (key):\n            writer.writerow([key, value])\n\nif __name__ == \"__main__\":\n    try:\n        main()\n    except KeyboardInterrupt:\n        sys.exit(\" [X] Shutting Down\")\n\n\n\n\n"
  },
  {
    "path": "malshare_download_list.py",
    "content": "#! /usr/bin/env python\n# Download files with given hashes.json file\n\nimport argparse\nimport json\nimport logging\nimport os\nimport sys\nfrom multiprocessing.pool import Pool\n\nimport requests\nimport tqdm\n\napi_key = \"<API_KEY>\"\n\nlogging.basicConfig(format = '%(asctime)s %(levelname)s:%(message)s', level = logging.WARNING)\n\n\ndef download_file_by_hash(file_hash):\n    logging.debug(\"Downloading {}\".format(file_hash))\n    try:\n        malshare_url = \"http://malshare.com/sampleshare.php\"\n        payload = {'action': 'getfile', 'api_key': api_key, 'hash': file_hash}\n        user_agent = {'User-agent': 'wget_malshare daily 1.0'}\n\n        r = requests.get(malshare_url, params = payload, headers = user_agent)\n        sample = r.content\n\n        if sample == \"Sample not found\":\n            logging.error(\"Sample not Found\")\n            return None\n        if sample == \"ERROR! => Account not activated\":\n            logging.error(\"Bad API Key\")\n            return None\n\n        with open(os.path.join(\"files\", file_hash), mode = \"wb\") as fh:\n            fh.write(sample)\n            logging.info(\"{} saved to files\".format(file_hash))\n\n    except Exception as e:\n        logging.error(\"download_file_by_hash: Problem connecting. Please Try again.\")\n        logging.exception(sys.exc_info())\n        logging.exception(type(e))\n        logging.exception(e.args)\n        logging.exception(e)\n        sys.exit(1)\n\n\ndef download_list(api_k, hash_list):\n    global api_key\n    if api_k:\n        api_key = api_k\n    files = json.load(open(hash_list))\n    pool = Pool(os.cpu_count())\n    for _ in tqdm.tqdm(pool.imap_unordered(download_file_by_hash, files), total = len(files)):\n        pass\n\n\nif __name__ == \"__main__\":\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"-k\", \"--apikey\", help = \"API Key\", required = False)\n    parser.add_argument(\"-f\", \"--hash_list\", help = \"File containing list of hashes in json format\", required = True)\n    args = parser.parse_args()\n    download_list(args.apikey, args.hash_list)\n"
  },
  {
    "path": "malshare_search",
    "content": "#!/usr/bin/env python\n# Copyright (C) 2013 - 2018 Malshare Developers.\n# Written by Silas Cutler\n# Quick search tool for  MalShare API\n\nimport sys\nimport json\nimport requests\n\napi_key = \"\"\nif api_key != \"\":\n\turl=\"https://malshare.com/api.php?api_key=%s&action=search&query=%s\" % (api_key, sys.argv[1] )\n\tr = requests.get(url)\n\tprint json.dumps(r.json(), sort_keys=True, indent=4, separators=(',', ': '))\nelse:\n\tprint \"Please set API key\"\n\n"
  },
  {
    "path": "malshare_upload",
    "content": "#!/bin/bash\n# Copyright (C) 2013 - 2017 Malshare Developers.\n# Written by Silas Cutler\n# Upload tool for  MalShare API\n\n\n\nAPI_KEY=\"\"\n\nfunction usage(){\n\techo \"$0 <File to Upload>\"\n\techo \" - File Uploader for MalShare.com\"\n}\n\n\nif [ -z \"$API_KEY\" ]\nthen\n\tusage\n\techo \"\"\n\techo \"Please set API Key in script\"\n\texit 1\nfi\n\nif [ ! -f \"$1\" ]\nthen\n\tusage\n\techo \"\"\n\techo \" [x] Please specify file to upload\"\n\texit 1\nfi\n\nRES=$(curl -s -X POST -F \"upload=@$1\" \"https://malshare.com/api.php?api_key=$API_KEY&action=upload\")\necho $RES\n\nif [ \"$RES\" = \"Success\" ]\nthen\n\tMD5HASH=$(md5sum $1 | awk '{ print $1 }' )\n\techo \"Sample can be viewed at https://malshare.com/sample.php?action=detail&hash=$MD5HASH\"\nelse\n\techo \"Problem with upload\"\nfi\n\n"
  },
  {
    "path": "wget_malshare",
    "content": "#! /usr/bin/env python\n# Copyright (C) 2013 - 2016 Malshare Developers.\n# Pull sample by [MD5 | SHA1 | SHA256] Hash\n\nimport os\nimport re\nimport sys\nimport logging\nimport requests\nimport argparse\n\napi_key =\"\"\n\ndef main():\n    global api_key\n\n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"-k\", \"--apikey\", help=\"API Key\", required=False)\n    parser.add_argument(\"-d\", \"--download\", help=\"Search / Download Hash\", required=True)\n    parser.add_argument(\"-x\", \"--vxcage\", help=\"VXCage server\", required=False)\n\n    args = parser.parse_args()\n    if stored_api_check() == False:\n        if args.apikey:\n            api_key = args.apikey\n\n    if (not api_key):\n        logging.error(\"API Key not entered\")\n        sys.exit(1)\n    \n    pull_file(args.download, args.vxcage, api_key )\n    \ndef pull_file(file_hash, vxcage, api_key):\n    try:\n     malshare_url = \"http://api.malshare.com/sampleshare.php\"\n     payload = {'action': 'getfile', 'api_key': api_key, 'hash' : file_hash }\n     user_agent = {'User-agent': 'wget_malshare daily 1.0'}\n\n     r = requests.get(malshare_url, params=payload, headers=user_agent)\n\n     sample = r.content\n\n     if (sample == \"Sample not found\"):\n         logging.error(\"Sample not Found\")\n         return None\n     if (sample == \"ERROR! => Account not activated\"):\n         logging.error(\"Bad API Key\")\n         return None\n     open(os.path.join(file_hash),\"wb\").write(sample)\n     logging.info(\"Saved %s\" % file_hash)\n\n     if vxcage:\n         vxcage_url = vxcage + \"/malware/add\"\n         files = {'file': sample }\n         payload = {'tags' : 'malshare'}\n         r = requests.post(vxcage_url, files=files, data=payload, headers=user_agent)\n         if r.json()['message'] == 'added':\n             logging.info(\"Uploaded %s to VXCage\" % file_hash)\n    except Exception as e:\n        logging.error(\"Problem connecting. Please Try again.\")\n        logging.exception(sys.exc_info())\n        logging.exception(type(e))\n        logging.exception(e.args)\n        logging.exception(e)\n        sys.exit(1)\n\n# Read ~/.malshare and read the first line.  This file only needs the API string in it.\ndef stored_api_check():\n    global api_key\n    try:\n        if ( os.path.exists(os.path.expanduser('~') + '/.malshare' ) ): \n            with open( os.path.expanduser('~') + '/.malshare' ) as handle_api_file:\n                api_key = func_parse_api_key(handle_api_file.readlines())\n\t\treturn True\n        elif (  os.path.exists('.malshare' ) ): \n            with open( '.malshare' ) as handle_api_file:\n                api_key = func_parse_api_key(handle_api_file.readlines())\n\t\treturn True\n    except IOError:\n        pass\n    return False\n\n# Parse the API key and exit if the API key contains any non [A-Za-z0-9]+\ndef func_parse_api_key(lst_tmp_key):\n    str_tmp_key = \"\".join(lst_tmp_key).rstrip()\n    if re.match(\"^[A-Za-z0-9]+$\", str_tmp_key): \n        return str_tmp_key\n\n\n\nif __name__ == \"__main__\":\n    main()\n    \n"
  },
  {
    "path": "wget_malshare_daily",
    "content": "#! /usr/bin/env python\n# Copyright (C) 2013 Malshare Developers.\n# Pull All Daily MD5 Hashes\n\n# 02/21/2014 Modified by Jun Xie <jxie2004@gmail.com>\n#     to download a single day: wget_malshare_daily -d 2014-01-27\n#     to download samples within a range: wget_malshare_daily -s 2014-01-27 -e 2014-02-07\n#\n# Sciprt will create the folder named by date automatically under current directory\n\nimport argparse\nimport logging\nimport requests\nimport sys\nimport os\nimport re\nimport sys\nimport string\nfrom datetime import datetime, date, timedelta\n\napi_key =\"\"\n\nlogging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.WARNING)\n\ndef main(): \n    global api_key\n   \n    parser = argparse.ArgumentParser()\n    parser.add_argument(\"-k\", \"--apikey\", help=\"API Key\", required=False)\n    parser.add_argument(\"-o\", \"--outfolder\", help=\"Folder to save samples to\", required=False)\n    parser.add_argument(\"-x\", \"--vxcage\", help=\"VXCage server\", required=False)\n    parser.add_argument(\"-d\", \"--date\", type=str, help=\"Specify the date to download. If not specified, download today's. Format:yyyy-mm-dd.\", required=False)\n    parser.add_argument(\"-s\", \"--sdate\", type=str, help=\"Specify the start date to download. Format:yyyy-mm-dd.\", required=False)\n    parser.add_argument(\"-e\", \"--edate\", type=str, help=\"Specify the end date to download. Format:yyyy-mm-dd.\", required=False)\n    global api_key\n\n    args = parser.parse_args()\n    if args.apikey:\n        api_key = args.apikey\n\n    if (not api_key):\n        logging.error(\"API Key not entered\")\n        sys.exit(1)\n\n    if args.sdate and args.edate:\n        start_date = datetime.strptime(args.sdate, '%Y-%m-%d').date()\n        end_date = datetime.strptime(args.edate, '%Y-%m-%d').date()\n        if end_date < start_date:\n            print(\"end_date(%s) is earlier than start_date(%s)\" % (str(end_date), str(start_date)))\n            sys.exit(1)\n        temp_date = start_date\n        if not args.outfolder:\n            args.outfolder=\"./\"\n        while temp_date <= end_date:\n            temp_date_str = str(temp_date)\n            temp_date += timedelta(days=1)\n            print(\"%s\" % temp_date_str)\n            sub_path = temp_date_str+'/malshare_fileList.'+temp_date_str+'.txt'\n            #if not args.outfolder:\n            outfolder = args.outfolder+temp_date_str\n            if (os.path.exists(outfolder)):\n                #if the directory exist, bypass it, cause we already downloaded this folder\n                continue\n            download_daily(args.vxcage, outfolder, sub_path)\n        sys.exit(0)\n\n    if args.date:\n        date_str = str(datetime.strptime(args.date, '%Y-%m-%d').date())\n        sub_path = date_str+'/malshare_fileList.'+date_str+'.txt'\n\n        # automatically create date directory under current directory if outfolder is not specified\n        if not args.outfolder:\n            args.outfolder = date_str\n    else:\n        sub_path = 'malshare.current.txt'\n    print \"sub_path\", sub_path\n    #sys.exit(0)\n\n    #download samples of this date\n    download_daily(args.vxcage, args.outfolder, sub_path)\n\ndef download_daily(vxcage, outfolder, sub_path):\n    if outfolder:\n        if (not os.path.exists(outfolder)):\n            os.makedirs(outfolder)\n        #os.chdir(args.outfolder)\n    \n    for md5_hash in pull_daily_list(sub_path):\n        if \"<!DOCTYPE HTML PUBLIC\" in md5_hash:\n            print(\"%s doesn't exist! skip.\" % sub_path)\n            os.rmdir(outfolder)\n            break\n        if (md5_hash):\n            logging.info(\"Downloading %s\" % md5_hash)\n            print md5_hash\n            pull_file(md5_hash, vxcage, outfolder)\n\ndef pull_daily_list(sub_path):\n    try:\n        url = \"http://www.malshare.com/daily/\"+sub_path\n        print url\n        user_agent = {'User-agent': 'wget_malshare daily 1.0'}\n\n        r = requests.get(url, headers=user_agent)\n        for line in r.content.split('\\n'):\n            logging.debug(\"Yield line: %s\" % line)\n            yield line\n        logging.debug(\"No more lines\")\n\n    except Exception as e:\n        logging.error(\"Problem connecting.  Please Try again.\")\n        logging.exception(sys.exc_info())\n        logging.exception(type(e))\n        logging.exception(e.args)\n        logging.exception(e)\n        logging.error(\"Return None\")\n        yield None\n        pass     # in batch download mode, if one date doesn't exist, skip to next date\n\ndef pull_file(file_hash, vxcage, outfolder):\n    try:\n        if not outfolder:\n            outfolder = '.'\n\n        malshare_url = \"http://malshare.com/sampleshare.php\"\n        payload = {'action': 'getfile', 'api_key': api_key, 'hash' : file_hash }\n        user_agent = {'User-agent': 'wget_malshare daily 1.0'}\n\n        r = requests.get(malshare_url, params=payload, headers=user_agent)\n\n        sample = r.content\n\n        if (sample == \"Sample not found\"):\n            logging.error(\"Sample not Found\")\n            return None\n        if (sample == \"ERROR! => Account not activated\"):\n            logging.error(\"Bad API Key\")\n            return None\n\n        if outfolder:\n            open(os.path.join(outfolder, file_hash),\"wb\").write(sample)\n            logging.info(\"Saved %s\" % file_hash)\n\n        if vxcage:\n            vxcage_url = vxcage + \"/malware/add\"\n            files = {'file': sample }\n            payload = {'tags' : 'malshare'}\n            r = requests.post(vxcage_url, files=files, data=payload, headers=user_agent)\n            if r.json()['message'] == 'added':\n                logging.info(\"Uploaded %s to VXCage\" % file_hash)\n    except Exception as e:\n        logging.error(\"pull_file: Problem connecting. Please Try again.\")\n        logging.exception(sys.exc_info())\n        logging.exception(type(e))\n        logging.exception(e.args)\n        logging.exception(e)\n        sys.exit(1)\n\ndef stored_api_check():\n    global api_key\n    try:\n        if ( os.path.exists(os.path.expanduser('~') + '/.malshare' ) ):\n            with open( os.path.expanduser('~') + '/.malshare' ) as handle_api_file:\n                api_key = func_parse_api_key(handle_api_file.readlines())\n                return True\n        elif (  os.path.exists('.malshare' ) ):\n            with open( '.malshare' ) as handle_api_file:\n                api_key = func_parse_api_key(handle_api_file.readlines())\n                return True\n    except IOError:\n        pass\n    return False\n\ndef func_parse_api_key(lst_tmp_key):\n    str_tmp_key = \"\".join(lst_tmp_key).rstrip()\n    if re.match(\"^[A-Za-z0-9]+$\", str_tmp_key):\n        return str_tmp_key\n\n\n\nif __name__ == \"__main__\":\n    main()\n    \n"
  }
]