Repository: Malshare/MalShare-Toolkit Branch: master Commit: a87c66879f0b Files: 8 Total size: 18.3 KB Directory structure: gitextract_7z3fhjc4/ ├── README.md ├── malshare_api ├── malshare_digest ├── malshare_download_list.py ├── malshare_search ├── malshare_upload ├── wget_malshare └── wget_malshare_daily ================================================ FILE CONTENTS ================================================ ================================================ FILE: README.md ================================================ MalShare Toolkit ============== Set of tools for interacting with MalShare API Tools: malshare_digest -------------- *Generate a CSV file of daily MD5 list* usage: malshare_digest [-h] -o OUTFILE malshare_digest: error: argument -o/--outfile is required wget_malshare -------------- *Download sample* usage: wget_malshare [-h] [-k APIKEY] -d DOWNLOAD [-xVXCAGE] wget_malshare: error: argument -d/--download is required wget_malshare_daily -------------- *Download all samples from the day prior* usage: wget_malshare_daily [-h] [-k APIKEY] -d DOWNLOAD [-x VXCAGE] wget_malshare_daily: error: argument -d/--download is required Don't forget to set your API Key for wget_malshare && wget_malshare_daily ================================================ FILE: malshare_api ================================================ #!/usr/bin/env python # Copyright (C) 2013 - 2016 Malshare Developers. # Multitool for MalShare API import os import re import json import argparse import requests BASE_HTTP_PATH = "http://malshare.com/" API_PATHS = { "MD5LIST" : "api.php?api_key=%s&action=getlistraw", "SOURCES" : "api.php?api_key=%s&action=getsourcesraw", "DOWNLOAD" : "api.php?api_key=%s&action=getfile&hash=%s", "DETAILS" : "api.php?api_key=%s&action=details&hash=%s", "TYPE" : "api.php?api_key=%s&action=type&type=%s", } api_key = "" def main(): largs = parse_args() if largs['details']: uri = API_PATHS['DETAILS'] % (api_key, largs['details']) r = api_call(uri) if r is not None: details = r.json() print json.dumps(details, indent=4, sort_keys=True) elif largs['download']: uri = API_PATHS['DOWNLOAD'] % (api_key, largs['download']) r = api_call(uri) if r is not None: try: with open(str(largs['download']) + ".malshare", 'wb') as f: f.write( r.content ) except Exception, e: print "[X] Problem saving file" print "[E] %s" % e elif largs['type']: uri = API_PATHS['TYPE'] % (api_key, largs['type']) r = api_call(uri) if r is not None: for rhash in set(r.json()): print rhash elif largs['listmd5']: uri = API_PATHS['MD5LIST'] % (api_key) r = api_call(uri) print r.text.strip() elif largs['listsources']: uri = API_PATHS['SOURCES'] % (api_key) r = api_call(uri) print r.text.strip() def api_call(rpath): global api_key try: user_agent = {'User-Agent': 'MalShare API Tool v/0.1 beta'} r = requests.get(BASE_HTTP_PATH + rpath, headers=user_agent) if r.status_code == 200: if standard_error_check(r.content): return r else: if standard_error_check(r.content): print "[X] API Call Failed" return None else: return None except Exception, e: print "[X] API Call Failed: %s" % e return None def standard_error_check(rtext): if (rtext == "Sample not found"): print "[X] Sample not Found" return False if (rtext == "ERROR! => Account not activated"): print "[X] Bad API Key" return False if (rtext == "Invalid Hash"): print "[X] Invalid Hash" return False if ( "Sample not found by hash" in rtext ): print "[X] Hash not found" return False return True def parse_args(): global api_key parser = argparse.ArgumentParser() parser.add_argument("-m", "--listmd5", help="Pull MD5 List", required=False, action='store_true') parser.add_argument("-s", "--listsources", help="Pull MD5 List", required=False, action='store_true') parser.add_argument("-d", "--download", help="Download File by Hash", required=False) parser.add_argument("-l", "--details", help="List File Details", required=False) parser.add_argument("-t", "--type", help="Search For Daily files by Type", required=False) parser.add_argument("-a", "--apikey", help="Set API key for session", required=False) args = parser.parse_args() if stored_api_check() == False: if args.apikey: api_key = args.apikey return vars(args) # Read ~/.malshare and read the first line. This file only needs the API string in it. def stored_api_check(): global api_key try: if ( os.path.exists(os.path.expanduser('~') + '/.malshare' ) ): with open( os.path.expanduser('~') + '/.malshare' ) as handle_api_file: api_key = func_parse_api_key(handle_api_file.readlines()) return True elif ( os.path.exists('.malshare' ) ): with open( '.malshare' ) as handle_api_file: api_key = func_parse_api_key(handle_api_file.readlines()) return True except IOError: pass return False # Parse the API key and exit if the API key contains any non [A-Za-z0-9]+ def func_parse_api_key(lst_tmp_key): str_tmp_key = "".join(lst_tmp_key).rstrip() if re.match("^[A-Za-z0-9]+$", str_tmp_key): return str_tmp_key if __name__ == "__main__": main() ================================================ FILE: malshare_digest ================================================ #! /usr/bin/env python # Copyright (C) 2013 Malshare Developers. # Written by Blevene github.com/Blevene # Open and catalog list of samples observed for a given date # on Malshare.com, location: http://www.malshare.com/daily/malshare.current.txt # To Do: # [x] Create output file (csv or txt?) # [x] Store each md5 as a csv with 'hash' : 'date' mapping import urllib2 import argparse import csv from sys import argv from datetime import datetime, date def main(): parser = argparse.ArgumentParser() parser.add_argument("-o", "--outfile", help="Pull the most recent Malshare digest", required=True) args = parser.parse_args() mal_digest = urllib2.urlopen(url='http://www.malshare.com/daily/malshare.current.txt') mal_digest = list(mal_digest) pull_time = [str(date.today())] * len(mal_digest) strip_list = [x.strip('\n') for x in mal_digest] dictionary = dict(zip(strip_list, pull_time)) outfile_name = args.outfile + '.csv' writer = csv.writer(open( outfile_name , 'a')) for key, value in dictionary.items(): if (key): writer.writerow([key, value]) if __name__ == "__main__": try: main() except KeyboardInterrupt: sys.exit(" [X] Shutting Down") ================================================ FILE: malshare_download_list.py ================================================ #! /usr/bin/env python # Download files with given hashes.json file import argparse import json import logging import os import sys from multiprocessing.pool import Pool import requests import tqdm api_key = "" logging.basicConfig(format = '%(asctime)s %(levelname)s:%(message)s', level = logging.WARNING) def download_file_by_hash(file_hash): logging.debug("Downloading {}".format(file_hash)) try: malshare_url = "http://malshare.com/sampleshare.php" payload = {'action': 'getfile', 'api_key': api_key, 'hash': file_hash} user_agent = {'User-agent': 'wget_malshare daily 1.0'} r = requests.get(malshare_url, params = payload, headers = user_agent) sample = r.content if sample == "Sample not found": logging.error("Sample not Found") return None if sample == "ERROR! => Account not activated": logging.error("Bad API Key") return None with open(os.path.join("files", file_hash), mode = "wb") as fh: fh.write(sample) logging.info("{} saved to files".format(file_hash)) except Exception as e: logging.error("download_file_by_hash: Problem connecting. Please Try again.") logging.exception(sys.exc_info()) logging.exception(type(e)) logging.exception(e.args) logging.exception(e) sys.exit(1) def download_list(api_k, hash_list): global api_key if api_k: api_key = api_k files = json.load(open(hash_list)) pool = Pool(os.cpu_count()) for _ in tqdm.tqdm(pool.imap_unordered(download_file_by_hash, files), total = len(files)): pass if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("-k", "--apikey", help = "API Key", required = False) parser.add_argument("-f", "--hash_list", help = "File containing list of hashes in json format", required = True) args = parser.parse_args() download_list(args.apikey, args.hash_list) ================================================ FILE: malshare_search ================================================ #!/usr/bin/env python # Copyright (C) 2013 - 2018 Malshare Developers. # Written by Silas Cutler # Quick search tool for MalShare API import sys import json import requests api_key = "" if api_key != "": url="https://malshare.com/api.php?api_key=%s&action=search&query=%s" % (api_key, sys.argv[1] ) r = requests.get(url) print json.dumps(r.json(), sort_keys=True, indent=4, separators=(',', ': ')) else: print "Please set API key" ================================================ FILE: malshare_upload ================================================ #!/bin/bash # Copyright (C) 2013 - 2017 Malshare Developers. # Written by Silas Cutler # Upload tool for MalShare API API_KEY="" function usage(){ echo "$0 " echo " - File Uploader for MalShare.com" } if [ -z "$API_KEY" ] then usage echo "" echo "Please set API Key in script" exit 1 fi if [ ! -f "$1" ] then usage echo "" echo " [x] Please specify file to upload" exit 1 fi RES=$(curl -s -X POST -F "upload=@$1" "https://malshare.com/api.php?api_key=$API_KEY&action=upload") echo $RES if [ "$RES" = "Success" ] then MD5HASH=$(md5sum $1 | awk '{ print $1 }' ) echo "Sample can be viewed at https://malshare.com/sample.php?action=detail&hash=$MD5HASH" else echo "Problem with upload" fi ================================================ FILE: wget_malshare ================================================ #! /usr/bin/env python # Copyright (C) 2013 - 2016 Malshare Developers. # Pull sample by [MD5 | SHA1 | SHA256] Hash import os import re import sys import logging import requests import argparse api_key ="" def main(): global api_key parser = argparse.ArgumentParser() parser.add_argument("-k", "--apikey", help="API Key", required=False) parser.add_argument("-d", "--download", help="Search / Download Hash", required=True) parser.add_argument("-x", "--vxcage", help="VXCage server", required=False) args = parser.parse_args() if stored_api_check() == False: if args.apikey: api_key = args.apikey if (not api_key): logging.error("API Key not entered") sys.exit(1) pull_file(args.download, args.vxcage, api_key ) def pull_file(file_hash, vxcage, api_key): try: malshare_url = "http://api.malshare.com/sampleshare.php" payload = {'action': 'getfile', 'api_key': api_key, 'hash' : file_hash } user_agent = {'User-agent': 'wget_malshare daily 1.0'} r = requests.get(malshare_url, params=payload, headers=user_agent) sample = r.content if (sample == "Sample not found"): logging.error("Sample not Found") return None if (sample == "ERROR! => Account not activated"): logging.error("Bad API Key") return None open(os.path.join(file_hash),"wb").write(sample) logging.info("Saved %s" % file_hash) if vxcage: vxcage_url = vxcage + "/malware/add" files = {'file': sample } payload = {'tags' : 'malshare'} r = requests.post(vxcage_url, files=files, data=payload, headers=user_agent) if r.json()['message'] == 'added': logging.info("Uploaded %s to VXCage" % file_hash) except Exception as e: logging.error("Problem connecting. Please Try again.") logging.exception(sys.exc_info()) logging.exception(type(e)) logging.exception(e.args) logging.exception(e) sys.exit(1) # Read ~/.malshare and read the first line. This file only needs the API string in it. def stored_api_check(): global api_key try: if ( os.path.exists(os.path.expanduser('~') + '/.malshare' ) ): with open( os.path.expanduser('~') + '/.malshare' ) as handle_api_file: api_key = func_parse_api_key(handle_api_file.readlines()) return True elif ( os.path.exists('.malshare' ) ): with open( '.malshare' ) as handle_api_file: api_key = func_parse_api_key(handle_api_file.readlines()) return True except IOError: pass return False # Parse the API key and exit if the API key contains any non [A-Za-z0-9]+ def func_parse_api_key(lst_tmp_key): str_tmp_key = "".join(lst_tmp_key).rstrip() if re.match("^[A-Za-z0-9]+$", str_tmp_key): return str_tmp_key if __name__ == "__main__": main() ================================================ FILE: wget_malshare_daily ================================================ #! /usr/bin/env python # Copyright (C) 2013 Malshare Developers. # Pull All Daily MD5 Hashes # 02/21/2014 Modified by Jun Xie # to download a single day: wget_malshare_daily -d 2014-01-27 # to download samples within a range: wget_malshare_daily -s 2014-01-27 -e 2014-02-07 # # Sciprt will create the folder named by date automatically under current directory import argparse import logging import requests import sys import os import re import sys import string from datetime import datetime, date, timedelta api_key ="" logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.WARNING) def main(): global api_key parser = argparse.ArgumentParser() parser.add_argument("-k", "--apikey", help="API Key", required=False) parser.add_argument("-o", "--outfolder", help="Folder to save samples to", required=False) parser.add_argument("-x", "--vxcage", help="VXCage server", required=False) parser.add_argument("-d", "--date", type=str, help="Specify the date to download. If not specified, download today's. Format:yyyy-mm-dd.", required=False) parser.add_argument("-s", "--sdate", type=str, help="Specify the start date to download. Format:yyyy-mm-dd.", required=False) parser.add_argument("-e", "--edate", type=str, help="Specify the end date to download. Format:yyyy-mm-dd.", required=False) global api_key args = parser.parse_args() if args.apikey: api_key = args.apikey if (not api_key): logging.error("API Key not entered") sys.exit(1) if args.sdate and args.edate: start_date = datetime.strptime(args.sdate, '%Y-%m-%d').date() end_date = datetime.strptime(args.edate, '%Y-%m-%d').date() if end_date < start_date: print("end_date(%s) is earlier than start_date(%s)" % (str(end_date), str(start_date))) sys.exit(1) temp_date = start_date if not args.outfolder: args.outfolder="./" while temp_date <= end_date: temp_date_str = str(temp_date) temp_date += timedelta(days=1) print("%s" % temp_date_str) sub_path = temp_date_str+'/malshare_fileList.'+temp_date_str+'.txt' #if not args.outfolder: outfolder = args.outfolder+temp_date_str if (os.path.exists(outfolder)): #if the directory exist, bypass it, cause we already downloaded this folder continue download_daily(args.vxcage, outfolder, sub_path) sys.exit(0) if args.date: date_str = str(datetime.strptime(args.date, '%Y-%m-%d').date()) sub_path = date_str+'/malshare_fileList.'+date_str+'.txt' # automatically create date directory under current directory if outfolder is not specified if not args.outfolder: args.outfolder = date_str else: sub_path = 'malshare.current.txt' print "sub_path", sub_path #sys.exit(0) #download samples of this date download_daily(args.vxcage, args.outfolder, sub_path) def download_daily(vxcage, outfolder, sub_path): if outfolder: if (not os.path.exists(outfolder)): os.makedirs(outfolder) #os.chdir(args.outfolder) for md5_hash in pull_daily_list(sub_path): if " Account not activated"): logging.error("Bad API Key") return None if outfolder: open(os.path.join(outfolder, file_hash),"wb").write(sample) logging.info("Saved %s" % file_hash) if vxcage: vxcage_url = vxcage + "/malware/add" files = {'file': sample } payload = {'tags' : 'malshare'} r = requests.post(vxcage_url, files=files, data=payload, headers=user_agent) if r.json()['message'] == 'added': logging.info("Uploaded %s to VXCage" % file_hash) except Exception as e: logging.error("pull_file: Problem connecting. Please Try again.") logging.exception(sys.exc_info()) logging.exception(type(e)) logging.exception(e.args) logging.exception(e) sys.exit(1) def stored_api_check(): global api_key try: if ( os.path.exists(os.path.expanduser('~') + '/.malshare' ) ): with open( os.path.expanduser('~') + '/.malshare' ) as handle_api_file: api_key = func_parse_api_key(handle_api_file.readlines()) return True elif ( os.path.exists('.malshare' ) ): with open( '.malshare' ) as handle_api_file: api_key = func_parse_api_key(handle_api_file.readlines()) return True except IOError: pass return False def func_parse_api_key(lst_tmp_key): str_tmp_key = "".join(lst_tmp_key).rstrip() if re.match("^[A-Za-z0-9]+$", str_tmp_key): return str_tmp_key if __name__ == "__main__": main()