Full Code of Malshare/MalShare-Toolkit for AI

Repository: Malshare/MalShare-Toolkit
Branch: master
Commit: a87c66879f0b
Files: 8
Total size: 18.3 KB

Directory structure:
gitextract_7z3fhjc4/

├── README.md
├── malshare_api
├── malshare_digest
├── malshare_download_list.py
├── malshare_search
├── malshare_upload
├── wget_malshare
└── wget_malshare_daily

================================================
FILE CONTENTS
================================================

================================================
FILE: README.md
================================================
MalShare Toolkit
==============


Set of tools for interacting with MalShare API
Tools:
malshare_digest 
--------------
*Generate a CSV file of daily MD5 list*

	usage: malshare_digest [-h] -o OUTFILE
	malshare_digest: error: argument -o/--outfile is required


wget_malshare
--------------
*Download sample*

	usage: wget_malshare [-h] [-k APIKEY] -d DOWNLOAD [-xVXCAGE]
	wget_malshare: error: argument -d/--download is required

wget_malshare_daily
--------------
*Download all samples from the day prior*

        usage: wget_malshare_daily [-h] [-k APIKEY] -d DOWNLOAD [-x VXCAGE]
        wget_malshare_daily: error: argument -d/--download is required


		 
Don't forget to set your API Key for wget_malshare && wget_malshare_daily


================================================
FILE: malshare_api
================================================
#!/usr/bin/env python
# Copyright (C) 2013 - 2016 Malshare Developers.
# Multitool for MalShare API

import os
import re
import json
import argparse
import requests

BASE_HTTP_PATH = "http://malshare.com/"
API_PATHS = {
				"MD5LIST" : "api.php?api_key=%s&action=getlistraw",
				"SOURCES" : "api.php?api_key=%s&action=getsourcesraw",

				"DOWNLOAD" : "api.php?api_key=%s&action=getfile&hash=%s",
				"DETAILS" : "api.php?api_key=%s&action=details&hash=%s",

				"TYPE" : "api.php?api_key=%s&action=type&type=%s",
			}

api_key = ""

def main():
	largs = parse_args()

	if largs['details']:
		uri = API_PATHS['DETAILS'] % (api_key, largs['details'])
		r = api_call(uri)
		if r is not None:
			details = r.json()
			print json.dumps(details, indent=4, sort_keys=True)


	elif largs['download']:
		uri = API_PATHS['DOWNLOAD'] % (api_key, largs['download'])
		r = api_call(uri)
		if r is not None:
			try:
				with open(str(largs['download']) + ".malshare", 'wb') as f:
					f.write( r.content )
			except Exception, e:
				print "[X] Problem saving file"
				print "[E] %s" % e

	elif largs['type']:
		uri = API_PATHS['TYPE'] % (api_key, largs['type'])
                r = api_call(uri)
                if r is not None:
			for rhash in set(r.json()):
				print rhash	

	elif largs['listmd5']:
		uri = API_PATHS['MD5LIST'] % (api_key)
                r = api_call(uri)
		print r.text.strip()


	elif largs['listsources']:
		uri = API_PATHS['SOURCES'] % (api_key)
                r = api_call(uri)
		print r.text.strip()

def api_call(rpath):
	global api_key
	try:
		user_agent = {'User-Agent': 'MalShare API Tool v/0.1 beta'}
		r = requests.get(BASE_HTTP_PATH + rpath, headers=user_agent)
			
		if r.status_code == 200:
			if standard_error_check(r.content):
				return r
		else:
			if standard_error_check(r.content):
				print "[X] API Call Failed"
				return None
			else:
				return None
			


	except Exception, e:
		print "[X] API Call Failed: %s" % e
		return None


def standard_error_check(rtext):
	if (rtext == "Sample not found"):
		print "[X] Sample not Found"
		return False

	if (rtext == "ERROR! => Account not activated"):
		print "[X] Bad API Key"
		return False	

        if (rtext == "Invalid Hash"):
                print "[X] Invalid Hash"
                return False

        if ( "Sample not found by hash" in rtext ):
                print "[X] Hash not found"
                return False

	return True



def parse_args():
	global api_key
	parser = argparse.ArgumentParser()
	parser.add_argument("-m", "--listmd5", help="Pull MD5 List", required=False, action='store_true')
	parser.add_argument("-s", "--listsources", help="Pull MD5 List", required=False, action='store_true')

	parser.add_argument("-d", "--download", help="Download File by Hash", required=False)
	parser.add_argument("-l", "--details", help="List File Details", required=False)
	parser.add_argument("-t", "--type", help="Search For Daily files by Type", required=False)

	parser.add_argument("-a", "--apikey", help="Set API key for session", required=False)


	args = parser.parse_args()
	if stored_api_check() == False:
		if args.apikey:
			api_key = args.apikey

	return vars(args)

# Read ~/.malshare and read the first line.  This file only needs the API string in it.
def stored_api_check():
	global api_key
	try:
		if ( os.path.exists(os.path.expanduser('~') + '/.malshare' ) ): 
			with open( os.path.expanduser('~') + '/.malshare' ) as handle_api_file:
				api_key = func_parse_api_key(handle_api_file.readlines())
			return True
		elif (  os.path.exists('.malshare' ) ): 
			with open( '.malshare' ) as handle_api_file:
				api_key = func_parse_api_key(handle_api_file.readlines())
		return True
	except IOError:
		pass
	return False

# Parse the API key and exit if the API key contains any non [A-Za-z0-9]+
def func_parse_api_key(lst_tmp_key):
	str_tmp_key = "".join(lst_tmp_key).rstrip()
	if re.match("^[A-Za-z0-9]+$", str_tmp_key): 
		return str_tmp_key

if __name__ == "__main__":
	main()



================================================
FILE: malshare_digest
================================================
#! /usr/bin/env python
# Copyright (C) 2013 Malshare Developers.
# Written by Blevene github.com/Blevene 

# Open and catalog list of samples observed for a given date
# on Malshare.com, location: http://www.malshare.com/daily/malshare.current.txt
# To Do:
# [x] Create output file (csv or txt?)
# [x] Store each md5 as a csv with 'hash' : 'date' mapping

import urllib2
import argparse
import csv
from sys import argv
from datetime import datetime, date

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-o", "--outfile", help="Pull the most recent Malshare digest", required=True)
    args = parser.parse_args()
    
    mal_digest = urllib2.urlopen(url='http://www.malshare.com/daily/malshare.current.txt')
    mal_digest = list(mal_digest)
    pull_time = [str(date.today())] * len(mal_digest) 
    strip_list = [x.strip('\n') for x in mal_digest]
    dictionary = dict(zip(strip_list, pull_time))
    outfile_name = args.outfile + '.csv'
    writer = csv.writer(open( outfile_name , 'a'))
    for key, value in dictionary.items():
        if (key):
            writer.writerow([key, value])

if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        sys.exit(" [X] Shutting Down")






================================================
FILE: malshare_download_list.py
================================================
#! /usr/bin/env python
# Download files with given hashes.json file

import argparse
import json
import logging
import os
import sys
from multiprocessing.pool import Pool

import requests
import tqdm

api_key = "<API_KEY>"

logging.basicConfig(format = '%(asctime)s %(levelname)s:%(message)s', level = logging.WARNING)


def download_file_by_hash(file_hash):
    logging.debug("Downloading {}".format(file_hash))
    try:
        malshare_url = "http://malshare.com/sampleshare.php"
        payload = {'action': 'getfile', 'api_key': api_key, 'hash': file_hash}
        user_agent = {'User-agent': 'wget_malshare daily 1.0'}

        r = requests.get(malshare_url, params = payload, headers = user_agent)
        sample = r.content

        if sample == "Sample not found":
            logging.error("Sample not Found")
            return None
        if sample == "ERROR! => Account not activated":
            logging.error("Bad API Key")
            return None

        with open(os.path.join("files", file_hash), mode = "wb") as fh:
            fh.write(sample)
            logging.info("{} saved to files".format(file_hash))

    except Exception as e:
        logging.error("download_file_by_hash: Problem connecting. Please Try again.")
        logging.exception(sys.exc_info())
        logging.exception(type(e))
        logging.exception(e.args)
        logging.exception(e)
        sys.exit(1)


def download_list(api_k, hash_list):
    global api_key
    if api_k:
        api_key = api_k
    files = json.load(open(hash_list))
    pool = Pool(os.cpu_count())
    for _ in tqdm.tqdm(pool.imap_unordered(download_file_by_hash, files), total = len(files)):
        pass


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-k", "--apikey", help = "API Key", required = False)
    parser.add_argument("-f", "--hash_list", help = "File containing list of hashes in json format", required = True)
    args = parser.parse_args()
    download_list(args.apikey, args.hash_list)


================================================
FILE: malshare_search
================================================
#!/usr/bin/env python
# Copyright (C) 2013 - 2018 Malshare Developers.
# Written by Silas Cutler
# Quick search tool for  MalShare API

import sys
import json
import requests

api_key = ""
if api_key != "":
	url="https://malshare.com/api.php?api_key=%s&action=search&query=%s" % (api_key, sys.argv[1] )
	r = requests.get(url)
	print json.dumps(r.json(), sort_keys=True, indent=4, separators=(',', ': '))
else:
	print "Please set API key"



================================================
FILE: malshare_upload
================================================
#!/bin/bash
# Copyright (C) 2013 - 2017 Malshare Developers.
# Written by Silas Cutler
# Upload tool for  MalShare API



API_KEY=""

function usage(){
	echo "$0 <File to Upload>"
	echo " - File Uploader for MalShare.com"
}


if [ -z "$API_KEY" ]
then
	usage
	echo ""
	echo "Please set API Key in script"
	exit 1
fi

if [ ! -f "$1" ]
then
	usage
	echo ""
	echo " [x] Please specify file to upload"
	exit 1
fi

RES=$(curl -s -X POST -F "upload=@$1" "https://malshare.com/api.php?api_key=$API_KEY&action=upload")
echo $RES

if [ "$RES" = "Success" ]
then
	MD5HASH=$(md5sum $1 | awk '{ print $1 }' )
	echo "Sample can be viewed at https://malshare.com/sample.php?action=detail&hash=$MD5HASH"
else
	echo "Problem with upload"
fi



================================================
FILE: wget_malshare
================================================
#! /usr/bin/env python
# Copyright (C) 2013 - 2016 Malshare Developers.
# Pull sample by [MD5 | SHA1 | SHA256] Hash

import os
import re
import sys
import logging
import requests
import argparse

api_key =""

def main():
    global api_key

    parser = argparse.ArgumentParser()
    parser.add_argument("-k", "--apikey", help="API Key", required=False)
    parser.add_argument("-d", "--download", help="Search / Download Hash", required=True)
    parser.add_argument("-x", "--vxcage", help="VXCage server", required=False)

    args = parser.parse_args()
    if stored_api_check() == False:
        if args.apikey:
            api_key = args.apikey

    if (not api_key):
        logging.error("API Key not entered")
        sys.exit(1)
    
    pull_file(args.download, args.vxcage, api_key )
    
def pull_file(file_hash, vxcage, api_key):
    try:
     malshare_url = "http://api.malshare.com/sampleshare.php"
     payload = {'action': 'getfile', 'api_key': api_key, 'hash' : file_hash }
     user_agent = {'User-agent': 'wget_malshare daily 1.0'}

     r = requests.get(malshare_url, params=payload, headers=user_agent)

     sample = r.content

     if (sample == "Sample not found"):
         logging.error("Sample not Found")
         return None
     if (sample == "ERROR! => Account not activated"):
         logging.error("Bad API Key")
         return None
     open(os.path.join(file_hash),"wb").write(sample)
     logging.info("Saved %s" % file_hash)

     if vxcage:
         vxcage_url = vxcage + "/malware/add"
         files = {'file': sample }
         payload = {'tags' : 'malshare'}
         r = requests.post(vxcage_url, files=files, data=payload, headers=user_agent)
         if r.json()['message'] == 'added':
             logging.info("Uploaded %s to VXCage" % file_hash)
    except Exception as e:
        logging.error("Problem connecting. Please Try again.")
        logging.exception(sys.exc_info())
        logging.exception(type(e))
        logging.exception(e.args)
        logging.exception(e)
        sys.exit(1)

# Read ~/.malshare and read the first line.  This file only needs the API string in it.
def stored_api_check():
    global api_key
    try:
        if ( os.path.exists(os.path.expanduser('~') + '/.malshare' ) ): 
            with open( os.path.expanduser('~') + '/.malshare' ) as handle_api_file:
                api_key = func_parse_api_key(handle_api_file.readlines())
		return True
        elif (  os.path.exists('.malshare' ) ): 
            with open( '.malshare' ) as handle_api_file:
                api_key = func_parse_api_key(handle_api_file.readlines())
		return True
    except IOError:
        pass
    return False

# Parse the API key and exit if the API key contains any non [A-Za-z0-9]+
def func_parse_api_key(lst_tmp_key):
    str_tmp_key = "".join(lst_tmp_key).rstrip()
    if re.match("^[A-Za-z0-9]+$", str_tmp_key): 
        return str_tmp_key



if __name__ == "__main__":
    main()
    


================================================
FILE: wget_malshare_daily
================================================
#! /usr/bin/env python
# Copyright (C) 2013 Malshare Developers.
# Pull All Daily MD5 Hashes

# 02/21/2014 Modified by Jun Xie <jxie2004@gmail.com>
#     to download a single day: wget_malshare_daily -d 2014-01-27
#     to download samples within a range: wget_malshare_daily -s 2014-01-27 -e 2014-02-07
#
# Sciprt will create the folder named by date automatically under current directory

import argparse
import logging
import requests
import sys
import os
import re
import sys
import string
from datetime import datetime, date, timedelta

api_key =""

logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', level=logging.WARNING)

def main(): 
    global api_key
   
    parser = argparse.ArgumentParser()
    parser.add_argument("-k", "--apikey", help="API Key", required=False)
    parser.add_argument("-o", "--outfolder", help="Folder to save samples to", required=False)
    parser.add_argument("-x", "--vxcage", help="VXCage server", required=False)
    parser.add_argument("-d", "--date", type=str, help="Specify the date to download. If not specified, download today's. Format:yyyy-mm-dd.", required=False)
    parser.add_argument("-s", "--sdate", type=str, help="Specify the start date to download. Format:yyyy-mm-dd.", required=False)
    parser.add_argument("-e", "--edate", type=str, help="Specify the end date to download. Format:yyyy-mm-dd.", required=False)
    global api_key

    args = parser.parse_args()
    if args.apikey:
        api_key = args.apikey

    if (not api_key):
        logging.error("API Key not entered")
        sys.exit(1)

    if args.sdate and args.edate:
        start_date = datetime.strptime(args.sdate, '%Y-%m-%d').date()
        end_date = datetime.strptime(args.edate, '%Y-%m-%d').date()
        if end_date < start_date:
            print("end_date(%s) is earlier than start_date(%s)" % (str(end_date), str(start_date)))
            sys.exit(1)
        temp_date = start_date
        if not args.outfolder:
            args.outfolder="./"
        while temp_date <= end_date:
            temp_date_str = str(temp_date)
            temp_date += timedelta(days=1)
            print("%s" % temp_date_str)
            sub_path = temp_date_str+'/malshare_fileList.'+temp_date_str+'.txt'
            #if not args.outfolder:
            outfolder = args.outfolder+temp_date_str
            if (os.path.exists(outfolder)):
                #if the directory exist, bypass it, cause we already downloaded this folder
                continue
            download_daily(args.vxcage, outfolder, sub_path)
        sys.exit(0)

    if args.date:
        date_str = str(datetime.strptime(args.date, '%Y-%m-%d').date())
        sub_path = date_str+'/malshare_fileList.'+date_str+'.txt'

        # automatically create date directory under current directory if outfolder is not specified
        if not args.outfolder:
            args.outfolder = date_str
    else:
        sub_path = 'malshare.current.txt'
    print "sub_path", sub_path
    #sys.exit(0)

    #download samples of this date
    download_daily(args.vxcage, args.outfolder, sub_path)

def download_daily(vxcage, outfolder, sub_path):
    if outfolder:
        if (not os.path.exists(outfolder)):
            os.makedirs(outfolder)
        #os.chdir(args.outfolder)
    
    for md5_hash in pull_daily_list(sub_path):
        if "<!DOCTYPE HTML PUBLIC" in md5_hash:
            print("%s doesn't exist! skip." % sub_path)
            os.rmdir(outfolder)
            break
        if (md5_hash):
            logging.info("Downloading %s" % md5_hash)
            print md5_hash
            pull_file(md5_hash, vxcage, outfolder)

def pull_daily_list(sub_path):
    try:
        url = "http://www.malshare.com/daily/"+sub_path
        print url
        user_agent = {'User-agent': 'wget_malshare daily 1.0'}

        r = requests.get(url, headers=user_agent)
        for line in r.content.split('\n'):
            logging.debug("Yield line: %s" % line)
            yield line
        logging.debug("No more lines")

    except Exception as e:
        logging.error("Problem connecting.  Please Try again.")
        logging.exception(sys.exc_info())
        logging.exception(type(e))
        logging.exception(e.args)
        logging.exception(e)
        logging.error("Return None")
        yield None
        pass     # in batch download mode, if one date doesn't exist, skip to next date

def pull_file(file_hash, vxcage, outfolder):
    try:
        if not outfolder:
            outfolder = '.'

        malshare_url = "http://malshare.com/sampleshare.php"
        payload = {'action': 'getfile', 'api_key': api_key, 'hash' : file_hash }
        user_agent = {'User-agent': 'wget_malshare daily 1.0'}

        r = requests.get(malshare_url, params=payload, headers=user_agent)

        sample = r.content

        if (sample == "Sample not found"):
            logging.error("Sample not Found")
            return None
        if (sample == "ERROR! => Account not activated"):
            logging.error("Bad API Key")
            return None

        if outfolder:
            open(os.path.join(outfolder, file_hash),"wb").write(sample)
            logging.info("Saved %s" % file_hash)

        if vxcage:
            vxcage_url = vxcage + "/malware/add"
            files = {'file': sample }
            payload = {'tags' : 'malshare'}
            r = requests.post(vxcage_url, files=files, data=payload, headers=user_agent)
            if r.json()['message'] == 'added':
                logging.info("Uploaded %s to VXCage" % file_hash)
    except Exception as e:
        logging.error("pull_file: Problem connecting. Please Try again.")
        logging.exception(sys.exc_info())
        logging.exception(type(e))
        logging.exception(e.args)
        logging.exception(e)
        sys.exit(1)

def stored_api_check():
    global api_key
    try:
        if ( os.path.exists(os.path.expanduser('~') + '/.malshare' ) ):
            with open( os.path.expanduser('~') + '/.malshare' ) as handle_api_file:
                api_key = func_parse_api_key(handle_api_file.readlines())
                return True
        elif (  os.path.exists('.malshare' ) ):
            with open( '.malshare' ) as handle_api_file:
                api_key = func_parse_api_key(handle_api_file.readlines())
                return True
    except IOError:
        pass
    return False

def func_parse_api_key(lst_tmp_key):
    str_tmp_key = "".join(lst_tmp_key).rstrip()
    if re.match("^[A-Za-z0-9]+$", str_tmp_key):
        return str_tmp_key



if __name__ == "__main__":
    main()