Repository: ring04h/thorns
Branch: master
Commit: 65dda993878e
Files: 28
Total size: 164.8 KB

Directory structure:
gitextract_4m8vr42h/

├── ElasticsearchPlugin.py
├── README.md
├── libnmap/
│   ├── __init__.py
│   ├── diff.py
│   ├── objects/
│   │   ├── __init__.py
│   │   ├── cpe.py
│   │   ├── host.py
│   │   ├── os.py
│   │   ├── report.py
│   │   └── service.py
│   ├── parser.py
│   ├── plugins/
│   │   ├── __init__.py
│   │   ├── backend_host.py
│   │   ├── backend_service.py
│   │   ├── backendplugin.py
│   │   ├── backendpluginFactory.py
│   │   ├── es.py
│   │   ├── mongodb.py
│   │   ├── s3.py
│   │   └── sql.py
│   ├── process.py
│   └── reportjson.py
├── run.py
├── src/
│   ├── supervisord_client.conf
│   └── supervisord_server.conf
├── tasks.py
├── wyfunc.py
└── wyportmap.py

================================================
FILE CONTENTS
================================================

================================================
FILE: ElasticsearchPlugin.py
================================================
#!/usr/bin/env python

from libnmap.parser import NmapParser
from libnmap.reportjson import ReportDecoder
from libnmap.plugins.es import NmapElasticsearchPlugin
from datetime import datetime
import json

nmap_report = NmapParser.parse_fromfile('libnmap/test/files/1_hosts.xml')
mindex = datetime.fromtimestamp(nmap_report.started).strftime('%Y-%m-%d')
db = NmapElasticsearchPlugin(index=mindex)
dbid = db.insert(nmap_report)
nmap_json = db.get(dbid)

nmap_obj = json.loads(json.dumps(nmap_json), cls=ReportDecoder)
print(nmap_obj)
#print(db.getall())



================================================
FILE: README.md
================================================
# thorns
thorns_project 分布式异步队列系统

运行流程
-----------------------------------
* 启动redis内存服务器，作为队列存储数据库使用
* 配置芹菜(celery)运行环境，并连接redis队列内存，读取执行任务，并返回结果存储到后端MySQL数据库
* 配置任务控制台花花(flower)，并连接redis队列内存，管理所有worker客户端与执行的任务队列
* 通过run.py脚本调用celery向队列压入任务
* 通过flower的http api脚本调用api向队列压入任务
* 任务执行的结果自动存入后端数据库

反馈
-----------------------------------
> 微博：http://weibo.com/ringzero<br />
> 邮箱：ringzero@0x557.org<br />

运行环境
-----------------------------------
* CentOS、Kali Linux、Ubuntu、Debian
* Python 2.7.x
* Redis
* MysQL
* Celery
* Tornado
* Supervisord

安装配置说明
-----------------------------------
## CentOS 服务端

#### 安装 Redis-Server
	$ wget http://download.redis.io/releases/redis-2.8.19.tar.gz
	$ tar xzf redis-2.8.19.tar.gz
	$ cd redis-2.8.19
	$ make
	$ sudo cp src/redis-server /usr/bin/
	$ sudo cp redis.conf /etc/redis.conf
	/* 修改 /etc/redis.conf 37行，将daemonize no改为daemonize yes，让redis后台运行 */
	$ sudo vim /etc/redis.conf
	daemonize yes
	# 启动Redis-Server
	$ sudo redis-server /etc/redis.conf

#### 安装 pip
	$ wget https://pypi.python.org/packages/source/p/pip/pip-6.0.8.tar.gz
	$ tar zvxf pip-6.0.8.tar.gz
	$ cd pip-6.0.8
	$ sudo python setup.py install

#### 安装 MySQL-python
	$ sudo yum -y install python-devel mysql-devel subversion-devel
	$ sudo pip install MySQL-python SQLAlchemy

#### 安装 Celery
	$ sudo pip install -U celery[redis]

#### 安装 Flower & 下载thorns运行环境代码
	$ cd /home/
	$ sudo yum -y install git
	$ git clone https://github.com/ring04h/thorns.git
	$ cd /home/thorns/src
	$ tar zvxf flower.tar.gz
	$ cd flower-0.7.3
	$ python setup.py install
	/* 启动Flower 这里的redis ip可以配置为你的外网的IP */
	$ celery flower --port=8080 --broker=redis://127.0.0.1:6379/0 &
	建议使用Supervisord的守护进程来启动Flower，确保系统7*24小时的稳定性

#### 安装 Supervisord
	$ sudo pip install supervisor
	$ sudo cp /home/thorns/src/supervisord_server.conf /etc/supervisord.conf
	/* 修改 /etc/supervisord.conf 141行 修改redis ip为你自己的ip --broker=redis://127.0.0.1:6379/0 */
	/* 修改 /etc/supervisord.conf 153行 修改programe为你想定义的worker名称 [program:worker-ringzero] */
	$ sudo vim /etc/supervisord.conf
	/* 启动 supervisord */
	$ supervisord -c /etc/supervisord.conf
	http://127.0.0.1:9001/ 可以在线守护管理thorns的进程，实现远程重启

#### 检查各服务是否正常启动后，开始配置客户端任务脚本
	1、http://youip:8080/  thorns 控制台
	2、http://youip:9001/  supervisord 控制台
	3、修改tasks.py内的芹菜配置
	对应你自己的redis-server服务器IP
	BROKER_URL = 'redis://120.132.54.90:6379/0',
	对应你自己的MySQL-server服务器IP
	CELERY_RESULT_BACKEND = 'db+mysql://celery:celery1@3Wscan@42.62.52.62:443/wscan',

	配置完毕后，就可以部署多台客户端进行分布式任务执行了

## CentOS 客户端（建议大规模部署）
	# 安装 git & 下载 thorns_project
	$ sudo yum -y install git
	$ cd /home/
	$ git clone https://github.com/ring04h/thorns.git

	# 安装 pip
	$ wget https://pypi.python.org/packages/source/p/pip/pip-6.0.8.tar.gz
	$ tar zvxf pip-6.0.8.tar.gz
	$ cd pip-6.0.8
	$ sudo python setup.py install

	# 安装 MySQL-python
	$ sudo yum -y install python-devel mysql-devel subversion-devel
	$ sudo pip install MySQL-python SQLAlchemy

	# 安装 nmap
	# 32位系统
	$ sudo rpm -vhU https://nmap.org/dist/nmap-6.47-1.i386.rpm
	# 64位系统
	$ sudo rpm -vhU https://nmap.org/dist/nmap-6.47-1.x86_64.rpm

	# 安装 Celery
	$ sudo pip install -U celery[redis]

	# 安装 Supervisord
	$ sudo pip install supervisor
	$ sudo cp /home/thorns/src/supervisord_client.conf /etc/supervisord.conf
	/* 修改 /etc/supervisord.conf 140行 修改programe为你想定义的worker名称 [program:worker-ringzero] */
	$ sudo vim /etc/supervisord.conf
	/* 启动 supervisord */
	$ supervisord -c /etc/supervisord.conf
	http://127.0.0.1:9001/ 可以在线守护管理thorns的进程，实现远程重启

	# 修改tasks.py内的芹菜配置（分布式任务关键配置项）
	对应你自己的redis-server服务器IP
	BROKER_URL = 'redis://120.132.54.90:6379/0',
	对应你自己的MySQL-server服务器IP
	CELERY_RESULT_BACKEND = 'db+mysql://celery:celery1@3Wscan@42.62.52.62:443/wscan',

	# 环境搭建完毕，这时候访问thorns project的控制台，就会发现worker客户端已经出现在那里
	演示地址：http://thorns.wuyun.org:8080/
	你的请访问：http://youip:8080/


使用说明(可客户端发起任务也可http api发起任务)
-----------------------------------
#### 命令行调用
	在你的任意一台worker客户端，或者thorns服务端
	$ cd /home/thorns/
	$ python run.py 42.62.52.1-42.62.62.254 188
	$ python run.py 42.62.52.1-254 189
	均可以向redis压入nmap扫描任务，worker客户端的分布式集群会自动分发任务执行，并存储到后台数据库
	记得修改wyportmap.py里面的扫描结果，存到你自己的数据库
	
	reinhard-mbp:thorns reinhard$ python run.py 42.62.52.1-254 189
	--------------------------------------------------
	* push 42.62.52.1 to Redis
	* AsyncResult:23147d02-294d-41e5-84e5-5e1b15e72fc4
	--------------------------------------------------
	* push 42.62.52.2 to Redis
	* AsyncResult:542984a4-4434-475f-9a62-bfc81206ea57
	--------------------------------------------------
	* push 42.62.52.3 to Redis
	* AsyncResult:7d005661-d719-41ef-babc-4c853b2c49cc
	--------------------------------------------------
	* push 42.62.52.4 to Redis
	* AsyncResult:ddcf9486-09d9-4dd2-9bb4-2618e6a161b8
	--------------------------------------------------

	wyportmap相关帮助: https://github.com/ring04h/wyportmap

#### HTTP API 远程调用
    重启 worker 线程池:
    $ curl -X POST http://thorns.wuyun.org:8080/api/worker/pool/restart/myworker
    
    远程调用HTTP API启动一个nmap扫描任务：
    $ curl -X POST -d '{"args":["42.62.52.62",2222]}' http://thorns.wuyun.org:8080/api/task/send-task/tasks.nmap_dispath

    强制结束一个正在执行的任务：
    $ curl -X POST -d 'terminate=True' http://thorns.wuyun.org:8088/api/task/revoke/a9361c1b-fd1d-4f48-9be2-8656a57e906b



================================================
FILE: libnmap/__init__.py
================================================
# -*- coding: utf-8 -*-

__author__ = 'Ronald Bister, Mike Boutillier'
__credits__ = ['Ronald Bister', 'Mike Boutillier']
__maintainer__ = 'Ronald Bister'
__email__ = 'mini.pelle@gmail.com'
__license__ = 'CC-BY'
__version__ = '0.6.1'


================================================
FILE: libnmap/diff.py
================================================
# -*- coding: utf-8 -*-


class DictDiffer(object):
    """
        Calculate the difference between two dictionaries as:
        (1) items added
        (2) items removed
        (3) keys same in both but changed values
        (4) keys same in both and unchanged values
    """
    def __init__(self, current_dict, past_dict):
        self.current_dict = current_dict
        self.past_dict = past_dict
        self.set_current = set(current_dict.keys())
        self.set_past = set(past_dict.keys())
        self.intersect = self.set_current.intersection(self.set_past)

    def added(self):
        return self.set_current - self.intersect

    def removed(self):
        return self.set_past - self.intersect

    def changed(self):
        return (set(o for o in self.intersect
                if self.past_dict[o] != self.current_dict[o]))

    def unchanged(self):
        return (set(o for o in self.intersect
                if self.past_dict[o] == self.current_dict[o]))


class NmapDiff(DictDiffer):
    """
        NmapDiff compares two objects of same type to enable the user to check:

        - what has changed
        - what has been added
        - what has been removed
        - what was kept unchanged

        NmapDiff inherit from DictDiffer which makes the actual comparaison.
        The different methods from DictDiffer used by NmapDiff are the
        following:

        - NmapDiff.changed()
        - NmapDiff.added()
        - NmapDiff.removed()
        - NmapDiff.unchanged()

        Each of the returns a python set() of key which have changed in the
        compared objects. To check the different keys that could be returned,
        refer to the get_dict() method of the objects you which to
        compare (i.e: libnmap.objects.NmapHost, NmapService,...).
    """
    def __init__(self, nmap_obj1, nmap_obj2):
        """
            Constructor of NmapDiff:

            - Checks if the two objects are of the same class
            - Checks if the objects are "comparable" via a call to id() (dirty)
            - Inherits from DictDiffer and
        """
        if(nmap_obj1.__class__ != nmap_obj2.__class__ or
           nmap_obj1.id != nmap_obj2.id):
            raise NmapDiffException("Comparing objects with non-matching id")

        self.object1 = nmap_obj1.get_dict()
        self.object2 = nmap_obj2.get_dict()

        DictDiffer.__init__(self, self.object1, self.object2)

    def __repr__(self):
        return ("added: [{0}] -- changed: [{1}] -- "
                "unchanged: [{2}] -- removed [{3}]".format(self.added(),
                                                           self.changed(),
                                                           self.unchanged(),
                                                           self.removed()))


class NmapDiffException(Exception):
    def __init__(self, msg):
        self.msg = msg


================================================
FILE: libnmap/objects/__init__.py
================================================
# -*- coding: utf-8 -*-

from libnmap.objects.report import NmapReport
from libnmap.objects.host import NmapHost
from libnmap.objects.service import NmapService

__all__ = ['NmapReport', 'NmapHost', 'NmapService']


================================================
FILE: libnmap/objects/cpe.py
================================================
# -*- coding: utf-8 -*-


class CPE(object):
    """
        CPE class offers an API for basic CPE objects.
        These objects could be found in NmapService or in <os> tag
        within NmapHost.

        :todo: interpret CPE string and provide appropriate API
    """
    def __init__(self, cpestring):
        self._cpestring = cpestring
        self.cpedict = {}

        zk = ['cpe', 'part', 'vendor', 'product', 'version',
              'update', 'edition', 'language']
        self._cpedict = dict((k, '') for k in zk)
        splitup = cpestring.split(':')
        self._cpedict.update(dict(zip(zk, splitup)))

    @property
    def cpestring(self):
        """
            Accessor for the full CPE string.
        """
        return self._cpestring

    def __repr__(self):
        return self._cpestring

    def get_part(self):
        """
            Returns the cpe part (/o, /h, /a)
        """
        return self._cpedict['part']

    def get_vendor(self):
        """
            Returns the vendor name
        """
        return self._cpedict['vendor']

    def get_product(self):
        """
            Returns the product name
        """
        return self._cpedict['product']

    def get_version(self):
        """
            Returns the version of the cpe
        """
        return self._cpedict['version']

    def get_update(self):
        """
            Returns the update version
        """
        return self._cpedict['update']

    def get_edition(self):
        """
            Returns the cpe edition
        """
        return self._cpedict['edition']

    def get_language(self):
        """
            Returns the cpe language
        """
        return self._cpedict['language']

    def is_application(self):
        """
            Returns True if cpe describes an application
        """
        return (self.get_part() == '/a')

    def is_hardware(self):
        """
            Returns True if cpe describes a hardware
        """
        return (self.get_part() == '/h')

    def is_operating_system(self):
        """
            Returns True if cpe describes an operating system
        """
        return (self.get_part() == '/o')


================================================
FILE: libnmap/objects/host.py
================================================
# -*- coding: utf-8 -*-

from libnmap.diff import NmapDiff
from libnmap.objects.os import NmapOSFingerprint


class NmapHost(object):
    """
        NmapHost is a class representing a host object of NmapReport
    """
    def __init__(self, starttime='', endtime='', address=None, status=None,
                 hostnames=None, services=None, extras=None):
        """
            NmapHost constructor
            :param starttime: unix timestamp of when the scan against
            that host started
            :type starttime: string
            :param endtime: unix timestamp of when the scan against
            that host ended
            :type endtime: string
            :param address: dict ie :{'addr': '127.0.0.1', 'addrtype': 'ipv4'}
            :param status: dict ie:{'reason': 'localhost-response',
                                    'state': 'up'}
            :return: NmapHost:
        """
        self._starttime = starttime
        self._endtime = endtime
        self._hostnames = hostnames if hostnames is not None else []
        self._status = status if status is not None else {}
        self._services = services if services is not None else []
        self._extras = extras if extras is not None else {}
        self._osfingerprinted = False
        self.os = None
        if 'os' in self._extras:
            self.os = NmapOSFingerprint(self._extras['os'])
            self._osfingerprinted = True
        else:
            self.os = NmapOSFingerprint({})

        self._ipv4_addr = None
        self._ipv6_addr = None
        self._mac_addr = None
        self._vendor = None
        for addr in address:
            if addr['addrtype'] == "ipv4":
                self._ipv4_addr = addr['addr']
            elif addr['addrtype'] == 'ipv6':
                self._ipv6_addr = addr['addr']
            elif addr['addrtype'] == 'mac':
                self._mac_addr = addr['addr']
            if 'vendor' in addr:
                self._vendor = addr['vendor']

        self._main_address = self._ipv4_addr or self._ipv6_addr or ''
        self._address = address

    def __eq__(self, other):
        """
            Compare eq NmapHost based on :

                - hostnames
                - address
                - if an associated services has changed

            :return: boolean
        """
        rval = False
        if(self.__class__ == other.__class__ and self.id == other.id):
            rval = (self.changed(other) == 0)
        return rval

    def __ne__(self, other):
        """
            Compare ne NmapHost based on:

                - hostnames
                - address
                - if an associated services has changed

            :return: boolean
        """
        rval = True
        if(self.__class__ == other.__class__ and self.id == other.id):
            rval = (self.changed(other) > 0)
        return rval

    def __repr__(self):
        """
            String representing the object
            :return: string
        """
        return "{0}: [{1} ({2}) - {3}]".format(self.__class__.__name__,
                                               self.address,
                                               " ".join(self._hostnames),
                                               self.status)

    def __hash__(self):
        """
            Hash is needed to be able to use our object in sets
            :return: hash
        """
        return (hash(self.status) ^ hash(self.address) ^
                hash(frozenset(self._services)) ^
                hash(frozenset(" ".join(self._hostnames))))

    def changed(self, other):
        """
            return the number of attribute who have changed
            :param other: NmapHost object to compare
            :return int
        """
        return len(self.diff(other).changed())

    def save(self, backend):
        
        if backend is not None:
            _id = backend.insert(self)
        else:
            raise RuntimeError
        return _id

    @property
    def starttime(self):
        """
            Accessor for the unix timestamp of when the scan was started

            :return: string
        """
        return self._starttime

    @property
    def endtime(self):
        """
            Accessor for the unix timestamp of when the scan ended

            :return: string
        """
        return self._endtime

    @property
    def address(self):
        """
            Accessor for the IP address of the scanned host

            :return: IP address as a string
        """
        return self._main_address

    @address.setter
    def address(self, addrdict):
        """
            Setter for the address dictionnary.

            :param addrdict: valid dict is {'addr': '1.1.1.1',
                                            'addrtype': 'ipv4'}
        """
        if addrdict['addrtype'] == 'ipv4':
            self._ipv4_addr = addrdict['addr']
        elif addrdict['addrtype'] == 'ipv6':
            self._ipv6_addr = addrdict['addr']
        elif addrdict['addrtype'] == 'mac':
            self._mac_addr = addrdict['addr']
        if 'vendor' in addrdict:
            self._vendor = addrdict['vendor']

        self._main_address = self._ipv4_addr or self._ipv6_addr or ''
        self._address = addrdict

    @property
    def ipv4(self):
        """
            Accessor for the IPv4 address of the scanned host

            :return: IPv4 address as a string
        """
        return self._ipv4_addr or ''

    @property
    def mac(self):
        """
            Accessor for the MAC address of the scanned host

            :return: MAC address as a string
        """
        return self._mac_addr or ''

    @property
    def vendor(self):
        """
            Accessor for the vendor attribute of the scanned host

            :return: string (vendor) of empty string if no vendor defined
        """
        return self._vendor or ''

    @property
    def ipv6(self):
        """
            Accessor for the IPv6 address of the scanned host

            :return: IPv6 address as a string
        """
        return self._ipv6_addr or ''

    @property
    def status(self):
        """
            Accessor for the host's status (up, down, unknown...)

            :return: string
        """
        return self._status['state']

    @status.setter
    def status(self, statusdict):
        """
            Setter for the status dictionnary.

            :param statusdict: valid dict is {"state": "open",
                                              "reason": "syn-ack",
                                              "reason_ttl": "0"}
                                'state' is the only mandatory key.
        """
        self._status = statusdict

    def is_up(self):
        """
            method to determine if host is up or not

            :return: bool
        """
        rval = False
        if self.status == 'up':
            rval = True
        return rval

    @property
    def hostnames(self):
        """
            Accessor returning the list of hostnames (array of strings).

            :return: array of string
        """
        return self._hostnames

    @property
    def services(self):
        """
            Accessor for the array of scanned services for that host.

            An array of NmapService objects is returned.

            :return: array of NmapService
        """
        return self._services

    def get_ports(self):
        """
            Retrieve a list of the port used by each service of the NmapHost

            :return: list: of tuples (port,'proto') ie:[(22,'tcp'),(25, 'tcp')]
        """
        return [(p.port, p.protocol) for p in self._services]

    def get_open_ports(self):
        """
            Same as get_ports() but only for open ports

            :return: list: of tuples (port,'proto') ie:[(22,'tcp'),(25, 'tcp')]
        """
        return ([(p.port, p.protocol)
                for p in self._services if p.state == 'open'])

    def get_service(self, portno, protocol='tcp'):
        """
            :param portno: int the portnumber
            :param protocol='tcp': string ('tcp','udp')

            :return: NmapService or None
        """
        plist = [p for p in self._services if
                 p.port == portno and p.protocol == protocol]
        if len(plist) > 1:
            raise Exception("Duplicate services found in NmapHost object")
        return plist.pop() if len(plist) else None

    def get_service_byid(self, service_id):
        """
            Returns a NmapService by providing its id.

            The id of a nmap service is a python tupl made of (protocol, port)
        """
        rval = None
        for _tmpservice in self._services:
            if _tmpservice.id == service_id:
                rval = _tmpservice
        return rval

    def os_class_probabilities(self):
        """
            Returns an array of possible OS class detected during
            the OS fingerprinting.

            :return: Array of NmapOSClass objects
        """
        rval = []
        if self.os is not None:
            rval = self.os.osclasses
        return rval

    def os_match_probabilities(self):
        """
            Returns an array of possible OS match detected during
            the OS fingerprinting

            :return: array of NmapOSMatches objects
        """
        rval = []
        if self.os is not None:
            rval = self.os.osmatches
        return rval

    @property
    def os_fingerprinted(self):
        """
            Specify if the host has OS fingerprint data available

            :return: Boolean
        """
        return self._osfingerprinted

    @property
    def os_fingerprint(self):
        """
            Returns the fingerprint of the scanned system.

            :return: string
        """
        rval = ''
        if self.os is not None:
            rval = "\n".join(self.os.fingerprints)
        return rval

    def os_ports_used(self):
        """
            Returns an array of the ports used for OS fingerprinting

            :return: array of ports used: [{'portid': '22',
                                            'proto': 'tcp',
                                            'state': 'open'},]
        """
        rval = []
        try:
            rval = self._extras['os']['ports_used']
        except (KeyError, TypeError):
            pass
        return rval

    @property
    def tcpsequence(self):
        """
            Returns the difficulty to determine remotely predict
            the tcp sequencing.

            return: string
        """
        rval = ''
        try:
            rval = self._extras['tcpsequence']['difficulty']
        except (KeyError, TypeError):
            pass
        return rval

    @property
    def ipsequence(self):
        """
            Return the class of ip sequence of the remote hosts.

            :return: string
        """
        rval = ''
        try:
            rval = self._extras['ipidsequence']['class']
        except (KeyError, TypeError):
            pass
        return rval

    @property
    def uptime(self):
        """
            uptime of the remote host (if nmap was able to determine it)

            :return: string (in seconds)
        """
        rval = 0
        try:
            rval = int(self._extras['uptime']['seconds'])
        except (KeyError, TypeError):
            pass
        return rval

    @property
    def lastboot(self):
        """
            Since when the host was booted.

            :return: string
        """
        rval = ''
        try:
            rval = self._extras['uptime']['lastboot']
        except (KeyError, TypeError):
            pass
        return rval

    @property
    def distance(self):
        """
            Number of hops to host

            :return: int
        """
        rval = 0
        try:
            rval = int(self._extras['distance']['value'])
        except (KeyError, TypeError):
            pass
        return rval

    @property
    def scripts_results(self):
        """
            Scripts results specific to the scanned host

            :return: array of <script> dictionary
        """
        rval = {}
        try:
            rval = self._extras['hostscript']
        except (KeyError, TypeError):
            pass
        return rval

    @property
    def id(self):
        """
            id of the host. Used for diff()ing NmapObjects

            :return: string
        """
        return self.address

    @property
    def extraports_state(self):
        """
            dictionnary containing state and amount of extra ports scanned
            for which a common state, usually, closed was discovered.

            :return: dict with keys 'state' and 'count' or None
        """
        _xtrports = self._extras.get('extraports', None)

        if _xtrports is None:
            return None

        return {'state': _xtrports['state'], 'count': _xtrports['count']}

    @property
    def extraports_reasons(self):
        """
            dictionnary containing reasons why extra ports scanned
            for which a common state, usually, closed was discovered.

            :return: array of dict containing keys 'state' and 'count' or None
        """
        r = self._extras.get('extraports', {})

        return r.get('reasons', None)

    def get_dict(self):
        """
            Return a dict representation of the object.

            This is needed by NmapDiff to allow comparaison

            :return dict
        """
        d = dict([("{0}::{1}".format(s.__class__.__name__, str(s.id)),
                   hash(s))
                 for s in self.services])

        d.update({'address': self.address, 'status': self.status,
                  'hostnames': " ".join(self._hostnames)})
        return d

    def diff(self, other):
        """
            Calls NmapDiff to check the difference between self and
            another NmapHost object.

            Will return a NmapDiff object.

            This objects return python set() of keys describing the elements
            which have changed, were added, removed or kept unchanged.

            :param other: NmapHost to diff with

            :return: NmapDiff object
        """
        return NmapDiff(self, other)


================================================
FILE: libnmap/objects/os.py
================================================
# -*- coding: utf-8 -*-

import warnings
from libnmap.objects.cpe import CPE


class OSFPPortUsed(object):
    """
        Port used class: this enables the user of NmapOSFingerprint class
        to have a common and clear interface to access portused data which
        were collected and used during os fingerprint scan
    """
    def __init__(self, port_used_dict):
        try:
            self._state = port_used_dict['state']
            self._proto = port_used_dict['proto']
            self._portid = port_used_dict['portid']
        except KeyError:
            raise Exception("Cannot create OSFPPortUsed: missing required key")

    @property
    def state(self):
        """
            Accessor for the portused state (closed, open,...)
        """
        return self._state

    @property
    def proto(self):
        """
            Accessor for the portused protocol (tcp, udp,...)
        """
        return self._proto

    @property
    def portid(self):
        """
            Accessor for the referenced port number used
        """
        return self._portid


class NmapOSMatch(object):
    """
        NmapOSMatch is an internal class used for offering results
        from an nmap os fingerprint. This common interfaces makes
        a compatibility between old nmap xml (<1.04) and new nmap
        xml versions (used in nmapv6 for instance).

        In previous xml version, osclass tags from nmap fingerprints
        were not directly mapped to a osmatch. In new xml version,
        osclass could be embedded in osmatch tag.

        The approach to solve this is to create a common class
        which will, for older xml version, match based on the accuracy
        osclass to an osmatch. If no match, an osmatch will be made up
        from a concat of os class attributes: vendor and osfamily.
        Unmatched osclass will have a line attribute of -1.

        More info, see issue #26 or http://seclists.org/nmap-dev/2012/q2/252
    """
    def __init__(self, osmatch_dict):
        _osmatch_dict = osmatch_dict['osmatch']
        if('name' not in _osmatch_dict or
           'line' not in _osmatch_dict or
           'accuracy' not in _osmatch_dict):
            raise Exception("Cannot create NmapOSClass: missing required key")

        self._name = _osmatch_dict['name']
        self._line = _osmatch_dict['line']
        self._accuracy = _osmatch_dict['accuracy']

        # create osclass list
        self._osclasses = []
        try:
            for _osclass in osmatch_dict['osclasses']:
                try:
                    _osclassobj = NmapOSClass(_osclass)
                except:
                    raise Exception("Could not create NmapOSClass object")
                self._osclasses.append(_osclassobj)
        except KeyError:
            pass

    def add_osclass(self, osclass_obj):

        """
            Add a NmapOSClass object to the OSMatch object. This method is
            useful to implement compatibility with older versions of NMAP
            by providing a common interface to access os fingerprint data.
        """
        self._osclasses.append(osclass_obj)

    @property
    def osclasses(self):
        """
            Accessor for all NmapOSClass objects matching with this OS Match
        """
        return self._osclasses

    @property
    def name(self):
        """
            Accessor for name attribute (e.g.: Linux 2.4.26 (Slackware 10.0.0))
        """
        return self._name

    @property
    def line(self):
        """
            Accessor for line attribute as integer. value equals -1 if this
            osmatch holds orphans NmapOSClass objects. This could happen with
            older version of nmap xml engine (<1.04 (e.g: nmapv6)).

            :return: int
        """
        return int(self._line)

    @property
    def accuracy(self):
        """
            Accessor for accuracy

            :return: int
        """
        return int(self._accuracy)

    def get_cpe(self):
        """
            This method return a list of cpe stings and not CPE objects as
            the NmapOSClass.cpelist property. This method is a helper to
            simplify data management.

            For more advanced handling of CPE data, use NmapOSClass.cpelist
            and use the methods from CPE class
        """
        _cpelist = []
        for osc in self.osclasses:
            for cpe in osc.cpelist:
                _cpelist.append(cpe.cpestring)
        return _cpelist

    def __repr__(self):
        rval = "{0}: {1}".format(self.name, self.accuracy)
        for _osclass in self._osclasses:
            rval += "\r\n  |__ os class: {0}".format(str(_osclass))
        return rval


class NmapOSClass(object):
    """
        NmapOSClass offers an unified API to access data from analysed
        osclass tag. As implemented in libnmap and newer version of nmap,
        osclass objects will always be embedded in a NmapOSMatch.
        Unmatched NmapOSClass will be stored in "dummy" NmapOSMatch objects
        which will have the particularity of have a line attribute of -1.
        On top of this, NmapOSClass will have optional CPE objects
        embedded.
    """
    def __init__(self, osclass_dict):
        _osclass = osclass_dict['osclass']
        if('vendor' not in _osclass or
           'osfamily' not in _osclass or
           'accuracy' not in _osclass):
            raise Exception("Wrong osclass structure: missing required key")

        self._vendor = _osclass['vendor']
        self._osfamily = _osclass['osfamily']
        self._accuracy = _osclass['accuracy']

        self._osgen = ''
        self._type = ''
        # optional data
        if 'osgen' in _osclass:
            self._osgen = _osclass['osgen']
        if 'type' in _osclass:
            self._type = _osclass['type']

        self._cpelist = []
        for _cpe in osclass_dict['cpe']:
            self._cpelist.append(CPE(_cpe))

    @property
    def cpelist(self):
        """
            Returns a list of CPE Objects matching with this os class

            :return: list of CPE objects
            :rtype: Array
        """
        return self._cpelist

    @property
    def vendor(self):
        """
            Accessor for vendor information (Microsoft, Linux,...)

            :return: string
        """
        return self._vendor

    @property
    def osfamily(self):
        """
            Accessor for OS family information (Windows, Linux,...)

            :return: string
        """
        return self._osfamily

    @property
    def accuracy(self):
        """
            Accessor for OS class detection accuracy (int)

            :return: int
        """
        return int(self._accuracy)

    @property
    def osgen(self):
        """
            Accessor for OS class generation (7, 8, 2.4.X,...).

            :return: string
        """
        return self._osgen

    @property
    def type(self):
        """
            Accessor for OS class type (general purpose,...)

            :return: string
        """
        return self._type

    @property
    def description(self):
        """
            Accessor helper which returns a concataned string of
            the valuable attributes from NmapOSClass object

            :return: string
        """
        rval = "{0}: {1}, {2}".format(self.type, self.vendor, self.osfamily)
        if len(self.osgen):
            rval += "({0})".format(self.osgen)
        return rval

    def __repr__(self):
        rval = "{0}: {1}, {2}".format(self.type, self.vendor, self.osfamily)
        if len(self.osgen):
            rval += "({0})".format(self.osgen)
        for _cpe in self._cpelist:
            rval += "\r\n    |__ {0}".format(str(_cpe))
        return rval


class NmapOSFingerprint(object):
    """
        NmapOSFingerprint is a easier API for using os fingerprinting.
        Data for OS fingerprint (<os> tag) is instanciated from
        a NmapOSFingerprint which is accessible in NmapHost via NmapHost.os
    """
    def __init__(self, osfp_data):
        self.__osmatches = []
        self.__ports_used = []
        self.__fingerprints = []

        if 'osmatches' in osfp_data:
            for _osmatch in osfp_data['osmatches']:
                _osmatch_obj = NmapOSMatch(_osmatch)
                self.__osmatches.append(_osmatch_obj)
        if 'osclasses' in osfp_data:
            for _osclass in osfp_data['osclasses']:
                _osclass_obj = NmapOSClass(_osclass)
                _osmatched = self.get_osmatch(_osclass_obj)
                if _osmatched is not None:
                    _osmatched.add_osclass(_osclass_obj)
                else:
                    self._add_dummy_osmatch(_osclass_obj)
        if 'osfingerprints' in osfp_data:
            for _osfp in osfp_data['osfingerprints']:
                if 'fingerprint' in _osfp:
                    self.__fingerprints.append(_osfp['fingerprint'])
        if 'ports_used' in osfp_data:
            for _pused_dict in osfp_data['ports_used']:
                _pused = OSFPPortUsed(_pused_dict)
                self.__ports_used.append(_pused)

    def get_osmatch(self, osclass_obj):
        """
            This function enables NmapOSFingerprint to determine if an
            NmapOSClass object could be attached to an existing NmapOSMatch
            object in order to respect the common interface for
            the nmap xml version < 1.04 and >= 1.04

            This method will return an NmapOSMatch object matching with
            the NmapOSClass provided in parameter
            (match is performed based on accuracy)

            :return: NmapOSMatch object
        """
        rval = None
        for _osmatch in self.__osmatches:
            if _osmatch.accuracy == osclass_obj.accuracy:
                rval = _osmatch
                break  # sorry
        return rval

    def _add_dummy_osmatch(self, osclass_obj):
        """
            This functions creates a dummy NmapOSMatch object in order to
            encapsulate an NmapOSClass object which was not matched with an
            existing NmapOSMatch object
        """
        _dname = "{0}:{1}:{2}".format(osclass_obj.type,
                                      osclass_obj.vendor,
                                      osclass_obj.osfamily)
        _dummy_dict = {'osmatch': {'name': _dname,
                                   'accuracy': osclass_obj.accuracy,
                                   'line': -1},
                       'osclasses': []}
        _dummy_osmatch = NmapOSMatch(_dummy_dict)
        self.__osmatches.append(_dummy_osmatch)

    @property
    def osmatches(self, min_accuracy=0):
        _osmatches = []

        for _osmatch in self.__osmatches:
            if _osmatch.accuracy >= min_accuracy:
                _osmatches.append(_osmatch)

        return _osmatches

    @property
    def fingerprint(self):
        return "\r\n".join(self.__fingerprints)

    @property
    def fingerprints(self):
        return self.__fingerprints

    @property
    def ports_used(self):
        """
            Return an array of OSFPPortUsed object with the ports used to
            perform the os fingerprint. This dict might contain another dict
            embedded containing the ports_reason values.
        """
        return self.__ports_used

    def osmatch(self, min_accuracy=90):
        warnings.warn("NmapOSFingerprint.osmatch is deprecated: "
                      "use NmapOSFingerprint.osmatches", DeprecationWarning)
        os_array = []
        for _osmatch in self.__osmatches:
            if _osmatch.accuracy >= min_accuracy:
                os_array.append(_osmatch.name)
        return os_array

    def osclass(self, min_accuracy=90):
        warnings.warn("NmapOSFingerprint.osclass() is deprecated: "
                      "use NmapOSFingerprint.osclasses() if applicable",
                      DeprecationWarning)
        os_array = []
        for osmatch_entry in self.osmatches():
            if osmatch_entry.accuracy >= min_accuracy:
                for oclass in osmatch_entry.osclasses:
                    _ftstr = "type:{0}|vendor:{1}|osfamily{2}".format(
                             oclass.type,
                             oclass.vendor,
                             oclass.osfamily)
                    os_array.append(_ftstr)
        return os_array

    def os_cpelist(self):
        cpelist = []
        for _osmatch in self.osmatches:
            for oclass in _osmatch.osclasses:
                cpelist.extend(oclass.cpelist)
        return cpelist

    def __repr__(self):
        rval = ""
        for _osmatch in self.osmatches:
            rval += "\r\n{0}".format(_osmatch)
        rval += "Fingerprints: ".format(self.fingerprint)
        return rval


================================================
FILE: libnmap/objects/report.py
================================================
# -*- coding: utf-8 -*-
from libnmap.diff import NmapDiff


class NmapReport(object):
    """
        NmapReport is the usual interface for the end user to
        read scans output.

        A NmapReport as the following structure:

        - Scan headers data
        - A list of scanned hosts (NmapReport.hosts)
        - Scan footer data

        It is to note that each NmapHost comprised in NmapReport.hosts array
        contains also a list of scanned services (NmapService object).

        This means that if NmapParser.parse*() is the input interface for the
        end user of the lib. NmapReport is certainly the output interface for
        the end user of the lib.
    """
    def __init__(self, raw_data=None):
        """
            Constructor for NmapReport object.

            This is usually called by the NmapParser module.
        """
        self._nmaprun = {}
        self._scaninfo = {}
        self._hosts = []
        self._runstats = {}
        if raw_data is not None:
            self.__set_raw_data(raw_data)

    def save(self, backend):
        """
            This method gets a NmapBackendPlugin representing the backend.

            :param backend: libnmap.plugins.PluginBackend object.

            Object created by BackendPluginFactory and enabling nmap reports
            to be saved/stored in any type of backend implemented in plugins.

            The primary key of the stored object is returned.

            :return: str
        """
        if backend is not None:
            _id = backend.insert(self)
        else:
            raise RuntimeError
        return _id

    def diff(self, other):
        """
            Calls NmapDiff to check the difference between self and
            another NmapReport object.

            Will return a NmapDiff object.

            :return: NmapDiff object
            :todo: remove is_consistent approach, diff() should be generic.
        """
        if self.is_consistent() and other.is_consistent():
            _rdiff = NmapDiff(self, other)
        else:
            _rdiff = set()
        return _rdiff

    @property
    def started(self):
        """
            Accessor returning a unix timestamp of when the scan was started.

            :return: integer
        """
        rval = -1
        try:
            s_start = self._nmaprun['start']
            rval = int(s_start)
        except(KeyError, TypeError, ValueError):
            pass
        return rval

    @property
    def commandline(self):
        """
            Accessor returning the full nmap command line fired.

            :return: string
        """
        return self._nmaprun['args']

    @property
    def version(self):
        """
            Accessor returning the version of the
            nmap binary used to perform the scan.

            :return: string
        """
        return self._nmaprun['version']

    @property
    def scan_type(self):
        """
            Accessor returning a string which identifies what type of scan
            was launched (syn, ack, tcp,...).

            :return: string
        """
        return self._scaninfo['type']

    @property
    def hosts(self):
        """
            Accessor returning an array of scanned hosts.

            Scanned hosts are NmapHost objects.

            :return: array of NmapHost
        """
        return self._hosts

    def get_host_byid(self, host_id):
        """
           Gets a NmapHost object directly from the host array
           by looking it up by id.

           :param ip_addr: ip address of the host to lookup
           :type ip_addr: string

           :return: NmapHost
        """
        rval = None
        for _rhost in self._hosts:
            if _rhost.address == host_id:
                rval = _rhost
        return rval

    @property
    def endtime(self):
        """
            Accessor returning a unix timestamp of when the scan ended.

            :return: integer
        """
        rval = -1
        try:
            rval = int(self._runstats['finished']['time'])
        except(KeyError, TypeError, ValueError):
            pass
        return rval

    @property
    def endtimestr(self):
        """
            Accessor returning a human readable time string
            of when the scan ended.

            :return: string
        """
        rval = ''
        try:
            rval = self._runstats['finished']['timestr']
        except(KeyError, TypeError, ValueError):
            pass
        return rval

    @property
    def summary(self):
        """
            Accessor returning a string describing and
            summarizing the scan.

            :return: string
        """
        rval = ''
        try:
            rval = self._runstats['finished']['summary']
        except(KeyError, TypeError):
            pass

        if len(rval) == 0:
            rval = ("Nmap ended at {0} ; {1} IP addresses ({2} hosts up)"
                    " scanned in {3} seconds".format(self.endtimestr,
                                                     self.hosts_total,
                                                     self.hosts_up,
                                                     self.elapsed))
        return rval

    @property
    def elapsed(self):
        """
            Accessor returning the number of seconds the scan took

            :return: float (0 >= or -1)
        """
        rval = -1
        try:
            s_elapsed = self._runstats['finished']['elapsed']
            rval = float(s_elapsed)
        except (KeyError, TypeError, ValueError):
            rval = -1
        return rval

    @property
    def hosts_up(self):
        """
            Accessor returning the numer of host detected
            as 'up' during the scan.

            :return: integer (0 >= or -1)
        """
        rval = -1
        try:
            s_up = self._runstats['hosts']['up']
            rval = int(s_up)
        except (KeyError, TypeError, ValueError):
            rval = -1
        return rval

    @property
    def hosts_down(self):
        """
            Accessor returning the numer of host detected
            as 'down' during the scan.

            :return: integer (0 >= or -1)
        """
        rval = -1
        try:
            s_down = self._runstats['hosts']['down']
            rval = int(s_down)
        except (KeyError, TypeError, ValueError):
            rval = -1
        return rval

    @property
    def hosts_total(self):
        """
            Accessor returning the number of hosts scanned in total.

            :return: integer (0 >= or -1)
        """
        rval = -1
        try:
            s_total = self._runstats['hosts']['total']
            rval = int(s_total)
        except (KeyError, TypeError, ValueError):
            rval = -1
        return rval

    def get_raw_data(self):
        """
            Returns a dict representing the NmapReport object.

            :return: dict
            :todo: deprecate. get rid of this uglyness.
        """
        raw_data = {'_nmaprun': self._nmaprun,
                    '_scaninfo': self._scaninfo,
                    '_hosts': self._hosts,
                    '_runstats': self._runstats}
        return raw_data

    def __set_raw_data(self, raw_data):
        self._nmaprun = raw_data['_nmaprun']
        self._scaninfo = raw_data['_scaninfo']
        self._hosts = raw_data['_hosts']
        self._runstats = raw_data['_runstats']

    def is_consistent(self):
        """
            Checks if the report is consistent and can be diffed().

            This needs to be rewritten and removed: diff() should be generic.

            :return: boolean
        """
        rval = False
        rdata = self.get_raw_data()
        _consistent_keys = ['_nmaprun', '_scaninfo', '_hosts', '_runstats']
        if(set(_consistent_keys) == set(rdata.keys()) and
           len([dky for dky in rdata.keys() if rdata[dky] is not None]) == 4):
            rval = True
        return rval

    def get_dict(self):
        """
            Return a python dict representation of the NmapReport object.
            This is used to diff() NmapReport objects via NmapDiff.

            :return: dict
        """
        rdict = dict([("{0}::{1}".format(_host.__class__.__name__,
                                         str(_host.id)),
                     hash(_host)) for _host in self.hosts])
        rdict.update({'commandline': self.commandline,
                      'version': self.version,
                      'scan_type': self.scan_type,
                      'elapsed': self.elapsed,
                      'hosts_up': self.hosts_up,
                      'hosts_down': self.hosts_down,
                      'hosts_total': self.hosts_total})
        return rdict

    @property
    def id(self):
        """
            Dummy id() defined for reports.
        """
        return hash(1)

    def __eq__(self, other):
        """
            Compare eq NmapReport based on :

                - create a diff obj and check the result
                report are equal if added&changed&removed are empty

            :return: boolean
        """
        rval = False
        if(self.__class__ == other.__class__ and self.id == other.id):
            diffobj = self.diff(other)
            rval = (len(diffobj.changed()) == 0 and
                    len(diffobj.added()) == 0 and
                    len(diffobj.removed()) == 0
                    )
        return rval

    def __ne__(self, other):
        """
            Compare ne NmapReport based on:

                - create a diff obj and check the result
                report are ne if added|changed|removed are not empty

            :return: boolean
        """
        rval = True
        if(self.__class__ == other.__class__ and self.id == other.id):
            diffobj = self.diff(other)
            rval = (len(diffobj.changed()) != 0 or
                    len(diffobj.added()) != 0 or
                    len(diffobj.removed()) != 0
                    )
        return rval

    def __repr__(self):
        return "{0}: started at {1} hosts up {2}/{3}".format(
               self.__class__.__name__,
               self.started,
               self.hosts_up,
               self.hosts_total)


================================================
FILE: libnmap/objects/service.py
================================================
# -*- coding: utf-8 -*-
from libnmap.diff import NmapDiff
from libnmap.objects.os import CPE


class NmapService(object):
    """
        NmapService represents a nmap scanned service. Its id() is comprised
        of the protocol and the port.

        Depending on the scanning options, some additional details might be
        available or not. Like banner or extra datas from NSE (nmap scripts).
    """
    def __init__(self, portid, protocol='tcp', state=None,
                 service=None, owner=None, service_extras=None):
        """
            Constructor

            :param portid: port number
            :type portid: string
            :param protocol: protocol of port scanned (tcp, udp)
            :type protocol: string
            :param state: python dict describing the service status
            :type state: python dict
            :param service: python dict describing the service name and banner
            :type service: python dict
            :param service_extras: additional info about the tested service
            like scripts' data
        """
        try:
            self._portid = int(portid or -1)
        except (ValueError, TypeError):
            raise
        if self._portid < 0 or self._portid > 65535:
            raise ValueError

        self._protocol = protocol
        # self._taskid = taskid
        # self._address = address
        self._state = state if state is not None else {}
        self._service = service if service is not None else {}

        self._cpelist = []
        if 'cpelist' in self._service:
            for _cpe in self._service['cpelist']:
                _cpeobj = CPE(_cpe)
                self._cpelist.append(_cpeobj)

        self._owner = ''
        if owner is not None and 'name' in owner:
            self._owner = owner['name']

        self._reason = ''
        self._reason_ip = ''
        self._reason_ttl = ''
        self._servicefp = ''
        self._tunnel = ''

        if 'reason' in self._state:
            self._reason = self._state['reason']
        if 'reason_ttl' in self._state:
            self._reason_ttl = self._state['reason_ttl']
        if 'reason_ip' in self._state:
            self._reason_ip = self._state['reason_ip']

        if 'servicefp' in self._service:
            self._servicefp = self._service['servicefp']
        if 'tunnel' in self._service:
            self._tunnel = self._service['tunnel']

        self._service_extras = []
        if service_extras is not None:
            self._service_extras = service_extras

    def __eq__(self, other):
        """
            Compares two NmapService objects to see if they are the same or
            if one of them changed.

            :param other: NmapService

            :return: boolean
        """
        rval = False
        if(self.__class__ == other.__class__ and self.id == other.id):
            rval = (self.changed(other) == 0)
        return rval

    def __ne__(self, other):
        """
            Compares two NmapService objects to see if they are different
            if one of them changed.

            :param other: NmapService

            :return: boolean
        """
        rval = True
        if(self.__class__ == other.__class__ and self.id == other.id):
            rval = (self.changed(other) > 0)
        return rval

    def __repr__(self):
        return "{0}: [{1} {2}/{3} {4} ({5})]".format(self.__class__.__name__,
                                                     self.state,
                                                     str(self.port),
                                                     self.protocol,
                                                     self.service,
                                                     self.banner)

    def __hash__(self):
        return (hash(self.port) ^ hash(self.protocol) ^ hash(self.state) ^
                hash(self.reason) ^ hash(self.service) ^ hash(self.banner))

    def changed(self, other):
        """
            Checks if a NmapService is different from another.

            :param other: NmapService

            :return: boolean
        """
        return len(self.diff(other).changed())

    def save(self, backend):
        if backend is not None:
            _id = backend.insert(self)
        else:
            raise RuntimeError
        return _id

    @property
    def port(self):
        """
            Accessor for port.

            :return: integer or -1
        """
        return self._portid

    @property
    def protocol(self):
        """
            Accessor for protocol

            :return: string
        """
        return self._protocol

    @property
    def state(self):
        """
            Accessor for service's state (open, filtered, closed,...)

            :return: string
        """
        return self._state['state'] if 'state' in self._state else None

    @property
    def reason(self):
        """
            Accessor for service's state reason (syn-ack, filtered,...)

            :return: string or empty if not applicable
        """
        return self._reason

    @property
    def reason_ip(self):
        """
            Accessor for service's state reason ip

            :return: string or empty if not applicable
        """
        return self._reason_ip

    @property
    def reason_ttl(self):
        """
            Accessor for service's state reason ttl

            :return: string or empty if not applicable
        """
        return self._reason_ttl

    @property
    def service(self):
        """
            Accessor for service name.

            :return: string or empty
        """
        return self._service['name'] if 'name' in self._service else ''

    @property
    def service_dict(self):
        """
            Accessor for service dictionary.

            :return: dict or None
        """
        return self._service

    def open(self):
        """
            Tells if the port was open or not

            :return: boolean
        """
        return 'state' in self._state and self._state['state'] == 'open'

    @property
    def product(self):
        if 'product' in self._service.keys():
            return self._service['product'].rstrip()
        else:
            return None

    @property
    def product_version(self):
        if 'version' in self._service.keys():
            return self._service['version'].rstrip()
        else:
            return None

    @property
    def product_extrainfo(self):
        if 'extrainfo' in self._service.keys():
            return self._service['extrainfo'].rstrip()
        else:
            return None

    @property
    def owner(self):
        """
            Accessor for service owner if available
        """
        return self._owner

    @property
    def banner(self):
        """
            Accessor for the service's banner. Only available
            if the nmap option -sV or similar was used.

            :return: string
        """
        notrelevant = ['name', 'method', 'conf', 'cpelist',
                       'servicefp', 'tunnel']
        relevant = ['product', 'version', 'extrainfo']
        b = ''
        skeys = self._service.keys()
        if 'method' in self._service and self._service['method'] == "probed":
            for relk in relevant:
                if relk in skeys:
                    b += '{0}: {1} '.format(relk, self._service[relk])
            for mkey in skeys:
                if mkey not in notrelevant and mkey not in relevant:
                    b += '{0}: {1} '.format(mkey, self._service[mkey])
        return b.rstrip()

    @property
    def cpelist(self):
        """
            Accessor for list of CPE for this particular service
        """
        return self._cpelist

    @property
    def scripts_results(self):
        """
            Gives a python list of the nse scripts results.

            The dict key is the name (id) of the nse script and
            the value is the output of the script.

            :return: dict
        """
        scripts_dict = None
        try:
            scripts_dict = self._service_extras['scripts']
        except (KeyError, TypeError):
            pass
        return scripts_dict

    @property
    def servicefp(self):
        """
            Accessor for the service's fingerprint
            if the nmap option -sV or -A is used

            :return: string if available
        """
        return self._servicefp

    @property
    def tunnel(self):
        """
            Accessor for the service's tunnel type
            if applicable and available from scan
            results

            :return: string if available
        """
        return self._tunnel

    @property
    def id(self):
        """
            Accessor for the id() of the NmapService.

            This is used for diff()ing NmapService object via NmapDiff.

            :return: tuple
        """
        return "{0}.{1}".format(self.protocol, self.port)

    def get_dict(self):
        """
            Return a python dict representation of the NmapService object.

            This is used to diff() NmapService objects via NmapDiff.

            :return: dict
        """
        return ({'id': str(self.id), 'port': str(self.port),
                 'protocol': self.protocol, 'banner': self.banner,
                 'service': self.service, 'state': self.state,
                 'reason': self.reason})

    def diff(self, other):
        """
            Calls NmapDiff to check the difference between self and
            another NmapService object.

            Will return a NmapDiff object.

            This objects return python set() of keys describing the elements
            which have changed, were added, removed or kept unchanged.

            :return: NmapDiff object
        """
        return NmapDiff(self, other)


================================================
FILE: libnmap/parser.py
================================================
# -*- coding: utf-8 -*-


try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET
from libnmap.objects import NmapHost, NmapService, NmapReport


class NmapParser(object):
    @classmethod
    def parse(cls, nmap_data=None, data_type='XML', incomplete=False):
        """
            Generic class method of NmapParser class.

            The data to be parsed does not need to be a complete nmap
            scan report. You can possibly give <hosts>...</hosts>
            or <port> XML tags.

            :param nmap_data: any portion of nmap scan result. \
            nmap_data should always be a string representing a part \
            or a complete nmap scan report.
            :type nmap_data: string

            :param data_type: specifies the type of data to be parsed.
            :type data_type: string ("XML"|"JSON"|"YAML").

            :param incomplete: enable you to parse interrupted nmap scans \
            and/or incomplete nmap xml blocks by adding a </nmaprun> at \
            the end of the scan.
            :type incomplete: boolean

            As of today, only XML parsing is supported.

            :return: NmapObject (NmapHost, NmapService or NmapReport)
        """

        nmapobj = None
        if data_type == "XML":
            nmapobj = cls._parse_xml(nmap_data, incomplete)
        else:
            raise NmapParserException("Unknown data type provided. "
                                      "Please check documentation for "
                                      "supported data types.")
        return nmapobj

    @classmethod
    def _parse_xml(cls, nmap_data=None, incomplete=False):
        """
            Protected class method used to process a specific data type.
            In this case: XML. This method is called by cls.parse class
            method and receives nmap scan results data (in XML).

            :param nmap_data: any portion of nmap scan result can be given \
            as argument. nmap_data should always be a string representing \
            a part or a complete nmap scan report.
            :type nmap_data: string

            This method checks which portion of a nmap scan is given \
            as argument.
            It could be:

                1. a full nmap scan report;
                2. a scanned host: <host> tag in a nmap scan report
                3. a scanned service: <port> tag
                4. a list of hosts: <hosts/> tag (TODO)
                5. a list of ports: <ports/> tag

            :param incomplete: enable you to parse interrupted nmap scans \
            and/or incomplete nmap xml blocks by adding a </nmaprun> at \
            the end of the scan.
            :type incomplete: boolean

            :return: NmapObject (NmapHost, NmapService or NmapReport) \
                    or a list of NmapObject
        """

        if not nmap_data:
            raise NmapParserException("No report data to parse: please "
                                      "provide a valid XML nmap report")
        elif not isinstance(nmap_data, str):
            raise NmapParserException("wrong nmap_data type given as "
                                      "argument: cannot parse data")

        if incomplete is True:
            nmap_data += "</nmaprun>"

        try:
            root = ET.fromstring(nmap_data)
        except:
            raise NmapParserException("Wrong XML structure: cannot parse data")

        nmapobj = None
        if root.tag == 'nmaprun':
            nmapobj = cls._parse_xml_report(root)
        elif root.tag == 'host':
            nmapobj = cls._parse_xml_host(root)
        elif root.tag == 'ports':
            nmapobj = cls._parse_xml_ports(root)
        elif root.tag == 'port':
            nmapobj = cls._parse_xml_port(root)
        else:
            raise NmapParserException("Unpexpected data structure for XML "
                                      "root node")
        return nmapobj

    @classmethod
    def _parse_xml_report(cls, root=None):
        """
            This method parses out a full nmap scan report from its XML root
            node: <nmaprun>.

            :param root: Element from xml.ElementTree (top of XML the document)
            :type root: Element

            :return: NmapReport object
        """

        nmap_scan = {'_nmaprun': {}, '_scaninfo': {},
                     '_hosts': [], '_runstats': {}}

        if root is None:
            raise NmapParserException("No root node provided to parse XML "
                                      "report")

        nmap_scan['_nmaprun'] = cls.__format_attributes(root)
        for el in root:
            if el.tag == 'scaninfo':
                nmap_scan['_scaninfo'] = cls.__parse_scaninfo(el)
            elif el.tag == 'host':
                nmap_scan['_hosts'].append(cls._parse_xml_host(el))
            elif el.tag == 'runstats':
                nmap_scan['_runstats'] = cls.__parse_runstats(el)
            # else:
            #    print "struct pparse unknown attr: {0} value: {1}".format(
            #        el.tag,
            #        el.get(el.tag))
        return NmapReport(nmap_scan)

    @classmethod
    def parse_fromstring(cls, nmap_data, data_type="XML", incomplete=False):
        """
            Call generic cls.parse() method and ensure that a string is \
            passed on as argument. If not, an exception is raised.

            :param nmap_data: Same as for parse(), any portion of nmap scan. \
            Reports could be passed as argument. Data type _must_ be a string.

            :type nmap_data: string

            :param data_type: Specifies the type of data passed on as argument.

            :param incomplete: enable you to parse interrupted nmap scans \
            and/or incomplete nmap xml blocks by adding a </nmaprun> at \
            the end of the scan.
            :type incomplete: boolean

            :return: NmapObject
        """

        if not isinstance(nmap_data, str):
            raise NmapParserException("bad argument type for "
                                      "xarse_fromstring(): should be a string")
        return cls.parse(nmap_data, data_type, incomplete)

    @classmethod
    def parse_fromfile(cls, nmap_report_path,
                       data_type="XML",
                       incomplete=False):
        """
            Call generic cls.parse() method and ensure that a correct file \
            path is given as argument. If not, an exception is raised.

            :param nmap_data: Same as for parse(). \
            Any portion of nmap scan reports could be passed as argument. \
            Data type _must be a valid path to a file containing \
            nmap scan results.

            :param data_type: Specifies the type of serialization in the file.

            :param incomplete: enable you to parse interrupted nmap scans \
            and/or incomplete nmap xml blocks by adding a </nmaprun> at \
            the end of the scan.
            :type incomplete: boolean

            :return: NmapObject
        """

        try:
            with open(nmap_report_path, 'r') as fileobj:
                fdata = fileobj.read()
                rval = cls.parse(fdata, data_type, incomplete)
        except IOError:
            raise
        return rval

    @classmethod
    def parse_fromdict(cls, rdict):
        """
            Strange method which transforms a python dict \
            representation of a NmapReport and turns it into an \
            NmapReport object. \
            Needs to be reviewed and possibly removed.

            :param rdict: python dict representation of an NmapReport
            :type rdict: dict

            :return: NmapReport
        """

        nreport = {}
        if list(rdict.keys())[0] == '__NmapReport__':
            r = rdict['__NmapReport__']
            nreport['_runstats'] = r['_runstats']
            nreport['_scaninfo'] = r['_scaninfo']
            nreport['_nmaprun'] = r['_nmaprun']
            hlist = []
            for h in r['_hosts']:
                slist = []
                for s in h['__NmapHost__']['_services']:
                    cname = '__NmapService__'
                    slist.append(NmapService(portid=s[cname]['_portid'],
                                             protocol=s[cname]['_protocol'],
                                             state=s[cname]['_state'],
                                             owner=s[cname]['_owner'],
                                             service=s[cname]['_service']))

                nh = NmapHost(starttime=h['__NmapHost__']['_starttime'],
                              endtime=h['__NmapHost__']['_endtime'],
                              address=h['__NmapHost__']['_address'],
                              status=h['__NmapHost__']['_status'],
                              hostnames=h['__NmapHost__']['_hostnames'],
                              extras=h['__NmapHost__']['_extras'],
                              services=slist)
                hlist.append(nh)
            nreport['_hosts'] = hlist
            nmapobj = NmapReport(nreport)
        return nmapobj

    @classmethod
    def __parse_scaninfo(cls, scaninfo_data):
        """
            Private method parsing a portion of a nmap scan result.
            Receives a <scaninfo> XML tag.

            :param scaninfo_data: <scaninfo> XML tag from a nmap scan
            :type scaninfo_data: xml.ElementTree.Element or a string

            :return: python dict representing the XML scaninfo tag
        """

        xelement = cls.__format_element(scaninfo_data)
        return cls.__format_attributes(xelement)

    @classmethod
    def _parse_xml_host(cls, scanhost_data):
        """
            Protected method parsing a portion of a nmap scan result.
            Receives a <host> XML tag representing a scanned host with
            its services.

            :param scaninfo_data: <host> XML tag from a nmap scan
            :type scaninfo_data: xml.ElementTree.Element or a string

            :return: NmapHost object
        """

        xelement = cls.__format_element(scanhost_data)
        _host_header = cls.__format_attributes(xelement)
        _hostnames = []
        _services = []
        _status = {}
        _addresses = []
        _host_extras = {}
        extra_tags = ['uptime', 'distance', 'tcpsequence',
                      'ipidsequence', 'tcptssequence', 'times']
        for xh in xelement:
            if xh.tag == 'hostnames':
                for hostname in cls.__parse_hostnames(xh):
                    _hostnames.append(hostname)
            elif xh.tag == 'ports':
                ports_dict = cls._parse_xml_ports(xh)
                for port in ports_dict['ports']:
                    _services.append(port)
                _host_extras['extraports'] = ports_dict['extraports']
            elif xh.tag == 'status':
                _status = cls.__format_attributes(xh)
            elif xh.tag == 'address':
                _addresses.append(cls.__format_attributes(xh))
            elif xh.tag == 'os':
                _os_extra = cls.__parse_os_fingerprint(xh)
                _host_extras.update({'os': _os_extra})
            elif xh.tag == 'hostscript':
                _host_scripts = cls.__parse_host_scripts(xh)
                _host_extras.update({'hostscript': _host_scripts})
            elif xh.tag in extra_tags:
                _host_extras[xh.tag] = cls.__format_attributes(xh)
            # else:
            #    print "struct host unknown attr: %s value: %s" %
            #           (h.tag, h.get(h.tag))
        _stime = ''
        _etime = ''
        if 'starttime' in _host_header:
            _stime = _host_header['starttime']
        if 'endtime' in _host_header:
            _etime = _host_header['endtime']
        nhost = NmapHost(_stime,
                         _etime,
                         _addresses,
                         _status,
                         _hostnames,
                         _services,
                         _host_extras)
        return nhost

    @classmethod
    def __parse_hostnames(cls, scanhostnames_data):
        """
            Private method parsing the hostnames list within a <host> XML tag.

            :param scanhostnames_data: <hostnames> XML tag from a nmap scan
            :type scanhostnames_data: xml.ElementTree.Element or a string

            :return: list of hostnames
        """

        xelement = cls.__format_element(scanhostnames_data)
        hostnames = []
        for hname in xelement:
            if hname.tag == 'hostname':
                hostnames.append(hname.get('name'))
        return hostnames

    @classmethod
    def _parse_xml_ports(cls, scanports_data):
        """
            Protected method parsing the list of scanned services from
            a targeted host. This protected method cannot be called directly
            with a string. A <ports/> tag can be directly passed to parse()
            and the below method will be called and return a list of nmap
            scanned services.

            :param scanports_data: <ports> XML tag from a nmap scan
            :type scanports_data: xml.ElementTree.Element or a string

            :return: list of NmapService
        """

        xelement = cls.__format_element(scanports_data)

        rdict = {'ports': [], 'extraports': None}
        for xservice in xelement:
            if xservice.tag == 'port':
                nport = cls._parse_xml_port(xservice)
                rdict['ports'].append(nport)
            elif xservice.tag == 'extraports':
                extraports = cls.__parse_extraports(xservice)
                rdict['extraports'] = extraports
            # else:
            #    print "struct port unknown attr: %s value: %s" %
            #           (h.tag, h.get(h.tag))
        return rdict

    @classmethod
    def _parse_xml_port(cls, scanport_data):
        """
            Protected method parsing a scanned service from a targeted host.
            This protected method cannot be called directly.
            A <port/> tag can be directly passed to parse() and the below
            method will be called and return a NmapService object
            representing the state of the service.

            :param scanport_data: <port> XML tag from a nmap scan
            :type scanport_data: xml.ElementTree.Element or a string

            :return: NmapService
        """

        xelement = cls.__format_element(scanport_data)

        _port = cls.__format_attributes(xelement)
        _portid = _port['portid'] if 'portid' in _port else None
        _protocol = _port['protocol'] if 'protocol' in _port else None

        _state = None
        _service = None
        _owner = None
        _service_scripts = []
        _service_extras = {}
        for xport in xelement:
            if xport.tag == 'state':
                _state = cls.__format_attributes(xport)
            elif xport.tag == 'service':
                _service = cls.__parse_service(xport)
            elif xport.tag == 'owner':
                _owner = cls.__format_attributes(xport)
            elif xport.tag == 'script':
                _script_dict = cls.__parse_script(xport)
                _service_scripts.append(_script_dict)
        _service_extras['scripts'] = _service_scripts

        if(_portid is None or _protocol is None or _state is None):
            raise NmapParserException("XML <port> tag is incomplete. One "
                                      "of the following tags is missing: "
                                      "portid, protocol or state or tag.")

        nport = NmapService(_portid,
                            _protocol,
                            _state,
                            _service,
                            _owner,
                            _service_extras)
        return nport

    @classmethod
    def __parse_service(cls, xserv):
        """
            Parse <service> tag to manage CPE object
        """
        _service = cls.__format_attributes(xserv)
        _cpelist = []
        for _servnode in xserv:
            if _servnode.tag == 'cpe':
                _cpe_string = _servnode.text
                _cpelist.append(_cpe_string)
        _service['cpelist'] = _cpelist
        return _service

    @classmethod
    def __parse_extraports(cls, extraports_data):
        """
            Private method parsing the data from extra scanned ports.
            X extraports were in state "closed" server returned "conn-refused"
            tag: <extraports>

            :param extraports_data: XML data for extraports
            :type extraports_data: xml.ElementTree.Element or a string

            :return: python dict with following keys: state, count, reason
        """
        rdict = {'state': '', 'count': '', 'reasons': []}
        xelement = cls.__format_element(extraports_data)
        extraports_dict = cls.__format_attributes(xelement)

        if 'state' in extraports_dict:
            rdict['state'] = extraports_dict
        if 'count' in extraports_dict:
            rdict['count'] = extraports_dict
        for xelt in xelement:
            if xelt.tag == 'extrareasons':
                extrareasons_dict = cls.__format_attributes(xelt)
                rdict['reasons'].append(extrareasons_dict)
        return rdict

    @classmethod
    def __parse_script(cls, script_data):
        """
            Private method parsing the data from NSE scripts output

            :param script_data: portion of XML describing the results of the
            script data
            :type script_data: xml.ElementTree.Element or a string

            :return: python dict holding scripts output
        """
        _script_dict = cls.__format_attributes(script_data)

        _elt_dict = {}
        for script_elem in script_data:
            if script_elem.tag == 'elem':
                _elt_dict.update({script_elem.get('key'): script_elem.text})
            elif script_elem.tag == 'table':
                tdict = {}
                for telem in script_elem:
                    tdict[telem.get('key')] = telem.text
                _elt_dict[script_elem.get('key')] = tdict
        _script_dict['elements'] = _elt_dict
        return _script_dict

    @classmethod
    def __parse_host_scripts(cls, scripts_data):
        """
            Private method parsing the data from scripts affecting
            the target host.
            Contents of <hostscript> is returned as a list of dict.

            :param scripts_data: portion of XML describing the results of the
            scripts data
            :type scripts_data: xml.ElementTree.Element or a string

            :return: python list holding scripts output in a dict
        """
        _host_scripts = []
        for xscript in scripts_data:
            if xscript.tag == 'script':
                _script_dict = cls.__parse_script(xscript)
            _host_scripts.append(_script_dict)
        return _host_scripts

    @classmethod
    def __parse_os_fingerprint(cls, os_data):
        """
            Private method parsing the data from an OS fingerprint (-O).
            Contents of <os> is returned as a dict.

            :param os_data: portion of XML describing the results of the
            os fingerprinting attempt
            :type os_data: xml.ElementTree.Element or a string

            :return: python dict representing the XML os tag
        """
        rdict = {}
        xelement = cls.__format_element(os_data)

        os_class_probability = []
        os_match_probability = []
        os_ports_used = []
        os_fingerprints = []
        for xos in xelement:
            # for nmap xml version < 1.04, osclass is not
            # embedded in osmatch
            if xos.tag == 'osclass':
                os_class_proba = cls.__parse_osclass(xos)
                os_class_probability.append(os_class_proba)
            elif xos.tag == 'osmatch':
                os_match_proba = cls.__parse_osmatch(xos)
                os_match_probability.append(os_match_proba)
            elif xos.tag == 'portused':
                os_portused = cls.__format_attributes(xos)
                os_ports_used.append(os_portused)
            elif xos.tag == 'osfingerprint':
                os_fp_dict = cls.__format_attributes(xos)
                os_fingerprints.append(os_fp_dict)

        rdict['osmatches'] = os_match_probability
        rdict['osclasses'] = os_class_probability
        rdict['ports_used'] = os_ports_used
        rdict['osfingerprints'] = os_fingerprints

        return rdict

    @classmethod
    def __parse_osmatch(cls, osmatch_data):
        """
            This methods parses osmatch data and returns a dict. Depending
            on the nmap xml version, osmatch could contain an osclass
            dict.

            :param osmatch_data: <osmatch> XML tag from a nmap scan
            :type osmatch_data: xml.ElementTree.Element or a string

            :return: python dict representing the XML osmatch tag
        """
        rdict = {}
        xelement = cls.__format_element(osmatch_data)
        rdict['osmatch'] = cls.__format_attributes(xelement)
        rdict['osclasses'] = []
        for xmltag in xelement:
            if xmltag.tag == 'osclass':
                _osclass_dict = cls.__parse_osclass(xmltag)
                rdict['osclasses'].append(_osclass_dict)
            else:
                exmsg = "Unexcepted node in <osmatch>: {0}".format(xmltag.tag)
                raise NmapParserException(exmsg)
        return rdict

    @classmethod
    def __parse_osclass(cls, osclass_data):
        """
            This methods parses osclass data and returns a dict. Depending
            on the nmap xml version, osclass could contain a cpe
            dict.

            :param osclass_data: <osclass> XML tag from a nmap scan
            :type osclass_data: xml.ElementTree.Element or a string

            :return: python dict representing the XML osclass tag
        """
        rdict = {}
        xelement = cls.__format_element(osclass_data)
        rdict['osclass'] = cls.__format_attributes(xelement)
        rdict['cpe'] = []
        for xmltag in xelement:
            if xmltag.tag == 'cpe':
                _cpe_string = xmltag.text
                rdict['cpe'].append(_cpe_string)
            else:
                exmsg = "Unexcepted node in <osclass>: {0}".format(xmltag.tag)
                raise NmapParserException(exmsg)
        return rdict

    @classmethod
    def __parse_runstats(cls, scanrunstats_data):
        """
            Private method parsing a portion of a nmap scan result.
            Receives a <runstats> XML tag.

            :param scanrunstats_data: <runstats> XML tag from a nmap scan
            :type scanrunstats_data: xml.ElementTree.Element or a string

            :return: python dict representing the XML runstats tag
        """

        xelement = cls.__format_element(scanrunstats_data)

        rdict = {}
        for xmltag in xelement:
            if xmltag.tag in ['finished', 'hosts']:
                rdict[xmltag.tag] = cls.__format_attributes(xmltag)
            else:
                exmsg = "Unexcepted node in <runstats>: {0}".format(xmltag.tag)
                raise NmapParserException(exmsg)

        return rdict

    @staticmethod
    def __format_element(elt_data):
        """
            Private method which ensures that a XML portion to be parsed is
            of type xml.etree.ElementTree.Element.
            If elt_data is a string, then it is converted to an
            XML Element type.

            :param elt_data: XML Element to be parsed or string
            to be converted to a XML Element

            :return: Element
        """
        if isinstance(elt_data, str):
            try:
                xelement = ET.fromstring(elt_data)
            except:
                raise NmapParserException("Error while trying "
                                          "to instanciate XML Element from "
                                          "string {0}".format(elt_data))
        elif ET.iselement(elt_data):
            xelement = elt_data
        else:
            raise NmapParserException("Error while trying to parse supplied "
                                      "data: unsupported format")
        return xelement

    @staticmethod
    def __format_attributes(elt_data):
        """
            Private method which converts a single XML tag to a python dict.
            It also checks that the elt_data given as argument is of type
            xml.etree.ElementTree.Element

            :param elt_data: XML Element to be parsed or string
            to be converted to a XML Element

            :return: Element
        """

        rval = {}
        if not ET.iselement(elt_data):
            raise NmapParserException("Error while trying to parse supplied "
                                      "data attributes: format is not XML or "
                                      "XML tag is empty")
        try:
            for dkey in elt_data.keys():
                rval[dkey] = elt_data.get(dkey)
                if rval[dkey] is None:
                    raise NmapParserException("Error while trying to build-up "
                                              "element attributes: empty "
                                              "attribute {0}".format(dkey))
        except:
            raise
        return rval


class NmapParserException(Exception):
    def __init__(self, msg):
        self.msg = msg

    def __str__(self):
        return self.msg


================================================
FILE: libnmap/plugins/__init__.py
================================================


================================================
FILE: libnmap/plugins/backend_host.py
================================================
#!/usr/bin/env python
from sqlalchemy import create_engine
from sqlalchemy.schema import Column
from sqlalchemy.types import Integer, DateTime, LargeBinary, Text, String, Boolean
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker

from libnmap.plugins.backendplugin import NmapBackendPlugin
from libnmap.reportjson import ReportEncoder, ReportDecoder

import json
from datetime import datetime

Base = declarative_base()

class NmapSqlPlugin(NmapBackendPlugin):
    """
        This class handle the persistence of NmapRepport object in SQL backend
        Implementation is made using sqlalchemy(0.8.1)
        usage :

        #get a nmapReport object
        from libnmap.parser import NmapParser
        from libnmap.reportjson import ReportDecoder, ReportEncoder
        import json
        nmap_report_obj = NmapParser.parse_fromfile(
               '/home/vagrant/python-nmap-lib/libnmap/test/files/1_hosts.xml')

         #get a backend with in memory sqlite
         from libnmap.plugins.backendpluginFactory import BackendPluginFactory
         mybackend_mem = BackendPluginFactory.create(plugin_name='sql',
                                                     url='sqlite://',
                                                     echo=True)

         mybackend_mysql = BackendPluginFactory.create(plugin_name='sql',
                            url='mysql+mysqldb://scott:tiger@localhost/foo',
                            echo=True)
         mybackend = BackendPluginFactory.create(plugin_name='sql',
                                        url='sqlite:////tmp/reportdb.sql',
                                        echo=True)
         #lets save!!
         nmap_report_obj.save(mybackend)
         mybackend.getall()
         mybackend.get(1)
    """
    class Reports(Base):
        """
            Embeded class for ORM map NmapReport to a
            simple three column table
        """
        __tablename__ = 'result_ip'

        id = Column('id', Integer, primary_key=True)
        taskid = Column('taskid', Integer)
        inserted = Column('inserted', DateTime(), default='now')
        domain = Column('domain', String(256))
        address = Column('address', String(32))
        is_up = Column('is_up', Boolean)
        os = Column('os', String(256))
    
        def __init__(self, obj_NmapReport):
            self.inserted = datetime.fromtimestamp(int(obj_NmapReport.endtime))
            self.taskid = obj_NmapReport.taskid
            self.is_up = obj_NmapReport.is_up()

            if len(obj_NmapReport.hostnames) > 0:
                self.domain = obj_NmapReport.hostnames[0]
            else:
                self.domain = None

            self.address = obj_NmapReport.address
            
            if len(obj_NmapReport.os.osmatch()) > 0:
                self.os = obj_NmapReport.os.osmatch()[0]
            else:
                self.os = None

        def decode(self):
            json_decoded = self.report_json.decode('utf-8')
            nmap_report_obj = json.loads(json_decoded,
                                         cls=ReportDecoder)
            return nmap_report_obj

    def __init__(self, **kwargs):
        """
            constructor receive a **kwargs as the **kwargs in the sqlalchemy
            create_engine() method (see sqlalchemy docs)
            You must add to this **kwargs an 'url' key with the url to your
            database
            This constructor will :
            - create all the necessary obj to discuss with the DB
            - create all the mapping(ORM)

            todo : suport the : sqlalchemy.engine_from_config

            :param **kwargs:
            :raises: ValueError if no url is given,
                    all exception sqlalchemy can throw
            ie sqlite in memory url='sqlite://' echo=True
            ie sqlite file on hd url='sqlite:////tmp/reportdb.sql' echo=True
            ie mysql url='mysql+mysqldb://scott:tiger@localhost/foo'
        """
        NmapBackendPlugin.__init__(self)
        self.engine = None
        self.url = None
        self.Session = sessionmaker()

        if 'url' not in kwargs:
            raise ValueError
        self.url = kwargs['url']
        del kwargs['url']
        try:
            self.engine = create_engine(self.url, **kwargs)
            Base.metadata.create_all(bind=self.engine, checkfirst=True)
            self.Session.configure(bind=self.engine)
        except:
            raise

    def insert(self, nmap_report):
        """
           insert NmapReport in the backend

           :param NmapReport:

           :returns: the ident of the object in the backend for future usage \
           or None
        """
        sess = self.Session()
        report = NmapSqlPlugin.Reports(nmap_report)
        sess.add(report)
        sess.commit()
        reportid = report.id
        sess.close()
        return reportid if reportid else None

    def get(self, report_id=None):
        """
            retreive a NmapReport from the backend

            :param id: str

            :returns: NmapReport
        """
        if report_id is None:
            raise ValueError
        sess = self.Session()
        our_report = (
            sess.query(NmapSqlPlugin.Reports).filter_by(id=report_id).first())
        sess.close()
        return our_report.decode() if our_report else None

    def getall(self):
        """
            :param filter: Nice to have implement a filter capability

            :returns: collection of tuple (id,NmapReport)
        """
        sess = self.Session()
        nmapreportList = []
        for report in (
                sess.query(NmapSqlPlugin.Reports).
                order_by(NmapSqlPlugin.Reports.inserted)):
            nmapreportList.append((report.id, report.decode()))
        sess.close()
        return nmapreportList

    def delete(self, report_id=None):
        """
            Remove a report from the backend

            :param id: str

            :returns: The number of rows deleted
        """
        if report_id is None:
            raise ValueError
        nb_line = 0
        sess = self.Session()
        nb_line = sess.query(NmapSqlPlugin.Reports).\
            filter_by(id=report_id).delete()
        sess.commit()
        sess.close()
        return nb_line


================================================
FILE: libnmap/plugins/backend_service.py
================================================
#!/usr/bin/env python
from sqlalchemy import create_engine
from sqlalchemy.schema import Column
from sqlalchemy.types import Integer, DateTime, LargeBinary, Text, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker

from libnmap.plugins.backendplugin import NmapBackendPlugin
from libnmap.reportjson import ReportEncoder, ReportDecoder

import json
from datetime import datetime
import binascii

Base = declarative_base()

class NmapSqlPlugin(NmapBackendPlugin):
    """
        This class handle the persistence of NmapRepport object in SQL backend
        Implementation is made using sqlalchemy(0.8.1)
        usage :

        #get a nmapReport object
        from libnmap.parser import NmapParser
        from libnmap.reportjson import ReportDecoder, ReportEncoder
        import json
        nmap_report_obj = NmapParser.parse_fromfile(
               '/home/vagrant/python-nmap-lib/libnmap/test/files/1_hosts.xml')

         #get a backend with in memory sqlite
         from libnmap.plugins.backendpluginFactory import BackendPluginFactory
         mybackend_mem = BackendPluginFactory.create(plugin_name='sql',
                                                     url='sqlite://',
                                                     echo=True)

         mybackend_mysql = BackendPluginFactory.create(plugin_name='sql',
                            url='mysql+mysqldb://scott:tiger@localhost/foo',
                            echo=True)
         mybackend = BackendPluginFactory.create(plugin_name='sql',
                                        url='sqlite:////tmp/reportdb.sql',
                                        echo=True)
         #lets save!!
         nmap_report_obj.save(mybackend)
         mybackend.getall()
         mybackend.get(1)
    """
    class Reports(Base):
        """
            Embeded class for ORM map NmapReport to a
            simple three column table
        """
        __tablename__ = 'result_ports'

        id = Column('id', Integer, primary_key=True)
        taskid = Column('taskid', Integer)
        inserted = Column('inserted', DateTime(), default='now')
        address = Column('address', String(256))
        port = Column('port', Integer)
        service = Column('service', String(256))
        state = Column('state', String(12))
        protocol = Column('protocol', String(12))
        product = Column('product', String(64))
        product_version = Column('product_version', String(64))
        product_extrainfo = Column('product_extrainfo', String(128))
        # banner = Column('banner', String(256))
        scripts_results = Column('scripts_results', Text)

        def __init__(self, obj_NmapReport):
            self.inserted = datetime.fromtimestamp(int(obj_NmapReport.endtime))
            self.taskid = obj_NmapReport.taskid
            self.address = obj_NmapReport.address
            self.port = obj_NmapReport.port
            self.service = obj_NmapReport.service
            self.state = obj_NmapReport.state
            self.protocol = str(obj_NmapReport.protocol)
            self.product = str(obj_NmapReport.product)
            self.product_version = str(obj_NmapReport.product_version)
            self.product_extrainfo = str(obj_NmapReport.product_extrainfo)
            # self.banner = str(obj_NmapReport.banner)
            # self.scripts_results = binascii.b2a_hex(str(obj_NmapReport.scripts_results))

            if len(obj_NmapReport.scripts_results) > 0:                
                self.scripts_results = obj_NmapReport.scripts_results[0]['output']
            else:
                self.scripts_results = None


        def decode(self):
            json_decoded = self.report_json.decode('utf-8')
            nmap_report_obj = json.loads(json_decoded,
                                         cls=ReportDecoder)
            return nmap_report_obj

    def __init__(self, **kwargs):
        """
            constructor receive a **kwargs as the **kwargs in the sqlalchemy
            create_engine() method (see sqlalchemy docs)
            You must add to this **kwargs an 'url' key with the url to your
            database
            This constructor will :
            - create all the necessary obj to discuss with the DB
            - create all the mapping(ORM)

            todo : suport the : sqlalchemy.engine_from_config

            :param **kwargs:
            :raises: ValueError if no url is given,
                    all exception sqlalchemy can throw
            ie sqlite in memory url='sqlite://' echo=True
            ie sqlite file on hd url='sqlite:////tmp/reportdb.sql' echo=True
            ie mysql url='mysql+mysqldb://scott:tiger@localhost/foo'
        """
        NmapBackendPlugin.__init__(self)
        self.engine = None
        self.url = None
        self.Session = sessionmaker()

        if 'url' not in kwargs:
            raise ValueError
        self.url = kwargs['url']
        del kwargs['url']
        try:
            self.engine = create_engine(self.url, **kwargs)
            Base.metadata.create_all(bind=self.engine, checkfirst=True)
            self.Session.configure(bind=self.engine)
        except:
            raise

    def insert(self, nmap_report):
        """
           insert NmapReport in the backend

           :param NmapReport:

           :returns: the ident of the object in the backend for future usage \
           or None
        """
        sess = self.Session()
        report = NmapSqlPlugin.Reports(nmap_report)
        sess.add(report)
        sess.commit()
        reportid = report.id
        sess.close()
        return reportid if reportid else None

    def get(self, report_id=None):
        """
            retreive a NmapReport from the backend

            :param id: str

            :returns: NmapReport
        """
        if report_id is None:
            raise ValueError
        sess = self.Session()
        our_report = (
            sess.query(NmapSqlPlugin.Reports).filter_by(id=report_id).first())
        sess.close()
        return our_report.decode() if our_report else None

    def getall(self):
        """
            :param filter: Nice to have implement a filter capability

            :returns: collection of tuple (id,NmapReport)
        """
        sess = self.Session()
        nmapreportList = []
        for report in (
                sess.query(NmapSqlPlugin.Reports).
                order_by(NmapSqlPlugin.Reports.inserted)):
            nmapreportList.append((report.id, report.decode()))
        sess.close()
        return nmapreportList

    def delete(self, report_id=None):
        """
            Remove a report from the backend

            :param id: str

            :returns: The number of rows deleted
        """
        if report_id is None:
            raise ValueError
        nb_line = 0
        sess = self.Session()
        nb_line = sess.query(NmapSqlPlugin.Reports).\
            filter_by(id=report_id).delete()
        sess.commit()
        sess.close()
        return nb_line


================================================
FILE: libnmap/plugins/backendplugin.py
================================================
#!/usr/bin/env python


class NmapBackendPlugin(object):
    """
        Abstract class showing to the minimal implementation for a plugin
        All subclass MUST at least implement the following methods
    """
    def __init__(self):
        self.dbname = 'nmapdb'
        self.store = 'reports'

    def insert(self, NmapReport):
        """
            insert NmapReport in the backend
            :param NmapReport:
            :return: str the ident of the object in the backend for
            future usage
            or None
        """
        raise NotImplementedError

    def delete(self, id):
        """
            delete NmapReport if the backend
            :param id: str
        """
        raise NotImplementedError

    def get(self, id):
        """
            retreive a NmapReport from the backend
            :param id: str
            :return: NmapReport
        """
        raise NotImplementedError

    def getall(self, filter):
        """
            :return: collection of tuple (id,NmapReport)
            :param filter: Nice to have implement a filter capability
        """
        raise NotImplementedError


================================================
FILE: libnmap/plugins/backendpluginFactory.py
================================================
#!/usr/bin/env python
import sys
import inspect


class BackendPluginFactory(object):
    """
        This is a backend plugin factory a backend instance MUST be
        created via the static method create()
        ie : mybackend = BackendPluginFactory.create()
    """
    @classmethod
    def create(cls, plugin_name="mongodb", **kwargs):
        """Import the needed lib and return an object NmapBackendPlugin
           representing the backend of your desire.
           NmapBackendPlugin is an abstract class, to know what argument
           need to be given, review the code of the subclass you need
           :param plugin_name: str : name of the py file without .py
           :return: NmapBackend (abstract class on top of all plugin)
        """
        backendplugin = None
        plugin_path = "libnmap.plugins.{0}".format(plugin_name)
        __import__(plugin_path)
        pluginobj = sys.modules[plugin_path]
        pluginclasses = inspect.getmembers(pluginobj, inspect.isclass)
        for classname, classobj in pluginclasses:
            if inspect.getmodule(classobj).__name__.find(plugin_path) == 0:
                try:
                    backendplugin = classobj(**kwargs)
                except Exception as error:
                    raise Exception("Cannot create Backend: {0}".format(error))
        return backendplugin


================================================
FILE: libnmap/plugins/es.py
================================================
# -*- coding: utf-8 -*-

import json
from libnmap.reportjson import ReportEncoder
from libnmap.plugins.backendplugin import NmapBackendPlugin
from elasticsearch import Elasticsearch
from datetime import datetime


class NmapElasticsearchPlugin(NmapBackendPlugin):
    """
        This class enables the user to store and manipulate nmap reports \
        in a elastic search db.
    """
    def __init__(self, index=None):
        if index is None:
            self.index = "nmap.{0}".format(datetime.now().strftime('%Y-%m-%d'))
        else:
            self.index = index
        self._esapi = Elasticsearch()

    def insert(self, report, doc_type=None):
        """
            insert NmapReport in the backend
            :param NmapReport:
            :return: str the ident of the object in the backend for
            future usage
            or None
        """
        if doc_type is None:
            doc_type = 'NmapReport'
        j = json.dumps(report, cls=ReportEncoder)
        res = self._esapi.index(
            index=self.index,
            doc_type=doc_type,
            body=json.loads(j))
        rc = res['_id']
        return rc

    def delete(self, id):
        """
            delete NmapReport if the backend
            :param id: str
        """
        raise NotImplementedError

    def get(self, id):
        """
            retreive a NmapReport from the backend
            :param id: str
            :return: NmapReport
        """
        res = self._esapi.get(index=self.index,
                              doc_type="NmapReport",
                              id=id)['_source']
        return res

    def getall(self, filter=None):
        """
            :return: collection of tuple (id,NmapReport)
            :param filter: Nice to have implement a filter capability
        """
        rsearch = self._esapi.search(index=self.index,
                                     body={"query": {"match_all": {}}})
        print("--------------------")
        print(type(rsearch))
        print(rsearch)
        print("------------")


================================================
FILE: libnmap/plugins/mongodb.py
================================================
#!/usr/bin/env python
import json
from pymongo import MongoClient
from bson.objectid import ObjectId

from libnmap.reportjson import ReportEncoder
from libnmap.parser import NmapParser
from libnmap.plugins.backendplugin import NmapBackendPlugin


class NmapMongodbPlugin(NmapBackendPlugin):
    """
        This class handle the persistence of NmapRepport object in mongodb
        Implementation is made using pymongo
        Object of this class must be create via the
        BackendPluginFactory.create(**url) where url is a named dict like
        {'plugin_name': "mongodb"} this dict may reeive all the param
        MongoClient() support
    """
    def __init__(self, dbname=None, store=None, **kwargs):
        NmapBackendPlugin.__init__(self)
        if dbname is not None:
            self.dbname = dbname
        if store is not None:
            self.store = store
        self.dbclient = MongoClient(**kwargs)
        self.collection = self.dbclient[self.dbname][self.store]

    def insert(self, report):
        """
            create a json object from an NmapReport instance
            :param NmapReport: obj to insert
            :return: str id
        """
        j = json.dumps(report, cls=ReportEncoder)
        try:
            oid = self.collection.insert(json.loads(j))
        except:
            raise Exception("Failed to insert nmap object in MongoDB")
        return str(oid)

    def get(self, str_report_id=None):
        """ select a NmapReport by Id
            :param str: id
            :return: NmapReport object
        """
        rid = str_report_id
        nmapreport = None
        if str_report_id is not None and isinstance(str_report_id, str):
            rid = ObjectId(str_report_id)

        if isinstance(rid, ObjectId):
            # get a specific report by mongo's id
            resultset = self.collection.find({'_id': rid})
            if resultset.count() == 1:
                # search by id means only one in the iterator
                record = resultset[0]
                # remove mongo's id to recreate the NmapReport Obj
                del record['_id']
                nmapreport = NmapParser.parse_fromdict(record)
        return nmapreport

    def getall(self, dict_filter=None):
        """return a list of tuple (id,NmapReport) saved in the backend
           TODO : add a filter capability
        """
        nmapreportlist = []
        resultset = self.collection.find()
        for report in resultset:
            oid = report['_id']
            del report['_id']
            nmapreport = NmapParser.parse_fromdict(report)
            nmapreportlist.append((oid, nmapreport))
        return nmapreportlist

    def delete(self, report_id=None):
        """
            delete an obj from the backend
            :param str: id
            :return: dict document with result or None
        """
        if report_id is not None and isinstance(report_id, str):
            return self.collection.remove({'_id': ObjectId(report_id)})
        else:
            return self.collection.remove({'_id': report_id})


================================================
FILE: libnmap/plugins/s3.py
================================================
#!/usr/bin/env python
"""
:mod:`libnmap.plugin.s3` -- S3 Backend Plugin
=============================================

.. module:: libnmap.plugin.s3

:platform: Linux
:synopsis: a plugin is representation of a S3 backend using boto

.. moduleauthor:: Ronald Bister
.. moduleauthor:: Mike Boutillier
"""
import json
from bson.objectid import ObjectId
from boto.s3.connection import S3Connection, OrdinaryCallingFormat
from boto.s3.key import Key
from boto.s3.bucketlistresultset import bucket_lister
from boto.exception import S3ResponseError
from libnmap.reportjson import ReportEncoder
from libnmap.parser import NmapParser
from libnmap.plugins.backendplugin import NmapBackendPlugin


class NmapS3Plugin(NmapBackendPlugin):
    """
        This plugin save the reports on S3 and compatible.
    """
    def __init__(self, **kwargs):
        """
            - create the conn object
            - create the bucket (if it doesn't exist)
                - if not given, awsaccessKey_nmapreport
            - may raise exception (ie in case of conflict bucket name)
            - sample :
            To connect to walrus:
            from libnmap.plugins.backendpluginFactory import
                            BackendPluginFactory
            walrusBackend =
              BackendPluginFactory.create(
                    plugin_name='s3',
                    host="walrus.ecc.eucalyptus.com",
                    path="/services/Walrus",port=8773,
                    is_secure=False,
                    aws_access_key_id='UU72FLVJCAYRATLXI70YH',
                    aws_secret_access_key=
                               'wFg7gP5YFHjVlxakw1g1uCC8UR2xVW5ax9ErZCut')
           To connect to S3:
           mybackend_S3 =
             BackendPluginFactory.create(
                plugin_name='s3',
                is_secure=True,
                aws_access_key_id='MYACCESSKEY',
                aws_secret_access_key='MYSECRET')
        """
        NmapBackendPlugin.__init__(self)
        try:
            calling_format = OrdinaryCallingFormat()
            if 'bucket' not in kwargs:
                self.bucket_name = ''.join(
                    [kwargs['aws_access_key_id'].lower(),
                     "_nmapreport"])
            else:
                self.bucket_name = kwargs['bucket']
                del kwargs['bucket']
            kwargs['calling_format'] = calling_format
            self.conn = S3Connection(**kwargs)
            self.bucket = self.conn.lookup(self.bucket_name)
            if self.bucket is None:
                self.bucket = self.conn.create_bucket(self.bucket_name)
        except:
            raise

    def insert(self, report):
        """
            create a json string from an NmapReport instance
            and push it to S3 bucket.

            :param NmapReport: obj to insert
            :rtype: string
            :return: str id
            :todo: Add tagging option
        """
        try:
            oid = ObjectId()
            mykey = Key(self.bucket)
            mykey.key = str(oid)
            strjsonnmapreport = json.dumps(report, cls=ReportEncoder)
            mykey.set_contents_from_string(strjsonnmapreport)
        except:
            raise Exception("Failed to add nmap object in s3 bucket")
        return str(oid)

    def get(self, str_report_id=None):
        """
            select a NmapReport by Id.

            :param str: id
            :rtype: NmapReport
            :return: NmapReport object
        """
        nmapreport = None
        if str_report_id is not None and isinstance(str_report_id, str):
            try:
                mykey = Key(self.bucket)
                mykey.key = str_report_id
                nmapreportjson = json.loads(mykey.get_contents_as_string())
                nmapreport = NmapParser.parse_fromdict(nmapreportjson)
            except S3ResponseError:
                pass
        return nmapreport

    def getall(self, dict_filter=None):
        """
            :rtype: List of tuple
            :return: list of key/report
            :todo: add a filter capability
        """
        nmapreportlist = []
        for key in bucket_lister(self.bucket):
            if isinstance(key, Key):
                nmapreportjson = json.loads(key.get_contents_as_string())
                nmapreport = NmapParser.parse_fromdict(nmapreportjson)
                nmapreportlist.append((key.key, nmapreport))
        return nmapreportlist

    def delete(self, report_id=None):
        """
            delete an obj from the backend

            :param str: id
            :return: dict document with result or None
        """
        rcode = None
        if report_id is not None and isinstance(report_id, str):
            rcode = self.bucket.delete_key(report_id)
        return rcode


================================================
FILE: libnmap/plugins/sql.py
================================================
#!/usr/bin/env python
from sqlalchemy import create_engine
from sqlalchemy.schema import Column
from sqlalchemy.types import Integer, DateTime, LargeBinary, Text, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker

from libnmap.plugins.backendplugin import NmapBackendPlugin
from libnmap.reportjson import ReportEncoder, ReportDecoder

import json
from datetime import datetime

Base = declarative_base()

class NmapSqlPlugin(NmapBackendPlugin):
    """
        This class handle the persistence of NmapRepport object in SQL backend
        Implementation is made using sqlalchemy(0.8.1)
        usage :

        #get a nmapReport object
        from libnmap.parser import NmapParser
        from libnmap.reportjson import ReportDecoder, ReportEncoder
        import json
        nmap_report_obj = NmapParser.parse_fromfile(
               '/home/vagrant/python-nmap-lib/libnmap/test/files/1_hosts.xml')

         #get a backend with in memory sqlite
         from libnmap.plugins.backendpluginFactory import BackendPluginFactory
         mybackend_mem = BackendPluginFactory.create(plugin_name='sql',
                                                     url='sqlite://',
                                                     echo=True)

         mybackend_mysql = BackendPluginFactory.create(plugin_name='sql',
                            url='mysql+mysqldb://scott:tiger@localhost/foo',
                            echo=True)
         mybackend = BackendPluginFactory.create(plugin_name='sql',
                                        url='sqlite:////tmp/reportdb.sql',
                                        echo=True)
         #lets save!!
         nmap_report_obj.save(mybackend)
         mybackend.getall()
         mybackend.get(1)
    """
    class Reports(Base):
        """
            Embeded class for ORM map NmapReport to a
            simple three column table
        """
        __tablename__ = 'reports'

        id = Column('report_id', Integer, primary_key=True)
        inserted = Column('inserted', DateTime(), default='now')
        host = Column('host', String(256))
        command_line = Column('command_line', String(256))
        report_json = Column('report_json', LargeBinary())

        def __init__(self, obj_NmapReport):
            # self.inserted = datetime.fromtimestamp(obj_NmapReport.endtime)
            self.command_line = str(obj_NmapReport.status)

            self.host = obj_NmapReport.address
            self.command_line = 

            dumped_json = json.dumps(obj_NmapReport,
                                     cls=ReportEncoder)
            self.report_json = bytes(dumped_json.encode('UTF-8'))

        def decode(self):
            json_decoded = self.report_json.decode('utf-8')
            nmap_report_obj = json.loads(json_decoded,
                                         cls=ReportDecoder)
            return nmap_report_obj

    def __init__(self, **kwargs):
        """
            constructor receive a **kwargs as the **kwargs in the sqlalchemy
            create_engine() method (see sqlalchemy docs)
            You must add to this **kwargs an 'url' key with the url to your
            database
            This constructor will :
            - create all the necessary obj to discuss with the DB
            - create all the mapping(ORM)

            todo : suport the : sqlalchemy.engine_from_config

            :param **kwargs:
            :raises: ValueError if no url is given,
                    all exception sqlalchemy can throw
            ie sqlite in memory url='sqlite://' echo=True
            ie sqlite file on hd url='sqlite:////tmp/reportdb.sql' echo=True
            ie mysql url='mysql+mysqldb://scott:tiger@localhost/foo'
        """
        NmapBackendPlugin.__init__(self)
        self.engine = None
        self.url = None
        self.Session = sessionmaker()

        if 'url' not in kwargs:
            raise ValueError
        self.url = kwargs['url']
        del kwargs['url']
        try:
            self.engine = create_engine(self.url, **kwargs)
            Base.metadata.create_all(bind=self.engine, checkfirst=True)
            self.Session.configure(bind=self.engine)
        except:
            raise

    def insert(self, nmap_report):
        """
           insert NmapReport in the backend

           :param NmapReport:

           :returns: the ident of the object in the backend for future usage \
           or None
        """
        sess = self.Session()
        report = NmapSqlPlugin.Reports(nmap_report)
        sess.add(report)
        sess.commit()
        reportid = report.id
        sess.close()
        return reportid if reportid else None

    def get(self, report_id=None):
        """
            retreive a NmapReport from the backend

            :param id: str

            :returns: NmapReport
        """
        if report_id is None:
            raise ValueError
        sess = self.Session()
        our_report = (
            sess.query(NmapSqlPlugin.Reports).filter_by(id=report_id).first())
        sess.close()
        return our_report.decode() if our_report else None

    def getall(self):
        """
            :param filter: Nice to have implement a filter capability

            :returns: collection of tuple (id,NmapReport)
        """
        sess = self.Session()
        nmapreportList = []
        for report in (
                sess.query(NmapSqlPlugin.Reports).
                order_by(NmapSqlPlugin.Reports.inserted)):
            nmapreportList.append((report.id, report.decode()))
        sess.close()
        return nmapreportList

    def delete(self, report_id=None):
        """
            Remove a report from the backend

            :param id: str

            :returns: The number of rows deleted
        """
        if report_id is None:
            raise ValueError
        nb_line = 0
        sess = self.Session()
        nb_line = sess.query(NmapSqlPlugin.Reports).\
            filter_by(id=report_id).delete()
        sess.commit()
        sess.close()
        return nb_line


================================================
FILE: libnmap/process.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import pwd
import shlex
import subprocess
import multiprocessing
from threading import Thread
from xml.dom import pulldom
import warnings

try:
    from Queue import Empty, Full
except ImportError:
    from queue import Empty, Full

__all__ = [
    'NmapProcess'
]


class NmapTask(object):
    """
    NmapTask is a internal class used by process. Each time nmap
    starts a new task during the scan, a new class will be instanciated.
    Classes examples are: "Ping Scan", "NSE script", "DNS Resolve",..
    To each class an estimated time to complete is assigned and updated
    at least every second within the NmapProcess.
    A property NmapProcess.current_task points to the running task at
    time T and a dictionnary NmapProcess.tasks with "task name" as key
    is built during scan execution
    """
    def __init__(self, name, starttime=0, extrainfo=''):
        self.name = name
        self.etc = 0
        self.progress = 0
        self.percent = 0
        self.remaining = 0
        self.status = 'started'
        self.starttime = starttime
        self.endtime = 0
        self.extrainfo = extrainfo
        self.updated = 0


class NmapProcess(Thread):
    """
    NmapProcess is a class which wraps around the nmap executable.

    Consequently, in order to run an NmapProcess, nmap should be installed
    on the host running the script. By default NmapProcess will produce
    the output of the nmap scan in the nmap XML format. This could be then
    parsed out via the NmapParser class from libnmap.parser module.
    """
    def __init__(self, targets="127.0.0.1",
                 options="-sT", event_callback=None, safe_mode=True, fqp=None):
        """
        Constructor of NmapProcess class.

        :param targets: hosts to be scanned. Could be a string of hosts \
        separated with a coma or a python list of hosts/ip.
        :type targets: string or list

        :param options: list of nmap options to be applied to scan. \
        These options are all documented in nmap's man pages.

        :param event_callback: callable function which will be ran \
        each time nmap process outputs data. This function will receive \
        two parameters:

            1. the nmap process object
            2. the data produced by nmap process. See readme for examples.

        :param safe_mode: parameter to protect unsafe options like -oN, -oG, \
        -iL, -oA,...

        :param fqp: full qualified path, if None, nmap will be searched \
        in the PATH

        :return: NmapProcess object

        """
        Thread.__init__(self)
        unsafe_opts = set(['-oG', '-oN', '-iL', '-oA', '-oS', '-oX',
                           '--iflist', '--resume', '--stylesheet',
                           '--datadir'])
        if fqp:
            if os.path.isfile(fqp) and os.access(fqp, os.X_OK):
                self.__nmap_binary = fqp
            else:
                raise EnvironmentError(1, "wrong path or not executable", fqp)
        else:
            nmap_binary_name = "nmap"
            self.__nmap_binary = self._whereis(nmap_binary_name)
        # self.__nmap_fixed_options = "-oX - -vvv --stats-every 1s"
        self.__nmap_fixed_options = "-oX -"

        if self.__nmap_binary is None:
            raise EnvironmentError(1, "nmap is not installed or could "
                                      "not be found in system path")

        if isinstance(targets, str):
            self.__nmap_targets = targets.replace(" ", "").split(',')
        elif isinstance(targets, list):
            self.__nmap_targets = targets
        else:
            raise Exception("Supplied target list should be either a "
                            "string or a list")

        self._nmap_options = set(options.split())
        if safe_mode and not self._nmap_options.isdisjoint(unsafe_opts):
            raise Exception("unsafe options activated while safe_mode "
                            "is set True")
        self.__nmap_dynamic_options = options
        self.__sudo_run = ''
        self.__nmap_command_line = self.get_command_line()

        if event_callback and callable(event_callback):
            self.__nmap_event_callback = event_callback
        else:
            self.__nmap_event_callback = None
        (self.DONE, self.READY, self.RUNNING,
         self.CANCELLED, self.FAILED) = range(5)
        self._run_init()

    def _run_init(self):
        self.__process_killed = multiprocessing.Event()
        self.__nmap_command_line = self.get_command_line()
        # API usable in callback function
        self.__nmap_proc = None
        self.__qout = None
        self.__nmap_rc = 0
        self.__state = self.RUNNING
        self.__starttime = 0
        self.__endtime = 0
        self.__version = ''
        self.__elapsed = ''
        self.__summary = ''
        self.__stdout = ''
        self.__stderr = ''
        self.__current_task = ''
        self.__nmap_tasks = {}

    def _whereis(self, program):
        """
        Protected method enabling the object to find the full path of a binary
        from its PATH environment variable.

        :param program: name of a binary for which the full path needs to
        be discovered.

        :return: the full path to the binary.

        :todo: add a default path list in case PATH is empty.
        """
        for path in os.environ.get('PATH', '').split(':'):
            if (os.path.exists(os.path.join(path, program)) and not
               os.path.isdir(os.path.join(path, program))):
                return os.path.join(path, program)
        return None

    def get_command_line(self):
        """
        Public method returning the reconstructed command line ran via the lib

        :return: the full nmap command line to run
        :rtype: string
        """
        return ("{0} {1} {2} {3} {4}".format(self.__sudo_run,
                                             self.__nmap_binary,
                                             self.__nmap_fixed_options,
                                             self.__nmap_dynamic_options,
                                             " ".join(self.__nmap_targets)))

    def sudo_run(self, run_as='root'):
        """
        Public method enabling the library's user to run the scan with
        priviledges via sudo. The sudo configuration should be set manually
        on the local system otherwise sudo will prompt for a password.
        This method alters the command line by prefixing the sudo command to
        nmap and will then call self.run()

        :param run_as: user name to which the lib needs to sudo to run the scan

        :return: return code from nmap execution
        """
        sudo_user = run_as.split().pop()
        try:
            pwd.getpwnam(sudo_user).pw_uid
        except KeyError:
            _exmsg = ("Username {0} does not exists. Please supply"
                      " a valid username".format(run_as))
            raise EnvironmentError(_exmsg)

        sudo_path = self._whereis("sudo")
        if sudo_path is None:
            raise EnvironmentError(2, "sudo is not installed or "
                                      "could not be found in system path: "
                                      "cannot run nmap with sudo")

        self.__sudo_run = "{0} -u {1}".format(sudo_path, sudo_user)
        rc = self.run()
        self.__sudo_run = ""

        return rc

    def sudo_run_background(self, run_as='root'):
        """
        Public method enabling the library's user to run in background a
        nmap scan with priviledges via sudo.
        The sudo configuration should be set manually on the local system
        otherwise sudo will prompt for a password.
        This method alters the command line by prefixing the sudo command to
        nmap and will then call self.run()

        :param run_as: user name to which the lib needs to sudo to run the scan

        :return: return code from nmap execution
        """
        sudo_user = run_as.split().pop()
        try:
            pwd.getpwnam(sudo_user).pw_uid
        except KeyError:
            _exmsg = ("Username {0} does not exists. Please supply"
                      " a valid username".format(run_as))
            raise EnvironmentError(_exmsg)

        sudo_path = self._whereis("sudo")
        if sudo_path is None:
            raise EnvironmentError(2, "sudo is not installed or "
                                      "could not be found in system path: "
                                      "cannot run nmap with sudo")

        self.__sudo_run = "{0} -u {1}".format(sudo_path, sudo_user)
        super(NmapProcess, self).start()

    def run(self):
        """
        Public method which is usually called right after the constructor
        of NmapProcess. This method starts the nmap executable's subprocess.
        It will also bind a Process that will read from subprocess' stdout
        and stderr and push the lines read in a python queue for futher
        processing. This processing is waken-up each time data is pushed
        from the nmap binary into the stdout reading routine. Processing
        could be performed by a user-provided callback. The whole
        NmapProcess object could be accessible asynchroneously.

        return: return code from nmap execution
        """
        def ioreader_routine(proc_stdout, io_queue, data_pushed, producing):
            """
            local function that will read lines from a file descriptor
            and put the data in a python queue for futher processing.

            :param proc_stdout: file descriptor to read lines from.
            :param io_queue: queue in which read lines will be pushed.
            :param data_pushed: queue used to push data read from the
            nmap stdout back into the parent process
            :param producing: shared variable to notify the parent process
            that processing is either running, either over.
            """
            producing.value = 1
            for streamline in iter(proc_stdout.readline, b''):
                if self.__process_killed.is_set():
                    break
                if streamline is not None:
                    try:
                        io_queue.put(str(streamline.decode()))
                    except Full:
                        pass
                    data_pushed.set()
            producing.value = 0
            data_pushed.set()

        self._run_init()
        producing = multiprocessing.Value('i', 1)
        data_pushed = multiprocessing.Event()
        self.__qout = multiprocessing.Queue()

        _tmp_cmdline = shlex.split(self.__nmap_command_line)
        try:
            self.__nmap_proc = subprocess.Popen(args=_tmp_cmdline,
                                                stdout=subprocess.PIPE,
                                                stderr=subprocess.PIPE,
                                                bufsize=0)
            ioreader = multiprocessing.Process(target=ioreader_routine,
                                               args=(self.__nmap_proc.stdout,
                                                     self.__qout,
                                                     data_pushed,
                                                     producing))
            # ioreader.daemon=False
            ioreader.start()
            self.__state = self.RUNNING
        except OSError:
            self.__state = self.FAILED
            raise EnvironmentError(1, "nmap is not installed or could "
                                      "not be found in system path")

        thread_stream = ''
        while(self.__nmap_proc.poll() is None or producing.value == 1):
            if self.__process_killed.is_set():
                break
            if producing.value == 1 and self.__qout.empty():
                try:
                    data_pushed.wait()
                except KeyboardInterrupt:
                    break
            try:
                thread_stream = self.__qout.get_nowait()
            except Empty:
                pass
            except KeyboardInterrupt:
                break
            else:
                self.__stdout += thread_stream
                evnt = self.__process_event(thread_stream)
                if self.__nmap_event_callback and evnt:
                    self.__nmap_event_callback(self)
            data_pushed.clear()
        ioreader.join()
        # queue clean-up
        while not self.__qout.empty():
            self.__stdout += self.__qout.get_nowait()
        self.__stderr += str(self.__nmap_proc.stderr.read().decode())

        self.__nmap_rc = self.__nmap_proc.poll()
        if self.rc is None:
            self.__state = self.CANCELLED
        elif self.rc == 0:
            self.__state = self.DONE
            if self.current_task:
                self.__nmap_tasks[self.current_task.name].progress = 100
        else:
            self.__state = self.FAILED
        return self.rc

    def run_background(self):
        """
        run nmap scan in background as a thread.
        For privileged scans, consider NmapProcess.sudo_run_background()
        """
        self.__state = self.RUNNING
        super(NmapProcess, self).start()

    def is_running(self):
        """
        Checks if nmap is still running.

        :return: True if nmap is still running
        """
        return self.state == self.RUNNING

    def has_terminated(self):
        """
        Checks if nmap has terminated. Could have failed or succeeded

        :return: True if nmap process is not running anymore.
        """
        return (self.state == self.DONE or self.state == self.FAILED
                or self.state == self.CANCELLED)

    def has_failed(self):
        """
        Checks if nmap has failed.

        :return: True if nmap process errored.
        """
        return self.state == self.FAILED

    def is_successful(self):
        """
        Checks if nmap terminated successfully.

        :return: True if nmap terminated successfully.
        """
        return self.state == self.DONE

    def stop(self):
        """
        Send KILL -15 to the nmap subprocess and gently ask the threads to
        stop.
        """
        self.__state = self.CANCELLED
        if self.__nmap_proc.poll() is None:
            self.__nmap_proc.kill()
        self.__qout.cancel_join_thread()
        self.__process_killed.set()

    def __process_event(self, eventdata):
        """
        Private method called while nmap process is running. It enables the
        library to handle specific data/events produced by nmap process.
        So far, the following events are supported:

        1. task progress: updates estimated time to completion and percentage
           done while scan is running. Could be used in combination with a
           callback function which could then handle this data while scan is
           running.
        2. nmap run: header of the scan. Usually displayed when nmap is started
        3. finished: when nmap scan ends.

        :return: True is event is known.

        :todo: handle parsing directly via NmapParser.parse()
        """
        rval = False
        try:
            edomdoc = pulldom.parseString(eventdata)
            for xlmnt, xmlnode in edomdoc:
                if xlmnt is not None and xlmnt == pulldom.START_ELEMENT:
                    if (xmlnode.nodeName == 'taskbegin' and
                            xmlnode.attributes.keys()):
                        xt = xmlnode.attributes
                        taskname = xt['task'].value
                        starttime = xt['time'].value
                        xinfo = ''
                        if 'extrainfo' in xt.keys():
                            xinfo = xt['extrainfo'].value
                        newtask = NmapTask(taskname, starttime, xinfo)
                        self.__nmap_tasks[newtask.name] = newtask
                        self.__current_task = newtask.name
                        rval = True
                    elif (xmlnode.nodeName == 'taskend' and
                            xmlnode.attributes.keys()):
                        xt = xmlnode.attributes
                        tname = xt['task'].value
                        xinfo = ''
                        self.__nmap_tasks[tname].endtime = xt['time'].value
                        if 'extrainfo' in xt.keys():
                            xinfo = xt['extrainfo'].value
                        self.__nmap_tasks[tname].extrainfo = xinfo
                        self.__nmap_tasks[tname].status = "ended"
                        rval = True
                    elif (xmlnode.nodeName == 'taskprogress' and
                            xmlnode.attributes.keys()):
                        xt = xmlnode.attributes
                        tname = xt['task'].value
                        percent = xt['percent'].value
                        etc = xt['etc'].value
                        remaining = xt['remaining'].value
                        updated = xt['time'].value
                        self.__nmap_tasks[tname].percent = percent
                        self.__nmap_tasks[tname].progress = percent
                        self.__nmap_tasks[tname].etc = etc
                        self.__nmap_tasks[tname].remaining = remaining
                        self.__nmap_tasks[tname].updated = updated
                        rval = True
                    elif (xmlnode.nodeName == 'nmaprun' and
                            xmlnode.attributes.keys()):
                        self.__starttime = xmlnode.attributes['start'].value
                        self.__version = xmlnode.attributes['version'].value
                        rval = True
                    elif (xmlnode.nodeName == 'finished' and
                            xmlnode.attributes.keys()):
                        self.__endtime = xmlnode.attributes['time'].value
                        self.__elapsed = xmlnode.attributes['elapsed'].value
                        self.__summary = xmlnode.attributes['summary'].value
                        rval = True
        except:
            pass
        return rval

    @property
    def command(self):
        """
        return the constructed nmap command or empty string if not
        constructed yet.

        :return: string
        """
        return self.__nmap_command_line or ''

    @property
    def targets(self):
        """
        Provides the list of targets to scan

        :return: list of string
        """
        return self.__nmap_targets

    @property
    def options(self):
        """
        Provides the list of options for that scan

        :return: list of string (nmap options)
        """
        return self._nmap_options

    @property
    def state(self):
        """
        Accessor for nmap execution state. Possible states are:

        - self.READY
        - self.RUNNING
        - self.FAILED
        - self.CANCELLED
        - self.DONE

        :return: integer (from above documented enum)
        """
        return self.__state

    @property
    def starttime(self):
        """
        Accessor for time when scan started

        :return: string. Unix timestamp
        """
        return self.__starttime

    @property
    def endtime(self):
        """
        Accessor for time when scan ended

        :return: string. Unix timestamp
        """
        warnings.warn("data collected from finished events are deprecated."
                      "Use NmapParser.parse()", DeprecationWarning)
        return self.__endtime

    @property
    def elapsed(self):
        """
        Accessor returning for how long the scan ran (in seconds)

        :return: string
        """
        warnings.warn("data collected from finished events are deprecated."
                      "Use NmapParser.parse()", DeprecationWarning)
        return self.__elapsed

    @property
    def summary(self):
        """
        Accessor returning a short summary of the scan's results

        :return: string
        """
        warnings.warn("data collected from finished events are deprecated."
                      "Use NmapParser.parse()", DeprecationWarning)
        return self.__summary

    @property
    def tasks(self):
        """
        Accessor returning for the list of tasks ran during nmap scan

        :return: dict of NmapTask object
        """
        return self.__nmap_tasks

    @property
    def version(self):
        """
        Accessor for nmap binary version number

        :return: version number of nmap binary
        :rtype: string
        """
        return self.__version

    @property
    def current_task(self):
        """
        Accessor for the current NmapTask beeing run

        :return: NmapTask or None if no task started yet
        """
        rval = None
        if len(self.__current_task):
            rval = self.tasks[self.__current_task]
        return rval

    @property
    def etc(self):
        """
        Accessor for estimated time to completion

        :return:  estimated time to completion
        """
        rval = 0
        if self.current_task:
            rval = self.current_task.etc
        return rval

    @property
    def progress(self):
        """
        Accessor for progress status in percentage

        :return: percentage of job processed.
        """
        rval = 0
        if self.current_task:
            rval = self.current_task.progress
        return rval

    @property
    def rc(self):
        """
        Accessor for nmap execution's return code

        :return: nmap execution's return code
        """
        return self.__nmap_rc

    @property
    def stdout(self):
        """
        Accessor for nmap standart output

        :return: output from nmap scan in XML
        :rtype: string
        """
        return self.__stdout

    @property
    def stderr(self):
        """
        Accessor for nmap standart error

        :return: output from nmap when errors occured.
        :rtype: string
        """
        return self.__stderr


def main():
    def mycallback(nmapscan=None):
        if nmapscan.is_running() and nmapscan.current_task:
            ntask = nmapscan.current_task
            print("Task {0} ({1}): ETC: {2} DONE: {3}%".format(ntask.name,
                                                               ntask.status,
                                                               ntask.etc,
                                                               ntask.progress))
    nm = NmapProcess("scanme.nmap.org",
                     options="-A",
                     event_callback=mycallback)
    rc = nm.run()
    if rc == 0:
        print("Scan started at {0} nmap version: {1}").format(nm.starttime,
                                                              nm.version)
        print("state: {0} (rc: {1})").format(nm.state, nm.rc)
        print("results size: {0}").format(len(nm.stdout))
        print("Scan ended {0}: {1}").format(nm.endtime, nm.summary)
    else:
        print("state: {0} (rc: {1})").format(nm.state, nm.rc)
        print("Error: {stderr}").format(stderr=nm.stderr)
        print("Result: {0}").format(nm.stdout)

if __name__ == '__main__':
    main()


================================================
FILE: libnmap/reportjson.py
================================================
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json
from libnmap.objects import NmapHost, NmapService, NmapReport
from libnmap.objects.os import NmapOSFingerprint, NmapOSMatch, NmapOSClass
from libnmap.objects.os import CPE, OSFPPortUsed
from libnmap.parser import NmapParser


class ReportEncoder(json.JSONEncoder):
    def default(self, obj):
        otype = {'NmapHost': NmapHost,
                 'NmapOSFingerprint': NmapOSFingerprint,
                 'NmapOSMatch': NmapOSMatch,
                 'NmapOSClass': NmapOSClass,
                 'CPE': CPE,
                 'OSFPPortUsed': OSFPPortUsed,
                 'NmapService': NmapService,
                 'NmapReport': NmapReport}
        if isinstance(obj, tuple(otype.values())):
            key = ('__{0}__').format(obj.__class__.__name__)
            return {key: obj.__dict__}
        return json.JSONEncoder.default(self, obj)


class ReportDecoder(json.JSONDecoder):
    def decode(self, json_str):
        r = NmapParser.parse_fromdict(json.loads(json_str))
        return r


================================================
FILE: run.py
================================================
#!/usr/bin/env python
# encoding: utf-8
# tasks.py
# email: ringzero@0x557.org

import sys
from tasks import *
from wyfunc import make_target_list

def start_nmap_dispath(targets, taskid=None):
	print '-' * 50
	target_list = make_target_list(targets)

	for target in target_list:
		print '-' * 50
		print '* push %s to Redis' % target
		print '* AsyncResult:%s' % nmap_dispath.delay(target,taskid=taskid)

	print '-' * 50
	print '* Push nmapscan tasks complete.'
	print '-' * 50

if __name__ == "__main__":
	if len(sys.argv) == 2:
		start_nmap_dispath(sys.argv[1])
		sys.exit(0)
	elif len(sys.argv) == 3:
		start_nmap_dispath(sys.argv[1], sys.argv[2])
	else:
		print ("usage: %s targets taskid" % sys.argv[0])
		sys.exit(-1)

================================================
FILE: src/supervisord_client.conf
================================================
; Sample supervisor config file.
;
; For more information on the config file, please see:
; http://supervisord.org/configuration.html
;
; Notes:
;  - Shell expansion ("~" or "$HOME") is not supported.  Environment
;    variables can be expanded using this syntax: "%(ENV_HOME)s".
;  - Comments must have a leading space: "a=b ;comment" not "a=b;comment".

[unix_http_server]
file=/tmp/supervisor.sock   ; (the path to the socket file)
;chmod=0700                 ; socket file mode (default 0700)
;chown=nobody:nogroup       ; socket file uid:gid owner
;username=user              ; (default is no username (open server))
;password=123               ; (default is no password (open server))

[inet_http_server]         ; inet (TCP) server disabled by default
port=0.0.0.0:9001        ; (ip_address:port specifier, *:port for all iface)
;username=user              ; (default is no username (open server))
;password=123               ; (default is no password (open server))

[supervisord]
logfile=/tmp/supervisord.log ; (main log file;default $CWD/supervisord.log)
logfile_maxbytes=50MB        ; (max main logfile bytes b4 rotation;default 50MB)
logfile_backups=10           ; (num of main logfile rotation backups;default 10)
loglevel=info                ; (log level;default info; others: debug,warn,trace)
pidfile=/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
nodaemon=false               ; (start in foreground if true;default false)
minfds=1024                  ; (min. avail startup file descriptors;default 1024)
minprocs=200                 ; (min. avail process descriptors;default 200)
;umask=022                   ; (process file creation umask;default 022)
;user=chrism                 ; (default is current user, required if root)
;identifier=supervisor       ; (supervisord identifier, default is 'supervisor')
;directory=/tmp              ; (default is not to cd during start)
;nocleanup=true              ; (don't clean up tempfiles at start;default false)
;childlogdir=/tmp            ; ('AUTO' child log dir, default $TEMP)
;environment=KEY="value"     ; (key value pairs to add to environment)
;strip_ansi=false            ; (strip ansi escape codes in logs; def. false)

; the below section must remain in the config file for RPC
; (supervisorctl/web interface) to work, additional interfaces may be
; added by defining them in separate rpcinterface: sections
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[supervisorctl]
serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket
;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket
;username=chris              ; should be same as http_username if set
;password=123                ; should be same as http_password if set
;prompt=mysupervisor         ; cmd line prompt (default "supervisor")
;history_file=~/.sc_history  ; use readline history if available

; The below sample program section shows all possible program subsection values,
; create one or more 'real' program: sections to be able to control them under
; supervisor.

;[program:theprogramname]
;command=/bin/cat              ; the program (relative uses PATH, can take args)
;process_name=%(program_name)s ; process_name expr (default %(program_name)s)
;numprocs=1                    ; number of processes copies to start (def 1)
;directory=/tmp                ; directory to cwd to before exec (def no cwd)
;umask=022                     ; umask for process (default None)
;priority=999                  ; the relative start priority (default 999)
;autostart=true                ; start at supervisord start (default: true)
;autorestart=unexpected        ; whether/when to restart (default: unexpected)
;startsecs=1                   ; number of secs prog must stay running (def. 1)
;startretries=3                ; max # of serial start failures (default 3)
;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)
;stopsignal=QUIT               ; signal used to kill process (default TERM)
;stopwaitsecs=10               ; max num secs to wait b4 SIGKILL (default 10)
;stopasgroup=false             ; send stop signal to the UNIX process group (default false)
;killasgroup=false             ; SIGKILL the UNIX process group (def false)
;user=chrism                   ; setuid to this UNIX account to run the program
;redirect_stderr=true          ; redirect proc stderr to stdout (default false)
;stdout_logfile=/a/path        ; stdout log path, NONE for none; default AUTO
;stdout_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
;stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)
;stdout_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
;stdout_events_enabled=false   ; emit events on stdout writes (default false)
;stderr_logfile=/a/path        ; stderr log path, NONE for none; default AUTO
;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
;stderr_logfile_backups=10     ; # of stderr logfile backups (default 10)
;stderr_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
;stderr_events_enabled=false   ; emit events on stderr writes (default false)
;environment=A="1",B="2"       ; process environment additions (def no adds)
;serverurl=AUTO                ; override serverurl computation (childutils)

; The below sample eventlistener section shows all possible
; eventlistener subsection values, create one or more 'real'
; eventlistener: sections to be able to handle event notifications
; sent by supervisor.

;[eventlistener:theeventlistenername]
;command=/bin/eventlistener    ; the program (relative uses PATH, can take args)
;process_name=%(program_name)s ; process_name expr (default %(program_name)s)
;numprocs=1                    ; number of processes copies to start (def 1)
;events=EVENT                  ; event notif. types to subscribe to (req'd)
;buffer_size=10                ; event buffer queue size (default 10)
;directory=/tmp                ; directory to cwd to before exec (def no cwd)
;umask=022                     ; umask for process (default None)
;priority=-1                   ; the relative start priority (default -1)
;autostart=true                ; start at supervisord start (default: true)
;autorestart=unexpected        ; whether/when to restart (default: unexpected)
;startsecs=1                   ; number of secs prog must stay running (def. 1)
;startretries=3                ; max # of serial start failures (default 3)
;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)
;stopsignal=QUIT               ; signal used to kill process (default TERM)
;stopwaitsecs=10               ; max num secs to wait b4 SIGKILL (default 10)
;stopasgroup=false             ; send stop signal to the UNIX process group (default false)
;killasgroup=false             ; SIGKILL the UNIX process group (def false)
;user=chrism                   ; setuid to this UNIX account to run the program
;redirect_stderr=true          ; redirect proc stderr to stdout (default false)
;stdout_logfile=/a/path        ; stdout log path, NONE for none; default AUTO
;stdout_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
;stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)
;stdout_events_enabled=false   ; emit events on stdout writes (default false)
;stderr_logfile=/a/path        ; stderr log path, NONE for none; default AUTO
;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
;stderr_logfile_backups        ; # of stderr logfile backups (default 10)
;stderr_events_enabled=false   ; emit events on stderr writes (default false)
;environment=A="1",B="2"       ; process environment additions
;serverurl=AUTO                ; override serverurl computation (childutils)

; The below sample group section shows all possible group values,
; create one or more 'real' group: sections to create "heterogeneous"
; process groups.

;[group:thegroupname]
;programs=progname1,progname2  ; each refers to 'x' in [program:x] definitions
;priority=999                  ; the relative start priority (default 999)

; The [include] section can just contain the "files" setting.  This
; setting can list multiple files (separated by whitespace or
; newlines).  It can also contain wildcards.  The filenames are
; interpreted as relative to this file.  Included files *cannot*
; include files themselves.

[program:worker-ringzero]
command=celery -A tasks worker --loglevel=info --workdir=/home/thorns --concurrency=10 --hostname=%(program_name)s%(process_num)02d
directory=/home/thorns
user=root
process_name=%(program_name)s_%(process_num)02d
numprocs=1
autostart=true
autorestart=true
startetries=10
exitcodes=0
stopsignal=KILL
stopwaitsecs=10
redirect_stderr=true

;[include]
;files = relative/directory/*.ini


================================================
FILE: src/supervisord_server.conf
================================================
; Sample supervisor config file.
;
; For more information on the config file, please see:
; http://supervisord.org/configuration.html
;
; Notes:
;  - Shell expansion ("~" or "$HOME") is not supported.  Environment
;    variables can be expanded using this syntax: "%(ENV_HOME)s".
;  - Comments must have a leading space: "a=b ;comment" not "a=b;comment".

[unix_http_server]
file=/tmp/supervisor.sock   ; (the path to the socket file)
;chmod=0700                 ; socket file mode (default 0700)
;chown=nobody:nogroup       ; socket file uid:gid owner
;username=user              ; (default is no username (open server))
;password=123               ; (default is no password (open server))

[inet_http_server]         ; inet (TCP) server disabled by default
port=0.0.0.0:9001        ; (ip_address:port specifier, *:port for all iface)
;username=user              ; (default is no username (open server))
;password=123               ; (default is no password (open server))

[supervisord]
logfile=/tmp/supervisord.log ; (main log file;default $CWD/supervisord.log)
logfile_maxbytes=50MB        ; (max main logfile bytes b4 rotation;default 50MB)
logfile_backups=10           ; (num of main logfile rotation backups;default 10)
loglevel=info                ; (log level;default info; others: debug,warn,trace)
pidfile=/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
nodaemon=false               ; (start in foreground if true;default false)
minfds=1024                  ; (min. avail startup file descriptors;default 1024)
minprocs=200                 ; (min. avail process descriptors;default 200)
;umask=022                   ; (process file creation umask;default 022)
;user=chrism                 ; (default is current user, required if root)
;identifier=supervisor       ; (supervisord identifier, default is 'supervisor')
;directory=/tmp              ; (default is not to cd during start)
;nocleanup=true              ; (don't clean up tempfiles at start;default false)
;childlogdir=/tmp            ; ('AUTO' child log dir, default $TEMP)
;environment=KEY="value"     ; (key value pairs to add to environment)
;strip_ansi=false            ; (strip ansi escape codes in logs; def. false)

; the below section must remain in the config file for RPC
; (supervisorctl/web interface) to work, additional interfaces may be
; added by defining them in separate rpcinterface: sections
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[supervisorctl]
serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket
;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket
;username=chris              ; should be same as http_username if set
;password=123                ; should be same as http_password if set
;prompt=mysupervisor         ; cmd line prompt (default "supervisor")
;history_file=~/.sc_history  ; use readline history if available

; The below sample program section shows all possible program subsection values,
; create one or more 'real' program: sections to be able to control them under
; supervisor.

;[program:theprogramname]
;command=/bin/cat              ; the program (relative uses PATH, can take args)
;process_name=%(program_name)s ; process_name expr (default %(program_name)s)
;numprocs=1                    ; number of processes copies to start (def 1)
;directory=/tmp                ; directory to cwd to before exec (def no cwd)
;umask=022                     ; umask for process (default None)
;priority=999                  ; the relative start priority (default 999)
;autostart=true                ; start at supervisord start (default: true)
;autorestart=unexpected        ; whether/when to restart (default: unexpected)
;startsecs=1                   ; number of secs prog must stay running (def. 1)
;startretries=3                ; max # of serial start failures (default 3)
;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)
;stopsignal=QUIT               ; signal used to kill process (default TERM)
;stopwaitsecs=10               ; max num secs to wait b4 SIGKILL (default 10)
;stopasgroup=false             ; send stop signal to the UNIX process group (default false)
;killasgroup=false             ; SIGKILL the UNIX process group (def false)
;user=chrism                   ; setuid to this UNIX account to run the program
;redirect_stderr=true          ; redirect proc stderr to stdout (default false)
;stdout_logfile=/a/path        ; stdout log path, NONE for none; default AUTO
;stdout_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
;stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)
;stdout_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
;stdout_events_enabled=false   ; emit events on stdout writes (default false)
;stderr_logfile=/a/path        ; stderr log path, NONE for none; default AUTO
;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
;stderr_logfile_backups=10     ; # of stderr logfile backups (default 10)
;stderr_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)
;stderr_events_enabled=false   ; emit events on stderr writes (default false)
;environment=A="1",B="2"       ; process environment additions (def no adds)
;serverurl=AUTO                ; override serverurl computation (childutils)

; The below sample eventlistener section shows all possible
; eventlistener subsection values, create one or more 'real'
; eventlistener: sections to be able to handle event notifications
; sent by supervisor.

;[eventlistener:theeventlistenername]
;command=/bin/eventlistener    ; the program (relative uses PATH, can take args)
;process_name=%(program_name)s ; process_name expr (default %(program_name)s)
;numprocs=1                    ; number of processes copies to start (def 1)
;events=EVENT                  ; event notif. types to subscribe to (req'd)
;buffer_size=10                ; event buffer queue size (default 10)
;directory=/tmp                ; directory to cwd to before exec (def no cwd)
;umask=022                     ; umask for process (default None)
;priority=-1                   ; the relative start priority (default -1)
;autostart=true                ; start at supervisord start (default: true)
;autorestart=unexpected        ; whether/when to restart (default: unexpected)
;startsecs=1                   ; number of secs prog must stay running (def. 1)
;startretries=3                ; max # of serial start failures (default 3)
;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)
;stopsignal=QUIT               ; signal used to kill process (default TERM)
;stopwaitsecs=10               ; max num secs to wait b4 SIGKILL (default 10)
;stopasgroup=false             ; send stop signal to the UNIX process group (default false)
;killasgroup=false             ; SIGKILL the UNIX process group (def false)
;user=chrism                   ; setuid to this UNIX account to run the program
;redirect_stderr=true          ; redirect proc stderr to stdout (default false)
;stdout_logfile=/a/path        ; stdout log path, NONE for none; default AUTO
;stdout_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
;stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)
;stdout_events_enabled=false   ; emit events on stdout writes (default false)
;stderr_logfile=/a/path        ; stderr log path, NONE for none; default AUTO
;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)
;stderr_logfile_backups        ; # of stderr logfile backups (default 10)
;stderr_events_enabled=false   ; emit events on stderr writes (default false)
;environment=A="1",B="2"       ; process environment additions
;serverurl=AUTO                ; override serverurl computation (childutils)

; The below sample group section shows all possible group values,
; create one or more 'real' group: sections to create "heterogeneous"
; process groups.

;[group:thegroupname]
;programs=progname1,progname2  ; each refers to 'x' in [program:x] definitions
;priority=999                  ; the relative start priority (default 999)

; The [include] section can just contain the "files" setting.  This
; setting can list multiple files (separated by whitespace or
; newlines).  It can also contain wildcards.  The filenames are
; interpreted as relative to this file.  Included files *cannot*
; include files themselves.

[program:flower]
command=celery flower --port=8080 --broker=redis://127.0.0.1:6379/0
directory=/home/thorns
process_name=%(program_name)s_%(process_num)02d
numprocs=1
autostart=true
autorestart=true
startetries=10
exitcodes=0
stopsignal=KILL
stopwaitsecs=10
redirect_stderr=true

[program:worker-ringzero]
command=celery -A tasks worker --loglevel=info --workdir=/home/thorns --concurrency=10 --hostname=%(program_name)s%(process_num)02d
directory=/home/thorns
user=root
process_name=%(program_name)s_%(process_num)02d
numprocs=1
autostart=true
autorestart=true
startetries=10
exitcodes=0
stopsignal=KILL
stopwaitsecs=10
redirect_stderr=true

;[include]
;files = relative/directory/*.ini


================================================
FILE: tasks.py
================================================
#!/usr/bin/env python
# encoding: utf-8
# tasks.py
# email: ringzero@0x557.org

'''
	Thorns Project 分布式任务控制脚本
	tasks
		-- nmap_dispath			# nmap 扫描调度函数
		-- hydra_dispath 		# hydra 暴力破解调度函数
		-- medusa_dispath 		# medusa 暴力破解调度函数

	worker run()
		--workdir=/home/thorns
'''

import subprocess
from celery import Celery, platforms 

# 初始化芹菜对象
app = Celery()

# 允许celery以root权限启动
platforms.C_FORCE_ROOT = True

# 修改celery的全局配置
app.conf.update(
	CELERY_IMPORTS = ("tasks", ),
	BROKER_URL = 'redis://120.132.54.90:6379/0',
	CELERY_RESULT_BACKEND = 'db+mysql://celery:celery1@3Wscan@42.62.52.62:443/wscan',
	CELERY_TASK_SERIALIZER='json',
	CELERY_RESULT_SERIALIZER='json',
	CELERY_TIMEZONE='Asia/Shanghai',
	CELERY_ENABLE_UTC=True,
	CELERY_REDIS_MAX_CONNECTIONS=5000, # Redis 最大连接数
	BROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 3600}, # 如果任务没有在 可见性超时 内确认接收，任务会被重新委派给另一个Worker并执行  默认1 hour.
	# BROKER_TRANSPORT_OPTIONS = {'fanout_prefix': True},		# 设置一个传输选项来给消息加上前缀
)

# 失败任务重启休眠时间300秒，最大重试次数5次
# @app.task(bind=True, default_retry_delay=300, max_retries=5)

@app.task
def nmap_dispath(targets, taskid=None):
	# nmap环境参数配置
	run_script_path = '/home/thorns'
	if taskid == None:
		cmdline = 'python wyportmap.py %s' % targets
	else: 
		cmdline = 'python wyportmap.py %s %s' % (targets, taskid)
	nmap_proc = subprocess.Popen(cmdline,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
	process_output = nmap_proc.stdout.readlines()
	return process_output

@app.task
def hydra_dispath(targets, protocol, userdic, passdic, taskid=None):
	# 命令执行环境参数配置
	run_script_path = '/home/thorns/script/hydra'
	run_env = '{"LD_LIBRARY_PATH": "/home/thorns/libs/"}'

	if taskid == None:
		cmdline = 'python hydra.py %s %s %s %s' % (target, protocol, userdic, passdic)
	else:
		cmdline = 'python hydra.py %s %s %s %s %s' % (target, protocol, userdic, passdic, taskid)

	nmap_proc = subprocess.Popen(cmdline,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd=run_script_path,env=run_env)

	process_output = nmap_proc.stdout.readlines()
	return process_output

@app.task
def medusa_dispath(targets, protocol, userdic, passdic, taskid=None):
	# 命令执行环境参数配置
	run_script_path = '/home/thorns/script/medusa'
	run_env = '{"LD_LIBRARY_PATH": "/home/thorns/libs/"}'

	if taskid == None:
		cmdline = 'python medusa.py %s %s %s %s' % (target, protocol, userdic, passdic)
	else:
		cmdline = 'python medusa.py %s %s %s %s %s' % (target, protocol, userdic, passdic, taskid)

	nmap_proc = subprocess.Popen(cmdline,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd=run_script_path,env=run_env)

	process_output = nmap_proc.stdout.readlines()
	return process_output







================================================
FILE: wyfunc.py
================================================
#!/usr/bin/env python
# encoding: utf-8
# wyfunc.py
# email: ringzero@0x557.org

def ip2num(ip):
	ip = [int(x) for x in ip.split('.')]
	return ip[0] << 24 | ip[1] << 16 | ip[2] << 8 | ip[3]

def num2ip(num):
	return '%s.%s.%s.%s' % (
		(num & 0xff000000) >> 24,
		(num & 0x00ff0000) >> 16,
		(num & 0x0000ff00) >> 8,
		num & 0x000000ff
	)
 
def gen_ips(start, end):
	"""生成IP地址"""
	# if num & 0xff 过滤掉 最后一段为 0 的IP
	return [num2ip(num) for num in range(start, end + 1) if num & 0xff]

def make_ips_c_block(ipaddr):
	address = {}
	ipaddr = ipaddr.split('.')
	ipaddr[3] = '0'
	ipaddr = '.'.join(ipaddr)

	address[ipaddr] = gen_ips(ip2num(ipaddr),ip2num(ipaddr) + 254)
	return address

def ip_check(ip):
	q = ip.split('.')
	return len(q) == 4 and len(filter(lambda x: x >= 0 and x <= 255, \
		map(int, filter(lambda x: x.isdigit(), q)))) == 4

def make_target_list(targets):
	target_list = []
	process_nmap_count = 1
	startip, endip = targets.split('-')
	if not ip_check(startip):
		print '* StartIP format error'
	else:
		endip_len = len(endip.split('.'))
		startip_num = ip2num(startip)
		if endip_len == 1: # 结束IP为数字的场景
			startip_endnum = startip.split('.')[3]
			target_count =  int(endip) - int(startip_endnum)
			if target_count >= 0:
				target_list.append(num2ip(startip_num))
				# 每10个IP分配到一个wyportmap子进程上
				# 更新：为每个独立的IP创建一个单独的任务，并删除掉开始IP，修改的时候注释掉上面那行
				for i in xrange(0,(target_count+process_nmap_count-1)/process_nmap_count):
					ip_count = (i * process_nmap_count) + process_nmap_count
					remaining_count = target_count - ip_count
					if remaining_count > 0:
						scan_startip = num2ip(startip_num)
						endip_num = startip_num + process_nmap_count
						startip_num = endip_num
						# target_option = '%s-%s' % (scan_startip, num2ip(endip_num))
						target_option = num2ip(endip_num)
						target_list.append(target_option)
					else:
						scan_startip = num2ip(startip_num)
						endip_num = startip_num + process_nmap_count + remaining_count
						startip_num = endip_num
						# target_option = '%s-%s' % (scan_startip, num2ip(endip_num))
						target_option = num2ip(endip_num)
						target_list.append(target_option)
			else:
				print '* EndIP Less than StartIP'
		elif endip_len == 4:
			if ip_check(endip):
				startip_num = ip2num(startip)
				endip_num = ip2num(endip)
				if startip_num <= endip_num:
					target_count =  endip_num - startip_num
					if target_count >= 0:
						target_list.append(num2ip(startip_num))
						# 每10个IP分配到一个wyportmap子进程上
						# 更新：为每个独立的IP创建一个单独的任务，并删除掉开始IP，修改的时候注释掉上面那行
						for i in xrange(0,(target_count+process_nmap_count-1)/process_nmap_count):
							ip_count = (i * process_nmap_count) + process_nmap_count
							remaining_count = target_count - ip_count
							if remaining_count > 0:
								scan_startip = num2ip(startip_num)
								endip_num = startip_num + process_nmap_count
								startip_num = endip_num
								# target_option = '%s-%s' % (scan_startip, num2ip(endip_num))
								target_option = num2ip(endip_num)
								target_list.append(target_option)
							else:
								scan_startip = num2ip(startip_num)
								endip_num = startip_num + process_nmap_count + remaining_count
								startip_num = endip_num
								# target_option = '%s-%s' % (scan_startip, num2ip(endip_num))
								target_option = num2ip(endip_num)
								target_list.append(target_option)
				else:
					print '* EndIP Less than StartIP'
			else:
				print '* EndIP format error'	
		else:
			print '* EndIP format error'

	return target_list




================================================
FILE: wyportmap.py
================================================
#!/usr/bin/env python
# encoding: utf-8
# mail: ringzero@0x557.org

import json
import sys
from time import sleep
from libnmap.process import NmapProcess
from libnmap.reportjson import ReportDecoder, ReportEncoder
from libnmap.parser import NmapParser, NmapParserException
from libnmap.plugins.backendpluginFactory import BackendPluginFactory

# 重试次数 & 超时时间(s)
retrycnt = 3
timeout = 3600

# 数据库连接 & 全局扫描参数
global_dbcoon = 'mysql+mysqldb://celery:celery1@3Wscan@42.62.52.62:443/wscan'
# global_dbcoon = 'mysql+mysqldb://用户名:密码@数据库服务器IP:数据库端口/数据库名称'
global_options = '-sT -P0 -sV -O --script=banner -p T:21-25,80-89,110,143,443,513,873,1080,1433,1521,1158,3306-3308,3389,3690,5900,6379,7001,8000-8090,9000,9418,27017-27019,50060,111,11211,2049'

# 处理端口状态
global_log_states = ['open'] # open, filtered, closed, unfiltered

def do_nmap_scan(targets, options=global_options):
	# 运行次数初始化
	trycnt = 0

	while True:
		# 运行时间初始化
		runtime = 0

		if trycnt >= retrycnt:
			# print '-' * 50
			return 'retry overflow'

		try:
			nmap_proc = NmapProcess(targets=targets, options=options, safe_mode=False)
			nmap_proc.run_background()

			while nmap_proc.is_running():
				if runtime >= timeout:	# 运行超时，结束掉任务，休息1分钟, 再重启这个nmap任务
					# print '-' * 50
					# print "* timeout. terminate it..."
					nmap_proc.stop()
					# 休眠时间
					sleep(60)
					trycnt += 1
					break
				else:
					# print 'running[%ss]:%s' % (runtime, nmap_proc.command)
					sleep(5)
					runtime += 5
			if nmap_proc.is_successful():
				# print '-' * 50
				print nmap_proc.summary
				return nmap_proc.stdout

		except Exception, e:
			# raise e
			print e
			trycnt += 1
			if trycnt >= retrycnt:
				# print '-' * 50
				# print '* retry overflow'
				return e

def parse_nmap_report(nmap_stdout, taskid=None):
	try:
		# 处理结果并写入后台数据库
		nmap_report = NmapParser.parse(nmap_stdout)

		# 声明后台对应的ORM数据库处理模型
		my_services_backend = BackendPluginFactory.create(plugin_name='backend_service', url=global_dbcoon, echo=False, encoding='utf-8', pool_timeout=3600)
		my_hosts_backend = BackendPluginFactory.create(plugin_name='backend_host', url=global_dbcoon, echo=False, encoding='utf-8', pool_timeout=3600)

		# 开始处理扫描结果
		for host in nmap_report.hosts:

				# print("Nmap scan : {0}".format(host.address))
				host.taskid = taskid

				# 处理主机开放的服务和端口
				for serv in host.services:
					serv.address = host.address
					serv.taskid = taskid
					serv.endtime = host.endtime

					if serv.state in global_log_states:
						serv.save(my_services_backend)

				host.save(my_hosts_backend)

		return 'Scan finished'

	except Exception, e:
		# 处理报表出错，返回错误结果
		return e

def run_wyportmap(targets, taskid=None):
	# print '-' * 50
	# print '* Starting id:(%s) [%s] portmap scan' % (taskid, targets)
	# print '-' * 50
	nmap_result = do_nmap_scan(targets)
	# print '-' * 50
	return parse_nmap_report(nmap_result,taskid)

if __name__ == "__main__":
	if len(sys.argv) == 2:
		print run_wyportmap(sys.argv[1])
		sys.exit(0)
	elif len(sys.argv) == 3:
		print run_wyportmap(sys.argv[1], sys.argv[2])
	else:
		print ("usage: %s targets taskid" % sys.argv[0])
		sys.exit(-1)