[
  {
    "path": "ElasticsearchPlugin.py",
    "content": "#!/usr/bin/env python\n\nfrom libnmap.parser import NmapParser\nfrom libnmap.reportjson import ReportDecoder\nfrom libnmap.plugins.es import NmapElasticsearchPlugin\nfrom datetime import datetime\nimport json\n\nnmap_report = NmapParser.parse_fromfile('libnmap/test/files/1_hosts.xml')\nmindex = datetime.fromtimestamp(nmap_report.started).strftime('%Y-%m-%d')\ndb = NmapElasticsearchPlugin(index=mindex)\ndbid = db.insert(nmap_report)\nnmap_json = db.get(dbid)\n\nnmap_obj = json.loads(json.dumps(nmap_json), cls=ReportDecoder)\nprint(nmap_obj)\n#print(db.getall())\n\n"
  },
  {
    "path": "README.md",
    "content": "# thorns\nthorns_project 分布式异步队列系统\n\n运行流程\n-----------------------------------\n* 启动redis内存服务器，作为队列存储数据库使用\n* 配置芹菜(celery)运行环境，并连接redis队列内存，读取执行任务，并返回结果存储到后端MySQL数据库\n* 配置任务控制台花花(flower)，并连接redis队列内存，管理所有worker客户端与执行的任务队列\n* 通过run.py脚本调用celery向队列压入任务\n* 通过flower的http api脚本调用api向队列压入任务\n* 任务执行的结果自动存入后端数据库\n\n反馈\n-----------------------------------\n> 微博：http://weibo.com/ringzero<br />\n> 邮箱：ringzero@0x557.org<br />\n\n运行环境\n-----------------------------------\n* CentOS、Kali Linux、Ubuntu、Debian\n* Python 2.7.x\n* Redis\n* MysQL\n* Celery\n* Tornado\n* Supervisord\n\n安装配置说明\n-----------------------------------\n## CentOS 服务端\n\n#### 安装 Redis-Server\n\t$ wget http://download.redis.io/releases/redis-2.8.19.tar.gz\n\t$ tar xzf redis-2.8.19.tar.gz\n\t$ cd redis-2.8.19\n\t$ make\n\t$ sudo cp src/redis-server /usr/bin/\n\t$ sudo cp redis.conf /etc/redis.conf\n\t/* 修改 /etc/redis.conf 37行，将daemonize no改为daemonize yes，让redis后台运行 */\n\t$ sudo vim /etc/redis.conf\n\tdaemonize yes\n\t# 启动Redis-Server\n\t$ sudo redis-server /etc/redis.conf\n\n#### 安装 pip\n\t$ wget https://pypi.python.org/packages/source/p/pip/pip-6.0.8.tar.gz\n\t$ tar zvxf pip-6.0.8.tar.gz\n\t$ cd pip-6.0.8\n\t$ sudo python setup.py install\n\n#### 安装 MySQL-python\n\t$ sudo yum -y install python-devel mysql-devel subversion-devel\n\t$ sudo pip install MySQL-python SQLAlchemy\n\n#### 安装 Celery\n\t$ sudo pip install -U celery[redis]\n\n#### 安装 Flower & 下载thorns运行环境代码\n\t$ cd /home/\n\t$ sudo yum -y install git\n\t$ git clone https://github.com/ring04h/thorns.git\n\t$ cd /home/thorns/src\n\t$ tar zvxf flower.tar.gz\n\t$ cd flower-0.7.3\n\t$ python setup.py install\n\t/* 启动Flower 这里的redis ip可以配置为你的外网的IP */\n\t$ celery flower --port=8080 --broker=redis://127.0.0.1:6379/0 &\n\t建议使用Supervisord的守护进程来启动Flower，确保系统7*24小时的稳定性\n\n#### 安装 Supervisord\n\t$ sudo pip install supervisor\n\t$ sudo cp /home/thorns/src/supervisord_server.conf /etc/supervisord.conf\n\t/* 修改 /etc/supervisord.conf 141行 修改redis ip为你自己的ip --broker=redis://127.0.0.1:6379/0 */\n\t/* 修改 /etc/supervisord.conf 153行 修改programe为你想定义的worker名称 [program:worker-ringzero] */\n\t$ sudo vim /etc/supervisord.conf\n\t/* 启动 supervisord */\n\t$ supervisord -c /etc/supervisord.conf\n\thttp://127.0.0.1:9001/ 可以在线守护管理thorns的进程，实现远程重启\n\n#### 检查各服务是否正常启动后，开始配置客户端任务脚本\n\t1、http://youip:8080/  thorns 控制台\n\t2、http://youip:9001/  supervisord 控制台\n\t3、修改tasks.py内的芹菜配置\n\t对应你自己的redis-server服务器IP\n\tBROKER_URL = 'redis://120.132.54.90:6379/0',\n\t对应你自己的MySQL-server服务器IP\n\tCELERY_RESULT_BACKEND = 'db+mysql://celery:celery1@3Wscan@42.62.52.62:443/wscan',\n\n\t配置完毕后，就可以部署多台客户端进行分布式任务执行了\n\n## CentOS 客户端（建议大规模部署）\n\t# 安装 git & 下载 thorns_project\n\t$ sudo yum -y install git\n\t$ cd /home/\n\t$ git clone https://github.com/ring04h/thorns.git\n\n\t# 安装 pip\n\t$ wget https://pypi.python.org/packages/source/p/pip/pip-6.0.8.tar.gz\n\t$ tar zvxf pip-6.0.8.tar.gz\n\t$ cd pip-6.0.8\n\t$ sudo python setup.py install\n\n\t# 安装 MySQL-python\n\t$ sudo yum -y install python-devel mysql-devel subversion-devel\n\t$ sudo pip install MySQL-python SQLAlchemy\n\n\t# 安装 nmap\n\t# 32位系统\n\t$ sudo rpm -vhU https://nmap.org/dist/nmap-6.47-1.i386.rpm\n\t# 64位系统\n\t$ sudo rpm -vhU https://nmap.org/dist/nmap-6.47-1.x86_64.rpm\n\n\t# 安装 Celery\n\t$ sudo pip install -U celery[redis]\n\n\t# 安装 Supervisord\n\t$ sudo pip install supervisor\n\t$ sudo cp /home/thorns/src/supervisord_client.conf /etc/supervisord.conf\n\t/* 修改 /etc/supervisord.conf 140行 修改programe为你想定义的worker名称 [program:worker-ringzero] */\n\t$ sudo vim /etc/supervisord.conf\n\t/* 启动 supervisord */\n\t$ supervisord -c /etc/supervisord.conf\n\thttp://127.0.0.1:9001/ 可以在线守护管理thorns的进程，实现远程重启\n\n\t# 修改tasks.py内的芹菜配置（分布式任务关键配置项）\n\t对应你自己的redis-server服务器IP\n\tBROKER_URL = 'redis://120.132.54.90:6379/0',\n\t对应你自己的MySQL-server服务器IP\n\tCELERY_RESULT_BACKEND = 'db+mysql://celery:celery1@3Wscan@42.62.52.62:443/wscan',\n\n\t# 环境搭建完毕，这时候访问thorns project的控制台，就会发现worker客户端已经出现在那里\n\t演示地址：http://thorns.wuyun.org:8080/\n\t你的请访问：http://youip:8080/\n\n\n使用说明(可客户端发起任务也可http api发起任务)\n-----------------------------------\n#### 命令行调用\n\t在你的任意一台worker客户端，或者thorns服务端\n\t$ cd /home/thorns/\n\t$ python run.py 42.62.52.1-42.62.62.254 188\n\t$ python run.py 42.62.52.1-254 189\n\t均可以向redis压入nmap扫描任务，worker客户端的分布式集群会自动分发任务执行，并存储到后台数据库\n\t记得修改wyportmap.py里面的扫描结果，存到你自己的数据库\n\t\n\treinhard-mbp:thorns reinhard$ python run.py 42.62.52.1-254 189\n\t--------------------------------------------------\n\t* push 42.62.52.1 to Redis\n\t* AsyncResult:23147d02-294d-41e5-84e5-5e1b15e72fc4\n\t--------------------------------------------------\n\t* push 42.62.52.2 to Redis\n\t* AsyncResult:542984a4-4434-475f-9a62-bfc81206ea57\n\t--------------------------------------------------\n\t* push 42.62.52.3 to Redis\n\t* AsyncResult:7d005661-d719-41ef-babc-4c853b2c49cc\n\t--------------------------------------------------\n\t* push 42.62.52.4 to Redis\n\t* AsyncResult:ddcf9486-09d9-4dd2-9bb4-2618e6a161b8\n\t--------------------------------------------------\n\n\twyportmap相关帮助: https://github.com/ring04h/wyportmap\n\n#### HTTP API 远程调用\n    重启 worker 线程池:\n    $ curl -X POST http://thorns.wuyun.org:8080/api/worker/pool/restart/myworker\n    \n    远程调用HTTP API启动一个nmap扫描任务：\n    $ curl -X POST -d '{\"args\":[\"42.62.52.62\",2222]}' http://thorns.wuyun.org:8080/api/task/send-task/tasks.nmap_dispath\n\n    强制结束一个正在执行的任务：\n    $ curl -X POST -d 'terminate=True' http://thorns.wuyun.org:8088/api/task/revoke/a9361c1b-fd1d-4f48-9be2-8656a57e906b\n\n"
  },
  {
    "path": "libnmap/__init__.py",
    "content": "# -*- coding: utf-8 -*-\n\n__author__ = 'Ronald Bister, Mike Boutillier'\n__credits__ = ['Ronald Bister', 'Mike Boutillier']\n__maintainer__ = 'Ronald Bister'\n__email__ = 'mini.pelle@gmail.com'\n__license__ = 'CC-BY'\n__version__ = '0.6.1'\n"
  },
  {
    "path": "libnmap/diff.py",
    "content": "# -*- coding: utf-8 -*-\n\n\nclass DictDiffer(object):\n    \"\"\"\n        Calculate the difference between two dictionaries as:\n        (1) items added\n        (2) items removed\n        (3) keys same in both but changed values\n        (4) keys same in both and unchanged values\n    \"\"\"\n    def __init__(self, current_dict, past_dict):\n        self.current_dict = current_dict\n        self.past_dict = past_dict\n        self.set_current = set(current_dict.keys())\n        self.set_past = set(past_dict.keys())\n        self.intersect = self.set_current.intersection(self.set_past)\n\n    def added(self):\n        return self.set_current - self.intersect\n\n    def removed(self):\n        return self.set_past - self.intersect\n\n    def changed(self):\n        return (set(o for o in self.intersect\n                if self.past_dict[o] != self.current_dict[o]))\n\n    def unchanged(self):\n        return (set(o for o in self.intersect\n                if self.past_dict[o] == self.current_dict[o]))\n\n\nclass NmapDiff(DictDiffer):\n    \"\"\"\n        NmapDiff compares two objects of same type to enable the user to check:\n\n        - what has changed\n        - what has been added\n        - what has been removed\n        - what was kept unchanged\n\n        NmapDiff inherit from DictDiffer which makes the actual comparaison.\n        The different methods from DictDiffer used by NmapDiff are the\n        following:\n\n        - NmapDiff.changed()\n        - NmapDiff.added()\n        - NmapDiff.removed()\n        - NmapDiff.unchanged()\n\n        Each of the returns a python set() of key which have changed in the\n        compared objects. To check the different keys that could be returned,\n        refer to the get_dict() method of the objects you which to\n        compare (i.e: libnmap.objects.NmapHost, NmapService,...).\n    \"\"\"\n    def __init__(self, nmap_obj1, nmap_obj2):\n        \"\"\"\n            Constructor of NmapDiff:\n\n            - Checks if the two objects are of the same class\n            - Checks if the objects are \"comparable\" via a call to id() (dirty)\n            - Inherits from DictDiffer and\n        \"\"\"\n        if(nmap_obj1.__class__ != nmap_obj2.__class__ or\n           nmap_obj1.id != nmap_obj2.id):\n            raise NmapDiffException(\"Comparing objects with non-matching id\")\n\n        self.object1 = nmap_obj1.get_dict()\n        self.object2 = nmap_obj2.get_dict()\n\n        DictDiffer.__init__(self, self.object1, self.object2)\n\n    def __repr__(self):\n        return (\"added: [{0}] -- changed: [{1}] -- \"\n                \"unchanged: [{2}] -- removed [{3}]\".format(self.added(),\n                                                           self.changed(),\n                                                           self.unchanged(),\n                                                           self.removed()))\n\n\nclass NmapDiffException(Exception):\n    def __init__(self, msg):\n        self.msg = msg\n"
  },
  {
    "path": "libnmap/objects/__init__.py",
    "content": "# -*- coding: utf-8 -*-\n\nfrom libnmap.objects.report import NmapReport\nfrom libnmap.objects.host import NmapHost\nfrom libnmap.objects.service import NmapService\n\n__all__ = ['NmapReport', 'NmapHost', 'NmapService']\n"
  },
  {
    "path": "libnmap/objects/cpe.py",
    "content": "# -*- coding: utf-8 -*-\n\n\nclass CPE(object):\n    \"\"\"\n        CPE class offers an API for basic CPE objects.\n        These objects could be found in NmapService or in <os> tag\n        within NmapHost.\n\n        :todo: interpret CPE string and provide appropriate API\n    \"\"\"\n    def __init__(self, cpestring):\n        self._cpestring = cpestring\n        self.cpedict = {}\n\n        zk = ['cpe', 'part', 'vendor', 'product', 'version',\n              'update', 'edition', 'language']\n        self._cpedict = dict((k, '') for k in zk)\n        splitup = cpestring.split(':')\n        self._cpedict.update(dict(zip(zk, splitup)))\n\n    @property\n    def cpestring(self):\n        \"\"\"\n            Accessor for the full CPE string.\n        \"\"\"\n        return self._cpestring\n\n    def __repr__(self):\n        return self._cpestring\n\n    def get_part(self):\n        \"\"\"\n            Returns the cpe part (/o, /h, /a)\n        \"\"\"\n        return self._cpedict['part']\n\n    def get_vendor(self):\n        \"\"\"\n            Returns the vendor name\n        \"\"\"\n        return self._cpedict['vendor']\n\n    def get_product(self):\n        \"\"\"\n            Returns the product name\n        \"\"\"\n        return self._cpedict['product']\n\n    def get_version(self):\n        \"\"\"\n            Returns the version of the cpe\n        \"\"\"\n        return self._cpedict['version']\n\n    def get_update(self):\n        \"\"\"\n            Returns the update version\n        \"\"\"\n        return self._cpedict['update']\n\n    def get_edition(self):\n        \"\"\"\n            Returns the cpe edition\n        \"\"\"\n        return self._cpedict['edition']\n\n    def get_language(self):\n        \"\"\"\n            Returns the cpe language\n        \"\"\"\n        return self._cpedict['language']\n\n    def is_application(self):\n        \"\"\"\n            Returns True if cpe describes an application\n        \"\"\"\n        return (self.get_part() == '/a')\n\n    def is_hardware(self):\n        \"\"\"\n            Returns True if cpe describes a hardware\n        \"\"\"\n        return (self.get_part() == '/h')\n\n    def is_operating_system(self):\n        \"\"\"\n            Returns True if cpe describes an operating system\n        \"\"\"\n        return (self.get_part() == '/o')\n"
  },
  {
    "path": "libnmap/objects/host.py",
    "content": "# -*- coding: utf-8 -*-\n\nfrom libnmap.diff import NmapDiff\nfrom libnmap.objects.os import NmapOSFingerprint\n\n\nclass NmapHost(object):\n    \"\"\"\n        NmapHost is a class representing a host object of NmapReport\n    \"\"\"\n    def __init__(self, starttime='', endtime='', address=None, status=None,\n                 hostnames=None, services=None, extras=None):\n        \"\"\"\n            NmapHost constructor\n            :param starttime: unix timestamp of when the scan against\n            that host started\n            :type starttime: string\n            :param endtime: unix timestamp of when the scan against\n            that host ended\n            :type endtime: string\n            :param address: dict ie :{'addr': '127.0.0.1', 'addrtype': 'ipv4'}\n            :param status: dict ie:{'reason': 'localhost-response',\n                                    'state': 'up'}\n            :return: NmapHost:\n        \"\"\"\n        self._starttime = starttime\n        self._endtime = endtime\n        self._hostnames = hostnames if hostnames is not None else []\n        self._status = status if status is not None else {}\n        self._services = services if services is not None else []\n        self._extras = extras if extras is not None else {}\n        self._osfingerprinted = False\n        self.os = None\n        if 'os' in self._extras:\n            self.os = NmapOSFingerprint(self._extras['os'])\n            self._osfingerprinted = True\n        else:\n            self.os = NmapOSFingerprint({})\n\n        self._ipv4_addr = None\n        self._ipv6_addr = None\n        self._mac_addr = None\n        self._vendor = None\n        for addr in address:\n            if addr['addrtype'] == \"ipv4\":\n                self._ipv4_addr = addr['addr']\n            elif addr['addrtype'] == 'ipv6':\n                self._ipv6_addr = addr['addr']\n            elif addr['addrtype'] == 'mac':\n                self._mac_addr = addr['addr']\n            if 'vendor' in addr:\n                self._vendor = addr['vendor']\n\n        self._main_address = self._ipv4_addr or self._ipv6_addr or ''\n        self._address = address\n\n    def __eq__(self, other):\n        \"\"\"\n            Compare eq NmapHost based on :\n\n                - hostnames\n                - address\n                - if an associated services has changed\n\n            :return: boolean\n        \"\"\"\n        rval = False\n        if(self.__class__ == other.__class__ and self.id == other.id):\n            rval = (self.changed(other) == 0)\n        return rval\n\n    def __ne__(self, other):\n        \"\"\"\n            Compare ne NmapHost based on:\n\n                - hostnames\n                - address\n                - if an associated services has changed\n\n            :return: boolean\n        \"\"\"\n        rval = True\n        if(self.__class__ == other.__class__ and self.id == other.id):\n            rval = (self.changed(other) > 0)\n        return rval\n\n    def __repr__(self):\n        \"\"\"\n            String representing the object\n            :return: string\n        \"\"\"\n        return \"{0}: [{1} ({2}) - {3}]\".format(self.__class__.__name__,\n                                               self.address,\n                                               \" \".join(self._hostnames),\n                                               self.status)\n\n    def __hash__(self):\n        \"\"\"\n            Hash is needed to be able to use our object in sets\n            :return: hash\n        \"\"\"\n        return (hash(self.status) ^ hash(self.address) ^\n                hash(frozenset(self._services)) ^\n                hash(frozenset(\" \".join(self._hostnames))))\n\n    def changed(self, other):\n        \"\"\"\n            return the number of attribute who have changed\n            :param other: NmapHost object to compare\n            :return int\n        \"\"\"\n        return len(self.diff(other).changed())\n\n    def save(self, backend):\n        \n        if backend is not None:\n            _id = backend.insert(self)\n        else:\n            raise RuntimeError\n        return _id\n\n    @property\n    def starttime(self):\n        \"\"\"\n            Accessor for the unix timestamp of when the scan was started\n\n            :return: string\n        \"\"\"\n        return self._starttime\n\n    @property\n    def endtime(self):\n        \"\"\"\n            Accessor for the unix timestamp of when the scan ended\n\n            :return: string\n        \"\"\"\n        return self._endtime\n\n    @property\n    def address(self):\n        \"\"\"\n            Accessor for the IP address of the scanned host\n\n            :return: IP address as a string\n        \"\"\"\n        return self._main_address\n\n    @address.setter\n    def address(self, addrdict):\n        \"\"\"\n            Setter for the address dictionnary.\n\n            :param addrdict: valid dict is {'addr': '1.1.1.1',\n                                            'addrtype': 'ipv4'}\n        \"\"\"\n        if addrdict['addrtype'] == 'ipv4':\n            self._ipv4_addr = addrdict['addr']\n        elif addrdict['addrtype'] == 'ipv6':\n            self._ipv6_addr = addrdict['addr']\n        elif addrdict['addrtype'] == 'mac':\n            self._mac_addr = addrdict['addr']\n        if 'vendor' in addrdict:\n            self._vendor = addrdict['vendor']\n\n        self._main_address = self._ipv4_addr or self._ipv6_addr or ''\n        self._address = addrdict\n\n    @property\n    def ipv4(self):\n        \"\"\"\n            Accessor for the IPv4 address of the scanned host\n\n            :return: IPv4 address as a string\n        \"\"\"\n        return self._ipv4_addr or ''\n\n    @property\n    def mac(self):\n        \"\"\"\n            Accessor for the MAC address of the scanned host\n\n            :return: MAC address as a string\n        \"\"\"\n        return self._mac_addr or ''\n\n    @property\n    def vendor(self):\n        \"\"\"\n            Accessor for the vendor attribute of the scanned host\n\n            :return: string (vendor) of empty string if no vendor defined\n        \"\"\"\n        return self._vendor or ''\n\n    @property\n    def ipv6(self):\n        \"\"\"\n            Accessor for the IPv6 address of the scanned host\n\n            :return: IPv6 address as a string\n        \"\"\"\n        return self._ipv6_addr or ''\n\n    @property\n    def status(self):\n        \"\"\"\n            Accessor for the host's status (up, down, unknown...)\n\n            :return: string\n        \"\"\"\n        return self._status['state']\n\n    @status.setter\n    def status(self, statusdict):\n        \"\"\"\n            Setter for the status dictionnary.\n\n            :param statusdict: valid dict is {\"state\": \"open\",\n                                              \"reason\": \"syn-ack\",\n                                              \"reason_ttl\": \"0\"}\n                                'state' is the only mandatory key.\n        \"\"\"\n        self._status = statusdict\n\n    def is_up(self):\n        \"\"\"\n            method to determine if host is up or not\n\n            :return: bool\n        \"\"\"\n        rval = False\n        if self.status == 'up':\n            rval = True\n        return rval\n\n    @property\n    def hostnames(self):\n        \"\"\"\n            Accessor returning the list of hostnames (array of strings).\n\n            :return: array of string\n        \"\"\"\n        return self._hostnames\n\n    @property\n    def services(self):\n        \"\"\"\n            Accessor for the array of scanned services for that host.\n\n            An array of NmapService objects is returned.\n\n            :return: array of NmapService\n        \"\"\"\n        return self._services\n\n    def get_ports(self):\n        \"\"\"\n            Retrieve a list of the port used by each service of the NmapHost\n\n            :return: list: of tuples (port,'proto') ie:[(22,'tcp'),(25, 'tcp')]\n        \"\"\"\n        return [(p.port, p.protocol) for p in self._services]\n\n    def get_open_ports(self):\n        \"\"\"\n            Same as get_ports() but only for open ports\n\n            :return: list: of tuples (port,'proto') ie:[(22,'tcp'),(25, 'tcp')]\n        \"\"\"\n        return ([(p.port, p.protocol)\n                for p in self._services if p.state == 'open'])\n\n    def get_service(self, portno, protocol='tcp'):\n        \"\"\"\n            :param portno: int the portnumber\n            :param protocol='tcp': string ('tcp','udp')\n\n            :return: NmapService or None\n        \"\"\"\n        plist = [p for p in self._services if\n                 p.port == portno and p.protocol == protocol]\n        if len(plist) > 1:\n            raise Exception(\"Duplicate services found in NmapHost object\")\n        return plist.pop() if len(plist) else None\n\n    def get_service_byid(self, service_id):\n        \"\"\"\n            Returns a NmapService by providing its id.\n\n            The id of a nmap service is a python tupl made of (protocol, port)\n        \"\"\"\n        rval = None\n        for _tmpservice in self._services:\n            if _tmpservice.id == service_id:\n                rval = _tmpservice\n        return rval\n\n    def os_class_probabilities(self):\n        \"\"\"\n            Returns an array of possible OS class detected during\n            the OS fingerprinting.\n\n            :return: Array of NmapOSClass objects\n        \"\"\"\n        rval = []\n        if self.os is not None:\n            rval = self.os.osclasses\n        return rval\n\n    def os_match_probabilities(self):\n        \"\"\"\n            Returns an array of possible OS match detected during\n            the OS fingerprinting\n\n            :return: array of NmapOSMatches objects\n        \"\"\"\n        rval = []\n        if self.os is not None:\n            rval = self.os.osmatches\n        return rval\n\n    @property\n    def os_fingerprinted(self):\n        \"\"\"\n            Specify if the host has OS fingerprint data available\n\n            :return: Boolean\n        \"\"\"\n        return self._osfingerprinted\n\n    @property\n    def os_fingerprint(self):\n        \"\"\"\n            Returns the fingerprint of the scanned system.\n\n            :return: string\n        \"\"\"\n        rval = ''\n        if self.os is not None:\n            rval = \"\\n\".join(self.os.fingerprints)\n        return rval\n\n    def os_ports_used(self):\n        \"\"\"\n            Returns an array of the ports used for OS fingerprinting\n\n            :return: array of ports used: [{'portid': '22',\n                                            'proto': 'tcp',\n                                            'state': 'open'},]\n        \"\"\"\n        rval = []\n        try:\n            rval = self._extras['os']['ports_used']\n        except (KeyError, TypeError):\n            pass\n        return rval\n\n    @property\n    def tcpsequence(self):\n        \"\"\"\n            Returns the difficulty to determine remotely predict\n            the tcp sequencing.\n\n            return: string\n        \"\"\"\n        rval = ''\n        try:\n            rval = self._extras['tcpsequence']['difficulty']\n        except (KeyError, TypeError):\n            pass\n        return rval\n\n    @property\n    def ipsequence(self):\n        \"\"\"\n            Return the class of ip sequence of the remote hosts.\n\n            :return: string\n        \"\"\"\n        rval = ''\n        try:\n            rval = self._extras['ipidsequence']['class']\n        except (KeyError, TypeError):\n            pass\n        return rval\n\n    @property\n    def uptime(self):\n        \"\"\"\n            uptime of the remote host (if nmap was able to determine it)\n\n            :return: string (in seconds)\n        \"\"\"\n        rval = 0\n        try:\n            rval = int(self._extras['uptime']['seconds'])\n        except (KeyError, TypeError):\n            pass\n        return rval\n\n    @property\n    def lastboot(self):\n        \"\"\"\n            Since when the host was booted.\n\n            :return: string\n        \"\"\"\n        rval = ''\n        try:\n            rval = self._extras['uptime']['lastboot']\n        except (KeyError, TypeError):\n            pass\n        return rval\n\n    @property\n    def distance(self):\n        \"\"\"\n            Number of hops to host\n\n            :return: int\n        \"\"\"\n        rval = 0\n        try:\n            rval = int(self._extras['distance']['value'])\n        except (KeyError, TypeError):\n            pass\n        return rval\n\n    @property\n    def scripts_results(self):\n        \"\"\"\n            Scripts results specific to the scanned host\n\n            :return: array of <script> dictionary\n        \"\"\"\n        rval = {}\n        try:\n            rval = self._extras['hostscript']\n        except (KeyError, TypeError):\n            pass\n        return rval\n\n    @property\n    def id(self):\n        \"\"\"\n            id of the host. Used for diff()ing NmapObjects\n\n            :return: string\n        \"\"\"\n        return self.address\n\n    @property\n    def extraports_state(self):\n        \"\"\"\n            dictionnary containing state and amount of extra ports scanned\n            for which a common state, usually, closed was discovered.\n\n            :return: dict with keys 'state' and 'count' or None\n        \"\"\"\n        _xtrports = self._extras.get('extraports', None)\n\n        if _xtrports is None:\n            return None\n\n        return {'state': _xtrports['state'], 'count': _xtrports['count']}\n\n    @property\n    def extraports_reasons(self):\n        \"\"\"\n            dictionnary containing reasons why extra ports scanned\n            for which a common state, usually, closed was discovered.\n\n            :return: array of dict containing keys 'state' and 'count' or None\n        \"\"\"\n        r = self._extras.get('extraports', {})\n\n        return r.get('reasons', None)\n\n    def get_dict(self):\n        \"\"\"\n            Return a dict representation of the object.\n\n            This is needed by NmapDiff to allow comparaison\n\n            :return dict\n        \"\"\"\n        d = dict([(\"{0}::{1}\".format(s.__class__.__name__, str(s.id)),\n                   hash(s))\n                 for s in self.services])\n\n        d.update({'address': self.address, 'status': self.status,\n                  'hostnames': \" \".join(self._hostnames)})\n        return d\n\n    def diff(self, other):\n        \"\"\"\n            Calls NmapDiff to check the difference between self and\n            another NmapHost object.\n\n            Will return a NmapDiff object.\n\n            This objects return python set() of keys describing the elements\n            which have changed, were added, removed or kept unchanged.\n\n            :param other: NmapHost to diff with\n\n            :return: NmapDiff object\n        \"\"\"\n        return NmapDiff(self, other)\n"
  },
  {
    "path": "libnmap/objects/os.py",
    "content": "# -*- coding: utf-8 -*-\n\nimport warnings\nfrom libnmap.objects.cpe import CPE\n\n\nclass OSFPPortUsed(object):\n    \"\"\"\n        Port used class: this enables the user of NmapOSFingerprint class\n        to have a common and clear interface to access portused data which\n        were collected and used during os fingerprint scan\n    \"\"\"\n    def __init__(self, port_used_dict):\n        try:\n            self._state = port_used_dict['state']\n            self._proto = port_used_dict['proto']\n            self._portid = port_used_dict['portid']\n        except KeyError:\n            raise Exception(\"Cannot create OSFPPortUsed: missing required key\")\n\n    @property\n    def state(self):\n        \"\"\"\n            Accessor for the portused state (closed, open,...)\n        \"\"\"\n        return self._state\n\n    @property\n    def proto(self):\n        \"\"\"\n            Accessor for the portused protocol (tcp, udp,...)\n        \"\"\"\n        return self._proto\n\n    @property\n    def portid(self):\n        \"\"\"\n            Accessor for the referenced port number used\n        \"\"\"\n        return self._portid\n\n\nclass NmapOSMatch(object):\n    \"\"\"\n        NmapOSMatch is an internal class used for offering results\n        from an nmap os fingerprint. This common interfaces makes\n        a compatibility between old nmap xml (<1.04) and new nmap\n        xml versions (used in nmapv6 for instance).\n\n        In previous xml version, osclass tags from nmap fingerprints\n        were not directly mapped to a osmatch. In new xml version,\n        osclass could be embedded in osmatch tag.\n\n        The approach to solve this is to create a common class\n        which will, for older xml version, match based on the accuracy\n        osclass to an osmatch. If no match, an osmatch will be made up\n        from a concat of os class attributes: vendor and osfamily.\n        Unmatched osclass will have a line attribute of -1.\n\n        More info, see issue #26 or http://seclists.org/nmap-dev/2012/q2/252\n    \"\"\"\n    def __init__(self, osmatch_dict):\n        _osmatch_dict = osmatch_dict['osmatch']\n        if('name' not in _osmatch_dict or\n           'line' not in _osmatch_dict or\n           'accuracy' not in _osmatch_dict):\n            raise Exception(\"Cannot create NmapOSClass: missing required key\")\n\n        self._name = _osmatch_dict['name']\n        self._line = _osmatch_dict['line']\n        self._accuracy = _osmatch_dict['accuracy']\n\n        # create osclass list\n        self._osclasses = []\n        try:\n            for _osclass in osmatch_dict['osclasses']:\n                try:\n                    _osclassobj = NmapOSClass(_osclass)\n                except:\n                    raise Exception(\"Could not create NmapOSClass object\")\n                self._osclasses.append(_osclassobj)\n        except KeyError:\n            pass\n\n    def add_osclass(self, osclass_obj):\n\n        \"\"\"\n            Add a NmapOSClass object to the OSMatch object. This method is\n            useful to implement compatibility with older versions of NMAP\n            by providing a common interface to access os fingerprint data.\n        \"\"\"\n        self._osclasses.append(osclass_obj)\n\n    @property\n    def osclasses(self):\n        \"\"\"\n            Accessor for all NmapOSClass objects matching with this OS Match\n        \"\"\"\n        return self._osclasses\n\n    @property\n    def name(self):\n        \"\"\"\n            Accessor for name attribute (e.g.: Linux 2.4.26 (Slackware 10.0.0))\n        \"\"\"\n        return self._name\n\n    @property\n    def line(self):\n        \"\"\"\n            Accessor for line attribute as integer. value equals -1 if this\n            osmatch holds orphans NmapOSClass objects. This could happen with\n            older version of nmap xml engine (<1.04 (e.g: nmapv6)).\n\n            :return: int\n        \"\"\"\n        return int(self._line)\n\n    @property\n    def accuracy(self):\n        \"\"\"\n            Accessor for accuracy\n\n            :return: int\n        \"\"\"\n        return int(self._accuracy)\n\n    def get_cpe(self):\n        \"\"\"\n            This method return a list of cpe stings and not CPE objects as\n            the NmapOSClass.cpelist property. This method is a helper to\n            simplify data management.\n\n            For more advanced handling of CPE data, use NmapOSClass.cpelist\n            and use the methods from CPE class\n        \"\"\"\n        _cpelist = []\n        for osc in self.osclasses:\n            for cpe in osc.cpelist:\n                _cpelist.append(cpe.cpestring)\n        return _cpelist\n\n    def __repr__(self):\n        rval = \"{0}: {1}\".format(self.name, self.accuracy)\n        for _osclass in self._osclasses:\n            rval += \"\\r\\n  |__ os class: {0}\".format(str(_osclass))\n        return rval\n\n\nclass NmapOSClass(object):\n    \"\"\"\n        NmapOSClass offers an unified API to access data from analysed\n        osclass tag. As implemented in libnmap and newer version of nmap,\n        osclass objects will always be embedded in a NmapOSMatch.\n        Unmatched NmapOSClass will be stored in \"dummy\" NmapOSMatch objects\n        which will have the particularity of have a line attribute of -1.\n        On top of this, NmapOSClass will have optional CPE objects\n        embedded.\n    \"\"\"\n    def __init__(self, osclass_dict):\n        _osclass = osclass_dict['osclass']\n        if('vendor' not in _osclass or\n           'osfamily' not in _osclass or\n           'accuracy' not in _osclass):\n            raise Exception(\"Wrong osclass structure: missing required key\")\n\n        self._vendor = _osclass['vendor']\n        self._osfamily = _osclass['osfamily']\n        self._accuracy = _osclass['accuracy']\n\n        self._osgen = ''\n        self._type = ''\n        # optional data\n        if 'osgen' in _osclass:\n            self._osgen = _osclass['osgen']\n        if 'type' in _osclass:\n            self._type = _osclass['type']\n\n        self._cpelist = []\n        for _cpe in osclass_dict['cpe']:\n            self._cpelist.append(CPE(_cpe))\n\n    @property\n    def cpelist(self):\n        \"\"\"\n            Returns a list of CPE Objects matching with this os class\n\n            :return: list of CPE objects\n            :rtype: Array\n        \"\"\"\n        return self._cpelist\n\n    @property\n    def vendor(self):\n        \"\"\"\n            Accessor for vendor information (Microsoft, Linux,...)\n\n            :return: string\n        \"\"\"\n        return self._vendor\n\n    @property\n    def osfamily(self):\n        \"\"\"\n            Accessor for OS family information (Windows, Linux,...)\n\n            :return: string\n        \"\"\"\n        return self._osfamily\n\n    @property\n    def accuracy(self):\n        \"\"\"\n            Accessor for OS class detection accuracy (int)\n\n            :return: int\n        \"\"\"\n        return int(self._accuracy)\n\n    @property\n    def osgen(self):\n        \"\"\"\n            Accessor for OS class generation (7, 8, 2.4.X,...).\n\n            :return: string\n        \"\"\"\n        return self._osgen\n\n    @property\n    def type(self):\n        \"\"\"\n            Accessor for OS class type (general purpose,...)\n\n            :return: string\n        \"\"\"\n        return self._type\n\n    @property\n    def description(self):\n        \"\"\"\n            Accessor helper which returns a concataned string of\n            the valuable attributes from NmapOSClass object\n\n            :return: string\n        \"\"\"\n        rval = \"{0}: {1}, {2}\".format(self.type, self.vendor, self.osfamily)\n        if len(self.osgen):\n            rval += \"({0})\".format(self.osgen)\n        return rval\n\n    def __repr__(self):\n        rval = \"{0}: {1}, {2}\".format(self.type, self.vendor, self.osfamily)\n        if len(self.osgen):\n            rval += \"({0})\".format(self.osgen)\n        for _cpe in self._cpelist:\n            rval += \"\\r\\n    |__ {0}\".format(str(_cpe))\n        return rval\n\n\nclass NmapOSFingerprint(object):\n    \"\"\"\n        NmapOSFingerprint is a easier API for using os fingerprinting.\n        Data for OS fingerprint (<os> tag) is instanciated from\n        a NmapOSFingerprint which is accessible in NmapHost via NmapHost.os\n    \"\"\"\n    def __init__(self, osfp_data):\n        self.__osmatches = []\n        self.__ports_used = []\n        self.__fingerprints = []\n\n        if 'osmatches' in osfp_data:\n            for _osmatch in osfp_data['osmatches']:\n                _osmatch_obj = NmapOSMatch(_osmatch)\n                self.__osmatches.append(_osmatch_obj)\n        if 'osclasses' in osfp_data:\n            for _osclass in osfp_data['osclasses']:\n                _osclass_obj = NmapOSClass(_osclass)\n                _osmatched = self.get_osmatch(_osclass_obj)\n                if _osmatched is not None:\n                    _osmatched.add_osclass(_osclass_obj)\n                else:\n                    self._add_dummy_osmatch(_osclass_obj)\n        if 'osfingerprints' in osfp_data:\n            for _osfp in osfp_data['osfingerprints']:\n                if 'fingerprint' in _osfp:\n                    self.__fingerprints.append(_osfp['fingerprint'])\n        if 'ports_used' in osfp_data:\n            for _pused_dict in osfp_data['ports_used']:\n                _pused = OSFPPortUsed(_pused_dict)\n                self.__ports_used.append(_pused)\n\n    def get_osmatch(self, osclass_obj):\n        \"\"\"\n            This function enables NmapOSFingerprint to determine if an\n            NmapOSClass object could be attached to an existing NmapOSMatch\n            object in order to respect the common interface for\n            the nmap xml version < 1.04 and >= 1.04\n\n            This method will return an NmapOSMatch object matching with\n            the NmapOSClass provided in parameter\n            (match is performed based on accuracy)\n\n            :return: NmapOSMatch object\n        \"\"\"\n        rval = None\n        for _osmatch in self.__osmatches:\n            if _osmatch.accuracy == osclass_obj.accuracy:\n                rval = _osmatch\n                break  # sorry\n        return rval\n\n    def _add_dummy_osmatch(self, osclass_obj):\n        \"\"\"\n            This functions creates a dummy NmapOSMatch object in order to\n            encapsulate an NmapOSClass object which was not matched with an\n            existing NmapOSMatch object\n        \"\"\"\n        _dname = \"{0}:{1}:{2}\".format(osclass_obj.type,\n                                      osclass_obj.vendor,\n                                      osclass_obj.osfamily)\n        _dummy_dict = {'osmatch': {'name': _dname,\n                                   'accuracy': osclass_obj.accuracy,\n                                   'line': -1},\n                       'osclasses': []}\n        _dummy_osmatch = NmapOSMatch(_dummy_dict)\n        self.__osmatches.append(_dummy_osmatch)\n\n    @property\n    def osmatches(self, min_accuracy=0):\n        _osmatches = []\n\n        for _osmatch in self.__osmatches:\n            if _osmatch.accuracy >= min_accuracy:\n                _osmatches.append(_osmatch)\n\n        return _osmatches\n\n    @property\n    def fingerprint(self):\n        return \"\\r\\n\".join(self.__fingerprints)\n\n    @property\n    def fingerprints(self):\n        return self.__fingerprints\n\n    @property\n    def ports_used(self):\n        \"\"\"\n            Return an array of OSFPPortUsed object with the ports used to\n            perform the os fingerprint. This dict might contain another dict\n            embedded containing the ports_reason values.\n        \"\"\"\n        return self.__ports_used\n\n    def osmatch(self, min_accuracy=90):\n        warnings.warn(\"NmapOSFingerprint.osmatch is deprecated: \"\n                      \"use NmapOSFingerprint.osmatches\", DeprecationWarning)\n        os_array = []\n        for _osmatch in self.__osmatches:\n            if _osmatch.accuracy >= min_accuracy:\n                os_array.append(_osmatch.name)\n        return os_array\n\n    def osclass(self, min_accuracy=90):\n        warnings.warn(\"NmapOSFingerprint.osclass() is deprecated: \"\n                      \"use NmapOSFingerprint.osclasses() if applicable\",\n                      DeprecationWarning)\n        os_array = []\n        for osmatch_entry in self.osmatches():\n            if osmatch_entry.accuracy >= min_accuracy:\n                for oclass in osmatch_entry.osclasses:\n                    _ftstr = \"type:{0}|vendor:{1}|osfamily{2}\".format(\n                             oclass.type,\n                             oclass.vendor,\n                             oclass.osfamily)\n                    os_array.append(_ftstr)\n        return os_array\n\n    def os_cpelist(self):\n        cpelist = []\n        for _osmatch in self.osmatches:\n            for oclass in _osmatch.osclasses:\n                cpelist.extend(oclass.cpelist)\n        return cpelist\n\n    def __repr__(self):\n        rval = \"\"\n        for _osmatch in self.osmatches:\n            rval += \"\\r\\n{0}\".format(_osmatch)\n        rval += \"Fingerprints: \".format(self.fingerprint)\n        return rval\n"
  },
  {
    "path": "libnmap/objects/report.py",
    "content": "# -*- coding: utf-8 -*-\nfrom libnmap.diff import NmapDiff\n\n\nclass NmapReport(object):\n    \"\"\"\n        NmapReport is the usual interface for the end user to\n        read scans output.\n\n        A NmapReport as the following structure:\n\n        - Scan headers data\n        - A list of scanned hosts (NmapReport.hosts)\n        - Scan footer data\n\n        It is to note that each NmapHost comprised in NmapReport.hosts array\n        contains also a list of scanned services (NmapService object).\n\n        This means that if NmapParser.parse*() is the input interface for the\n        end user of the lib. NmapReport is certainly the output interface for\n        the end user of the lib.\n    \"\"\"\n    def __init__(self, raw_data=None):\n        \"\"\"\n            Constructor for NmapReport object.\n\n            This is usually called by the NmapParser module.\n        \"\"\"\n        self._nmaprun = {}\n        self._scaninfo = {}\n        self._hosts = []\n        self._runstats = {}\n        if raw_data is not None:\n            self.__set_raw_data(raw_data)\n\n    def save(self, backend):\n        \"\"\"\n            This method gets a NmapBackendPlugin representing the backend.\n\n            :param backend: libnmap.plugins.PluginBackend object.\n\n            Object created by BackendPluginFactory and enabling nmap reports\n            to be saved/stored in any type of backend implemented in plugins.\n\n            The primary key of the stored object is returned.\n\n            :return: str\n        \"\"\"\n        if backend is not None:\n            _id = backend.insert(self)\n        else:\n            raise RuntimeError\n        return _id\n\n    def diff(self, other):\n        \"\"\"\n            Calls NmapDiff to check the difference between self and\n            another NmapReport object.\n\n            Will return a NmapDiff object.\n\n            :return: NmapDiff object\n            :todo: remove is_consistent approach, diff() should be generic.\n        \"\"\"\n        if self.is_consistent() and other.is_consistent():\n            _rdiff = NmapDiff(self, other)\n        else:\n            _rdiff = set()\n        return _rdiff\n\n    @property\n    def started(self):\n        \"\"\"\n            Accessor returning a unix timestamp of when the scan was started.\n\n            :return: integer\n        \"\"\"\n        rval = -1\n        try:\n            s_start = self._nmaprun['start']\n            rval = int(s_start)\n        except(KeyError, TypeError, ValueError):\n            pass\n        return rval\n\n    @property\n    def commandline(self):\n        \"\"\"\n            Accessor returning the full nmap command line fired.\n\n            :return: string\n        \"\"\"\n        return self._nmaprun['args']\n\n    @property\n    def version(self):\n        \"\"\"\n            Accessor returning the version of the\n            nmap binary used to perform the scan.\n\n            :return: string\n        \"\"\"\n        return self._nmaprun['version']\n\n    @property\n    def scan_type(self):\n        \"\"\"\n            Accessor returning a string which identifies what type of scan\n            was launched (syn, ack, tcp,...).\n\n            :return: string\n        \"\"\"\n        return self._scaninfo['type']\n\n    @property\n    def hosts(self):\n        \"\"\"\n            Accessor returning an array of scanned hosts.\n\n            Scanned hosts are NmapHost objects.\n\n            :return: array of NmapHost\n        \"\"\"\n        return self._hosts\n\n    def get_host_byid(self, host_id):\n        \"\"\"\n           Gets a NmapHost object directly from the host array\n           by looking it up by id.\n\n           :param ip_addr: ip address of the host to lookup\n           :type ip_addr: string\n\n           :return: NmapHost\n        \"\"\"\n        rval = None\n        for _rhost in self._hosts:\n            if _rhost.address == host_id:\n                rval = _rhost\n        return rval\n\n    @property\n    def endtime(self):\n        \"\"\"\n            Accessor returning a unix timestamp of when the scan ended.\n\n            :return: integer\n        \"\"\"\n        rval = -1\n        try:\n            rval = int(self._runstats['finished']['time'])\n        except(KeyError, TypeError, ValueError):\n            pass\n        return rval\n\n    @property\n    def endtimestr(self):\n        \"\"\"\n            Accessor returning a human readable time string\n            of when the scan ended.\n\n            :return: string\n        \"\"\"\n        rval = ''\n        try:\n            rval = self._runstats['finished']['timestr']\n        except(KeyError, TypeError, ValueError):\n            pass\n        return rval\n\n    @property\n    def summary(self):\n        \"\"\"\n            Accessor returning a string describing and\n            summarizing the scan.\n\n            :return: string\n        \"\"\"\n        rval = ''\n        try:\n            rval = self._runstats['finished']['summary']\n        except(KeyError, TypeError):\n            pass\n\n        if len(rval) == 0:\n            rval = (\"Nmap ended at {0} ; {1} IP addresses ({2} hosts up)\"\n                    \" scanned in {3} seconds\".format(self.endtimestr,\n                                                     self.hosts_total,\n                                                     self.hosts_up,\n                                                     self.elapsed))\n        return rval\n\n    @property\n    def elapsed(self):\n        \"\"\"\n            Accessor returning the number of seconds the scan took\n\n            :return: float (0 >= or -1)\n        \"\"\"\n        rval = -1\n        try:\n            s_elapsed = self._runstats['finished']['elapsed']\n            rval = float(s_elapsed)\n        except (KeyError, TypeError, ValueError):\n            rval = -1\n        return rval\n\n    @property\n    def hosts_up(self):\n        \"\"\"\n            Accessor returning the numer of host detected\n            as 'up' during the scan.\n\n            :return: integer (0 >= or -1)\n        \"\"\"\n        rval = -1\n        try:\n            s_up = self._runstats['hosts']['up']\n            rval = int(s_up)\n        except (KeyError, TypeError, ValueError):\n            rval = -1\n        return rval\n\n    @property\n    def hosts_down(self):\n        \"\"\"\n            Accessor returning the numer of host detected\n            as 'down' during the scan.\n\n            :return: integer (0 >= or -1)\n        \"\"\"\n        rval = -1\n        try:\n            s_down = self._runstats['hosts']['down']\n            rval = int(s_down)\n        except (KeyError, TypeError, ValueError):\n            rval = -1\n        return rval\n\n    @property\n    def hosts_total(self):\n        \"\"\"\n            Accessor returning the number of hosts scanned in total.\n\n            :return: integer (0 >= or -1)\n        \"\"\"\n        rval = -1\n        try:\n            s_total = self._runstats['hosts']['total']\n            rval = int(s_total)\n        except (KeyError, TypeError, ValueError):\n            rval = -1\n        return rval\n\n    def get_raw_data(self):\n        \"\"\"\n            Returns a dict representing the NmapReport object.\n\n            :return: dict\n            :todo: deprecate. get rid of this uglyness.\n        \"\"\"\n        raw_data = {'_nmaprun': self._nmaprun,\n                    '_scaninfo': self._scaninfo,\n                    '_hosts': self._hosts,\n                    '_runstats': self._runstats}\n        return raw_data\n\n    def __set_raw_data(self, raw_data):\n        self._nmaprun = raw_data['_nmaprun']\n        self._scaninfo = raw_data['_scaninfo']\n        self._hosts = raw_data['_hosts']\n        self._runstats = raw_data['_runstats']\n\n    def is_consistent(self):\n        \"\"\"\n            Checks if the report is consistent and can be diffed().\n\n            This needs to be rewritten and removed: diff() should be generic.\n\n            :return: boolean\n        \"\"\"\n        rval = False\n        rdata = self.get_raw_data()\n        _consistent_keys = ['_nmaprun', '_scaninfo', '_hosts', '_runstats']\n        if(set(_consistent_keys) == set(rdata.keys()) and\n           len([dky for dky in rdata.keys() if rdata[dky] is not None]) == 4):\n            rval = True\n        return rval\n\n    def get_dict(self):\n        \"\"\"\n            Return a python dict representation of the NmapReport object.\n            This is used to diff() NmapReport objects via NmapDiff.\n\n            :return: dict\n        \"\"\"\n        rdict = dict([(\"{0}::{1}\".format(_host.__class__.__name__,\n                                         str(_host.id)),\n                     hash(_host)) for _host in self.hosts])\n        rdict.update({'commandline': self.commandline,\n                      'version': self.version,\n                      'scan_type': self.scan_type,\n                      'elapsed': self.elapsed,\n                      'hosts_up': self.hosts_up,\n                      'hosts_down': self.hosts_down,\n                      'hosts_total': self.hosts_total})\n        return rdict\n\n    @property\n    def id(self):\n        \"\"\"\n            Dummy id() defined for reports.\n        \"\"\"\n        return hash(1)\n\n    def __eq__(self, other):\n        \"\"\"\n            Compare eq NmapReport based on :\n\n                - create a diff obj and check the result\n                report are equal if added&changed&removed are empty\n\n            :return: boolean\n        \"\"\"\n        rval = False\n        if(self.__class__ == other.__class__ and self.id == other.id):\n            diffobj = self.diff(other)\n            rval = (len(diffobj.changed()) == 0 and\n                    len(diffobj.added()) == 0 and\n                    len(diffobj.removed()) == 0\n                    )\n        return rval\n\n    def __ne__(self, other):\n        \"\"\"\n            Compare ne NmapReport based on:\n\n                - create a diff obj and check the result\n                report are ne if added|changed|removed are not empty\n\n            :return: boolean\n        \"\"\"\n        rval = True\n        if(self.__class__ == other.__class__ and self.id == other.id):\n            diffobj = self.diff(other)\n            rval = (len(diffobj.changed()) != 0 or\n                    len(diffobj.added()) != 0 or\n                    len(diffobj.removed()) != 0\n                    )\n        return rval\n\n    def __repr__(self):\n        return \"{0}: started at {1} hosts up {2}/{3}\".format(\n               self.__class__.__name__,\n               self.started,\n               self.hosts_up,\n               self.hosts_total)\n"
  },
  {
    "path": "libnmap/objects/service.py",
    "content": "# -*- coding: utf-8 -*-\nfrom libnmap.diff import NmapDiff\nfrom libnmap.objects.os import CPE\n\n\nclass NmapService(object):\n    \"\"\"\n        NmapService represents a nmap scanned service. Its id() is comprised\n        of the protocol and the port.\n\n        Depending on the scanning options, some additional details might be\n        available or not. Like banner or extra datas from NSE (nmap scripts).\n    \"\"\"\n    def __init__(self, portid, protocol='tcp', state=None,\n                 service=None, owner=None, service_extras=None):\n        \"\"\"\n            Constructor\n\n            :param portid: port number\n            :type portid: string\n            :param protocol: protocol of port scanned (tcp, udp)\n            :type protocol: string\n            :param state: python dict describing the service status\n            :type state: python dict\n            :param service: python dict describing the service name and banner\n            :type service: python dict\n            :param service_extras: additional info about the tested service\n            like scripts' data\n        \"\"\"\n        try:\n            self._portid = int(portid or -1)\n        except (ValueError, TypeError):\n            raise\n        if self._portid < 0 or self._portid > 65535:\n            raise ValueError\n\n        self._protocol = protocol\n        # self._taskid = taskid\n        # self._address = address\n        self._state = state if state is not None else {}\n        self._service = service if service is not None else {}\n\n        self._cpelist = []\n        if 'cpelist' in self._service:\n            for _cpe in self._service['cpelist']:\n                _cpeobj = CPE(_cpe)\n                self._cpelist.append(_cpeobj)\n\n        self._owner = ''\n        if owner is not None and 'name' in owner:\n            self._owner = owner['name']\n\n        self._reason = ''\n        self._reason_ip = ''\n        self._reason_ttl = ''\n        self._servicefp = ''\n        self._tunnel = ''\n\n        if 'reason' in self._state:\n            self._reason = self._state['reason']\n        if 'reason_ttl' in self._state:\n            self._reason_ttl = self._state['reason_ttl']\n        if 'reason_ip' in self._state:\n            self._reason_ip = self._state['reason_ip']\n\n        if 'servicefp' in self._service:\n            self._servicefp = self._service['servicefp']\n        if 'tunnel' in self._service:\n            self._tunnel = self._service['tunnel']\n\n        self._service_extras = []\n        if service_extras is not None:\n            self._service_extras = service_extras\n\n    def __eq__(self, other):\n        \"\"\"\n            Compares two NmapService objects to see if they are the same or\n            if one of them changed.\n\n            :param other: NmapService\n\n            :return: boolean\n        \"\"\"\n        rval = False\n        if(self.__class__ == other.__class__ and self.id == other.id):\n            rval = (self.changed(other) == 0)\n        return rval\n\n    def __ne__(self, other):\n        \"\"\"\n            Compares two NmapService objects to see if they are different\n            if one of them changed.\n\n            :param other: NmapService\n\n            :return: boolean\n        \"\"\"\n        rval = True\n        if(self.__class__ == other.__class__ and self.id == other.id):\n            rval = (self.changed(other) > 0)\n        return rval\n\n    def __repr__(self):\n        return \"{0}: [{1} {2}/{3} {4} ({5})]\".format(self.__class__.__name__,\n                                                     self.state,\n                                                     str(self.port),\n                                                     self.protocol,\n                                                     self.service,\n                                                     self.banner)\n\n    def __hash__(self):\n        return (hash(self.port) ^ hash(self.protocol) ^ hash(self.state) ^\n                hash(self.reason) ^ hash(self.service) ^ hash(self.banner))\n\n    def changed(self, other):\n        \"\"\"\n            Checks if a NmapService is different from another.\n\n            :param other: NmapService\n\n            :return: boolean\n        \"\"\"\n        return len(self.diff(other).changed())\n\n    def save(self, backend):\n        if backend is not None:\n            _id = backend.insert(self)\n        else:\n            raise RuntimeError\n        return _id\n\n    @property\n    def port(self):\n        \"\"\"\n            Accessor for port.\n\n            :return: integer or -1\n        \"\"\"\n        return self._portid\n\n    @property\n    def protocol(self):\n        \"\"\"\n            Accessor for protocol\n\n            :return: string\n        \"\"\"\n        return self._protocol\n\n    @property\n    def state(self):\n        \"\"\"\n            Accessor for service's state (open, filtered, closed,...)\n\n            :return: string\n        \"\"\"\n        return self._state['state'] if 'state' in self._state else None\n\n    @property\n    def reason(self):\n        \"\"\"\n            Accessor for service's state reason (syn-ack, filtered,...)\n\n            :return: string or empty if not applicable\n        \"\"\"\n        return self._reason\n\n    @property\n    def reason_ip(self):\n        \"\"\"\n            Accessor for service's state reason ip\n\n            :return: string or empty if not applicable\n        \"\"\"\n        return self._reason_ip\n\n    @property\n    def reason_ttl(self):\n        \"\"\"\n            Accessor for service's state reason ttl\n\n            :return: string or empty if not applicable\n        \"\"\"\n        return self._reason_ttl\n\n    @property\n    def service(self):\n        \"\"\"\n            Accessor for service name.\n\n            :return: string or empty\n        \"\"\"\n        return self._service['name'] if 'name' in self._service else ''\n\n    @property\n    def service_dict(self):\n        \"\"\"\n            Accessor for service dictionary.\n\n            :return: dict or None\n        \"\"\"\n        return self._service\n\n    def open(self):\n        \"\"\"\n            Tells if the port was open or not\n\n            :return: boolean\n        \"\"\"\n        return 'state' in self._state and self._state['state'] == 'open'\n\n    @property\n    def product(self):\n        if 'product' in self._service.keys():\n            return self._service['product'].rstrip()\n        else:\n            return None\n\n    @property\n    def product_version(self):\n        if 'version' in self._service.keys():\n            return self._service['version'].rstrip()\n        else:\n            return None\n\n    @property\n    def product_extrainfo(self):\n        if 'extrainfo' in self._service.keys():\n            return self._service['extrainfo'].rstrip()\n        else:\n            return None\n\n    @property\n    def owner(self):\n        \"\"\"\n            Accessor for service owner if available\n        \"\"\"\n        return self._owner\n\n    @property\n    def banner(self):\n        \"\"\"\n            Accessor for the service's banner. Only available\n            if the nmap option -sV or similar was used.\n\n            :return: string\n        \"\"\"\n        notrelevant = ['name', 'method', 'conf', 'cpelist',\n                       'servicefp', 'tunnel']\n        relevant = ['product', 'version', 'extrainfo']\n        b = ''\n        skeys = self._service.keys()\n        if 'method' in self._service and self._service['method'] == \"probed\":\n            for relk in relevant:\n                if relk in skeys:\n                    b += '{0}: {1} '.format(relk, self._service[relk])\n            for mkey in skeys:\n                if mkey not in notrelevant and mkey not in relevant:\n                    b += '{0}: {1} '.format(mkey, self._service[mkey])\n        return b.rstrip()\n\n    @property\n    def cpelist(self):\n        \"\"\"\n            Accessor for list of CPE for this particular service\n        \"\"\"\n        return self._cpelist\n\n    @property\n    def scripts_results(self):\n        \"\"\"\n            Gives a python list of the nse scripts results.\n\n            The dict key is the name (id) of the nse script and\n            the value is the output of the script.\n\n            :return: dict\n        \"\"\"\n        scripts_dict = None\n        try:\n            scripts_dict = self._service_extras['scripts']\n        except (KeyError, TypeError):\n            pass\n        return scripts_dict\n\n    @property\n    def servicefp(self):\n        \"\"\"\n            Accessor for the service's fingerprint\n            if the nmap option -sV or -A is used\n\n            :return: string if available\n        \"\"\"\n        return self._servicefp\n\n    @property\n    def tunnel(self):\n        \"\"\"\n            Accessor for the service's tunnel type\n            if applicable and available from scan\n            results\n\n            :return: string if available\n        \"\"\"\n        return self._tunnel\n\n    @property\n    def id(self):\n        \"\"\"\n            Accessor for the id() of the NmapService.\n\n            This is used for diff()ing NmapService object via NmapDiff.\n\n            :return: tuple\n        \"\"\"\n        return \"{0}.{1}\".format(self.protocol, self.port)\n\n    def get_dict(self):\n        \"\"\"\n            Return a python dict representation of the NmapService object.\n\n            This is used to diff() NmapService objects via NmapDiff.\n\n            :return: dict\n        \"\"\"\n        return ({'id': str(self.id), 'port': str(self.port),\n                 'protocol': self.protocol, 'banner': self.banner,\n                 'service': self.service, 'state': self.state,\n                 'reason': self.reason})\n\n    def diff(self, other):\n        \"\"\"\n            Calls NmapDiff to check the difference between self and\n            another NmapService object.\n\n            Will return a NmapDiff object.\n\n            This objects return python set() of keys describing the elements\n            which have changed, were added, removed or kept unchanged.\n\n            :return: NmapDiff object\n        \"\"\"\n        return NmapDiff(self, other)\n"
  },
  {
    "path": "libnmap/parser.py",
    "content": "# -*- coding: utf-8 -*-\n\n\ntry:\n    import xml.etree.cElementTree as ET\nexcept ImportError:\n    import xml.etree.ElementTree as ET\nfrom libnmap.objects import NmapHost, NmapService, NmapReport\n\n\nclass NmapParser(object):\n    @classmethod\n    def parse(cls, nmap_data=None, data_type='XML', incomplete=False):\n        \"\"\"\n            Generic class method of NmapParser class.\n\n            The data to be parsed does not need to be a complete nmap\n            scan report. You can possibly give <hosts>...</hosts>\n            or <port> XML tags.\n\n            :param nmap_data: any portion of nmap scan result. \\\n            nmap_data should always be a string representing a part \\\n            or a complete nmap scan report.\n            :type nmap_data: string\n\n            :param data_type: specifies the type of data to be parsed.\n            :type data_type: string (\"XML\"|\"JSON\"|\"YAML\").\n\n            :param incomplete: enable you to parse interrupted nmap scans \\\n            and/or incomplete nmap xml blocks by adding a </nmaprun> at \\\n            the end of the scan.\n            :type incomplete: boolean\n\n            As of today, only XML parsing is supported.\n\n            :return: NmapObject (NmapHost, NmapService or NmapReport)\n        \"\"\"\n\n        nmapobj = None\n        if data_type == \"XML\":\n            nmapobj = cls._parse_xml(nmap_data, incomplete)\n        else:\n            raise NmapParserException(\"Unknown data type provided. \"\n                                      \"Please check documentation for \"\n                                      \"supported data types.\")\n        return nmapobj\n\n    @classmethod\n    def _parse_xml(cls, nmap_data=None, incomplete=False):\n        \"\"\"\n            Protected class method used to process a specific data type.\n            In this case: XML. This method is called by cls.parse class\n            method and receives nmap scan results data (in XML).\n\n            :param nmap_data: any portion of nmap scan result can be given \\\n            as argument. nmap_data should always be a string representing \\\n            a part or a complete nmap scan report.\n            :type nmap_data: string\n\n            This method checks which portion of a nmap scan is given \\\n            as argument.\n            It could be:\n\n                1. a full nmap scan report;\n                2. a scanned host: <host> tag in a nmap scan report\n                3. a scanned service: <port> tag\n                4. a list of hosts: <hosts/> tag (TODO)\n                5. a list of ports: <ports/> tag\n\n            :param incomplete: enable you to parse interrupted nmap scans \\\n            and/or incomplete nmap xml blocks by adding a </nmaprun> at \\\n            the end of the scan.\n            :type incomplete: boolean\n\n            :return: NmapObject (NmapHost, NmapService or NmapReport) \\\n                    or a list of NmapObject\n        \"\"\"\n\n        if not nmap_data:\n            raise NmapParserException(\"No report data to parse: please \"\n                                      \"provide a valid XML nmap report\")\n        elif not isinstance(nmap_data, str):\n            raise NmapParserException(\"wrong nmap_data type given as \"\n                                      \"argument: cannot parse data\")\n\n        if incomplete is True:\n            nmap_data += \"</nmaprun>\"\n\n        try:\n            root = ET.fromstring(nmap_data)\n        except:\n            raise NmapParserException(\"Wrong XML structure: cannot parse data\")\n\n        nmapobj = None\n        if root.tag == 'nmaprun':\n            nmapobj = cls._parse_xml_report(root)\n        elif root.tag == 'host':\n            nmapobj = cls._parse_xml_host(root)\n        elif root.tag == 'ports':\n            nmapobj = cls._parse_xml_ports(root)\n        elif root.tag == 'port':\n            nmapobj = cls._parse_xml_port(root)\n        else:\n            raise NmapParserException(\"Unpexpected data structure for XML \"\n                                      \"root node\")\n        return nmapobj\n\n    @classmethod\n    def _parse_xml_report(cls, root=None):\n        \"\"\"\n            This method parses out a full nmap scan report from its XML root\n            node: <nmaprun>.\n\n            :param root: Element from xml.ElementTree (top of XML the document)\n            :type root: Element\n\n            :return: NmapReport object\n        \"\"\"\n\n        nmap_scan = {'_nmaprun': {}, '_scaninfo': {},\n                     '_hosts': [], '_runstats': {}}\n\n        if root is None:\n            raise NmapParserException(\"No root node provided to parse XML \"\n                                      \"report\")\n\n        nmap_scan['_nmaprun'] = cls.__format_attributes(root)\n        for el in root:\n            if el.tag == 'scaninfo':\n                nmap_scan['_scaninfo'] = cls.__parse_scaninfo(el)\n            elif el.tag == 'host':\n                nmap_scan['_hosts'].append(cls._parse_xml_host(el))\n            elif el.tag == 'runstats':\n                nmap_scan['_runstats'] = cls.__parse_runstats(el)\n            # else:\n            #    print \"struct pparse unknown attr: {0} value: {1}\".format(\n            #        el.tag,\n            #        el.get(el.tag))\n        return NmapReport(nmap_scan)\n\n    @classmethod\n    def parse_fromstring(cls, nmap_data, data_type=\"XML\", incomplete=False):\n        \"\"\"\n            Call generic cls.parse() method and ensure that a string is \\\n            passed on as argument. If not, an exception is raised.\n\n            :param nmap_data: Same as for parse(), any portion of nmap scan. \\\n            Reports could be passed as argument. Data type _must_ be a string.\n\n            :type nmap_data: string\n\n            :param data_type: Specifies the type of data passed on as argument.\n\n            :param incomplete: enable you to parse interrupted nmap scans \\\n            and/or incomplete nmap xml blocks by adding a </nmaprun> at \\\n            the end of the scan.\n            :type incomplete: boolean\n\n            :return: NmapObject\n        \"\"\"\n\n        if not isinstance(nmap_data, str):\n            raise NmapParserException(\"bad argument type for \"\n                                      \"xarse_fromstring(): should be a string\")\n        return cls.parse(nmap_data, data_type, incomplete)\n\n    @classmethod\n    def parse_fromfile(cls, nmap_report_path,\n                       data_type=\"XML\",\n                       incomplete=False):\n        \"\"\"\n            Call generic cls.parse() method and ensure that a correct file \\\n            path is given as argument. If not, an exception is raised.\n\n            :param nmap_data: Same as for parse(). \\\n            Any portion of nmap scan reports could be passed as argument. \\\n            Data type _must be a valid path to a file containing \\\n            nmap scan results.\n\n            :param data_type: Specifies the type of serialization in the file.\n\n            :param incomplete: enable you to parse interrupted nmap scans \\\n            and/or incomplete nmap xml blocks by adding a </nmaprun> at \\\n            the end of the scan.\n            :type incomplete: boolean\n\n            :return: NmapObject\n        \"\"\"\n\n        try:\n            with open(nmap_report_path, 'r') as fileobj:\n                fdata = fileobj.read()\n                rval = cls.parse(fdata, data_type, incomplete)\n        except IOError:\n            raise\n        return rval\n\n    @classmethod\n    def parse_fromdict(cls, rdict):\n        \"\"\"\n            Strange method which transforms a python dict \\\n            representation of a NmapReport and turns it into an \\\n            NmapReport object. \\\n            Needs to be reviewed and possibly removed.\n\n            :param rdict: python dict representation of an NmapReport\n            :type rdict: dict\n\n            :return: NmapReport\n        \"\"\"\n\n        nreport = {}\n        if list(rdict.keys())[0] == '__NmapReport__':\n            r = rdict['__NmapReport__']\n            nreport['_runstats'] = r['_runstats']\n            nreport['_scaninfo'] = r['_scaninfo']\n            nreport['_nmaprun'] = r['_nmaprun']\n            hlist = []\n            for h in r['_hosts']:\n                slist = []\n                for s in h['__NmapHost__']['_services']:\n                    cname = '__NmapService__'\n                    slist.append(NmapService(portid=s[cname]['_portid'],\n                                             protocol=s[cname]['_protocol'],\n                                             state=s[cname]['_state'],\n                                             owner=s[cname]['_owner'],\n                                             service=s[cname]['_service']))\n\n                nh = NmapHost(starttime=h['__NmapHost__']['_starttime'],\n                              endtime=h['__NmapHost__']['_endtime'],\n                              address=h['__NmapHost__']['_address'],\n                              status=h['__NmapHost__']['_status'],\n                              hostnames=h['__NmapHost__']['_hostnames'],\n                              extras=h['__NmapHost__']['_extras'],\n                              services=slist)\n                hlist.append(nh)\n            nreport['_hosts'] = hlist\n            nmapobj = NmapReport(nreport)\n        return nmapobj\n\n    @classmethod\n    def __parse_scaninfo(cls, scaninfo_data):\n        \"\"\"\n            Private method parsing a portion of a nmap scan result.\n            Receives a <scaninfo> XML tag.\n\n            :param scaninfo_data: <scaninfo> XML tag from a nmap scan\n            :type scaninfo_data: xml.ElementTree.Element or a string\n\n            :return: python dict representing the XML scaninfo tag\n        \"\"\"\n\n        xelement = cls.__format_element(scaninfo_data)\n        return cls.__format_attributes(xelement)\n\n    @classmethod\n    def _parse_xml_host(cls, scanhost_data):\n        \"\"\"\n            Protected method parsing a portion of a nmap scan result.\n            Receives a <host> XML tag representing a scanned host with\n            its services.\n\n            :param scaninfo_data: <host> XML tag from a nmap scan\n            :type scaninfo_data: xml.ElementTree.Element or a string\n\n            :return: NmapHost object\n        \"\"\"\n\n        xelement = cls.__format_element(scanhost_data)\n        _host_header = cls.__format_attributes(xelement)\n        _hostnames = []\n        _services = []\n        _status = {}\n        _addresses = []\n        _host_extras = {}\n        extra_tags = ['uptime', 'distance', 'tcpsequence',\n                      'ipidsequence', 'tcptssequence', 'times']\n        for xh in xelement:\n            if xh.tag == 'hostnames':\n                for hostname in cls.__parse_hostnames(xh):\n                    _hostnames.append(hostname)\n            elif xh.tag == 'ports':\n                ports_dict = cls._parse_xml_ports(xh)\n                for port in ports_dict['ports']:\n                    _services.append(port)\n                _host_extras['extraports'] = ports_dict['extraports']\n            elif xh.tag == 'status':\n                _status = cls.__format_attributes(xh)\n            elif xh.tag == 'address':\n                _addresses.append(cls.__format_attributes(xh))\n            elif xh.tag == 'os':\n                _os_extra = cls.__parse_os_fingerprint(xh)\n                _host_extras.update({'os': _os_extra})\n            elif xh.tag == 'hostscript':\n                _host_scripts = cls.__parse_host_scripts(xh)\n                _host_extras.update({'hostscript': _host_scripts})\n            elif xh.tag in extra_tags:\n                _host_extras[xh.tag] = cls.__format_attributes(xh)\n            # else:\n            #    print \"struct host unknown attr: %s value: %s\" %\n            #           (h.tag, h.get(h.tag))\n        _stime = ''\n        _etime = ''\n        if 'starttime' in _host_header:\n            _stime = _host_header['starttime']\n        if 'endtime' in _host_header:\n            _etime = _host_header['endtime']\n        nhost = NmapHost(_stime,\n                         _etime,\n                         _addresses,\n                         _status,\n                         _hostnames,\n                         _services,\n                         _host_extras)\n        return nhost\n\n    @classmethod\n    def __parse_hostnames(cls, scanhostnames_data):\n        \"\"\"\n            Private method parsing the hostnames list within a <host> XML tag.\n\n            :param scanhostnames_data: <hostnames> XML tag from a nmap scan\n            :type scanhostnames_data: xml.ElementTree.Element or a string\n\n            :return: list of hostnames\n        \"\"\"\n\n        xelement = cls.__format_element(scanhostnames_data)\n        hostnames = []\n        for hname in xelement:\n            if hname.tag == 'hostname':\n                hostnames.append(hname.get('name'))\n        return hostnames\n\n    @classmethod\n    def _parse_xml_ports(cls, scanports_data):\n        \"\"\"\n            Protected method parsing the list of scanned services from\n            a targeted host. This protected method cannot be called directly\n            with a string. A <ports/> tag can be directly passed to parse()\n            and the below method will be called and return a list of nmap\n            scanned services.\n\n            :param scanports_data: <ports> XML tag from a nmap scan\n            :type scanports_data: xml.ElementTree.Element or a string\n\n            :return: list of NmapService\n        \"\"\"\n\n        xelement = cls.__format_element(scanports_data)\n\n        rdict = {'ports': [], 'extraports': None}\n        for xservice in xelement:\n            if xservice.tag == 'port':\n                nport = cls._parse_xml_port(xservice)\n                rdict['ports'].append(nport)\n            elif xservice.tag == 'extraports':\n                extraports = cls.__parse_extraports(xservice)\n                rdict['extraports'] = extraports\n            # else:\n            #    print \"struct port unknown attr: %s value: %s\" %\n            #           (h.tag, h.get(h.tag))\n        return rdict\n\n    @classmethod\n    def _parse_xml_port(cls, scanport_data):\n        \"\"\"\n            Protected method parsing a scanned service from a targeted host.\n            This protected method cannot be called directly.\n            A <port/> tag can be directly passed to parse() and the below\n            method will be called and return a NmapService object\n            representing the state of the service.\n\n            :param scanport_data: <port> XML tag from a nmap scan\n            :type scanport_data: xml.ElementTree.Element or a string\n\n            :return: NmapService\n        \"\"\"\n\n        xelement = cls.__format_element(scanport_data)\n\n        _port = cls.__format_attributes(xelement)\n        _portid = _port['portid'] if 'portid' in _port else None\n        _protocol = _port['protocol'] if 'protocol' in _port else None\n\n        _state = None\n        _service = None\n        _owner = None\n        _service_scripts = []\n        _service_extras = {}\n        for xport in xelement:\n            if xport.tag == 'state':\n                _state = cls.__format_attributes(xport)\n            elif xport.tag == 'service':\n                _service = cls.__parse_service(xport)\n            elif xport.tag == 'owner':\n                _owner = cls.__format_attributes(xport)\n            elif xport.tag == 'script':\n                _script_dict = cls.__parse_script(xport)\n                _service_scripts.append(_script_dict)\n        _service_extras['scripts'] = _service_scripts\n\n        if(_portid is None or _protocol is None or _state is None):\n            raise NmapParserException(\"XML <port> tag is incomplete. One \"\n                                      \"of the following tags is missing: \"\n                                      \"portid, protocol or state or tag.\")\n\n        nport = NmapService(_portid,\n                            _protocol,\n                            _state,\n                            _service,\n                            _owner,\n                            _service_extras)\n        return nport\n\n    @classmethod\n    def __parse_service(cls, xserv):\n        \"\"\"\n            Parse <service> tag to manage CPE object\n        \"\"\"\n        _service = cls.__format_attributes(xserv)\n        _cpelist = []\n        for _servnode in xserv:\n            if _servnode.tag == 'cpe':\n                _cpe_string = _servnode.text\n                _cpelist.append(_cpe_string)\n        _service['cpelist'] = _cpelist\n        return _service\n\n    @classmethod\n    def __parse_extraports(cls, extraports_data):\n        \"\"\"\n            Private method parsing the data from extra scanned ports.\n            X extraports were in state \"closed\" server returned \"conn-refused\"\n            tag: <extraports>\n\n            :param extraports_data: XML data for extraports\n            :type extraports_data: xml.ElementTree.Element or a string\n\n            :return: python dict with following keys: state, count, reason\n        \"\"\"\n        rdict = {'state': '', 'count': '', 'reasons': []}\n        xelement = cls.__format_element(extraports_data)\n        extraports_dict = cls.__format_attributes(xelement)\n\n        if 'state' in extraports_dict:\n            rdict['state'] = extraports_dict\n        if 'count' in extraports_dict:\n            rdict['count'] = extraports_dict\n        for xelt in xelement:\n            if xelt.tag == 'extrareasons':\n                extrareasons_dict = cls.__format_attributes(xelt)\n                rdict['reasons'].append(extrareasons_dict)\n        return rdict\n\n    @classmethod\n    def __parse_script(cls, script_data):\n        \"\"\"\n            Private method parsing the data from NSE scripts output\n\n            :param script_data: portion of XML describing the results of the\n            script data\n            :type script_data: xml.ElementTree.Element or a string\n\n            :return: python dict holding scripts output\n        \"\"\"\n        _script_dict = cls.__format_attributes(script_data)\n\n        _elt_dict = {}\n        for script_elem in script_data:\n            if script_elem.tag == 'elem':\n                _elt_dict.update({script_elem.get('key'): script_elem.text})\n            elif script_elem.tag == 'table':\n                tdict = {}\n                for telem in script_elem:\n                    tdict[telem.get('key')] = telem.text\n                _elt_dict[script_elem.get('key')] = tdict\n        _script_dict['elements'] = _elt_dict\n        return _script_dict\n\n    @classmethod\n    def __parse_host_scripts(cls, scripts_data):\n        \"\"\"\n            Private method parsing the data from scripts affecting\n            the target host.\n            Contents of <hostscript> is returned as a list of dict.\n\n            :param scripts_data: portion of XML describing the results of the\n            scripts data\n            :type scripts_data: xml.ElementTree.Element or a string\n\n            :return: python list holding scripts output in a dict\n        \"\"\"\n        _host_scripts = []\n        for xscript in scripts_data:\n            if xscript.tag == 'script':\n                _script_dict = cls.__parse_script(xscript)\n            _host_scripts.append(_script_dict)\n        return _host_scripts\n\n    @classmethod\n    def __parse_os_fingerprint(cls, os_data):\n        \"\"\"\n            Private method parsing the data from an OS fingerprint (-O).\n            Contents of <os> is returned as a dict.\n\n            :param os_data: portion of XML describing the results of the\n            os fingerprinting attempt\n            :type os_data: xml.ElementTree.Element or a string\n\n            :return: python dict representing the XML os tag\n        \"\"\"\n        rdict = {}\n        xelement = cls.__format_element(os_data)\n\n        os_class_probability = []\n        os_match_probability = []\n        os_ports_used = []\n        os_fingerprints = []\n        for xos in xelement:\n            # for nmap xml version < 1.04, osclass is not\n            # embedded in osmatch\n            if xos.tag == 'osclass':\n                os_class_proba = cls.__parse_osclass(xos)\n                os_class_probability.append(os_class_proba)\n            elif xos.tag == 'osmatch':\n                os_match_proba = cls.__parse_osmatch(xos)\n                os_match_probability.append(os_match_proba)\n            elif xos.tag == 'portused':\n                os_portused = cls.__format_attributes(xos)\n                os_ports_used.append(os_portused)\n            elif xos.tag == 'osfingerprint':\n                os_fp_dict = cls.__format_attributes(xos)\n                os_fingerprints.append(os_fp_dict)\n\n        rdict['osmatches'] = os_match_probability\n        rdict['osclasses'] = os_class_probability\n        rdict['ports_used'] = os_ports_used\n        rdict['osfingerprints'] = os_fingerprints\n\n        return rdict\n\n    @classmethod\n    def __parse_osmatch(cls, osmatch_data):\n        \"\"\"\n            This methods parses osmatch data and returns a dict. Depending\n            on the nmap xml version, osmatch could contain an osclass\n            dict.\n\n            :param osmatch_data: <osmatch> XML tag from a nmap scan\n            :type osmatch_data: xml.ElementTree.Element or a string\n\n            :return: python dict representing the XML osmatch tag\n        \"\"\"\n        rdict = {}\n        xelement = cls.__format_element(osmatch_data)\n        rdict['osmatch'] = cls.__format_attributes(xelement)\n        rdict['osclasses'] = []\n        for xmltag in xelement:\n            if xmltag.tag == 'osclass':\n                _osclass_dict = cls.__parse_osclass(xmltag)\n                rdict['osclasses'].append(_osclass_dict)\n            else:\n                exmsg = \"Unexcepted node in <osmatch>: {0}\".format(xmltag.tag)\n                raise NmapParserException(exmsg)\n        return rdict\n\n    @classmethod\n    def __parse_osclass(cls, osclass_data):\n        \"\"\"\n            This methods parses osclass data and returns a dict. Depending\n            on the nmap xml version, osclass could contain a cpe\n            dict.\n\n            :param osclass_data: <osclass> XML tag from a nmap scan\n            :type osclass_data: xml.ElementTree.Element or a string\n\n            :return: python dict representing the XML osclass tag\n        \"\"\"\n        rdict = {}\n        xelement = cls.__format_element(osclass_data)\n        rdict['osclass'] = cls.__format_attributes(xelement)\n        rdict['cpe'] = []\n        for xmltag in xelement:\n            if xmltag.tag == 'cpe':\n                _cpe_string = xmltag.text\n                rdict['cpe'].append(_cpe_string)\n            else:\n                exmsg = \"Unexcepted node in <osclass>: {0}\".format(xmltag.tag)\n                raise NmapParserException(exmsg)\n        return rdict\n\n    @classmethod\n    def __parse_runstats(cls, scanrunstats_data):\n        \"\"\"\n            Private method parsing a portion of a nmap scan result.\n            Receives a <runstats> XML tag.\n\n            :param scanrunstats_data: <runstats> XML tag from a nmap scan\n            :type scanrunstats_data: xml.ElementTree.Element or a string\n\n            :return: python dict representing the XML runstats tag\n        \"\"\"\n\n        xelement = cls.__format_element(scanrunstats_data)\n\n        rdict = {}\n        for xmltag in xelement:\n            if xmltag.tag in ['finished', 'hosts']:\n                rdict[xmltag.tag] = cls.__format_attributes(xmltag)\n            else:\n                exmsg = \"Unexcepted node in <runstats>: {0}\".format(xmltag.tag)\n                raise NmapParserException(exmsg)\n\n        return rdict\n\n    @staticmethod\n    def __format_element(elt_data):\n        \"\"\"\n            Private method which ensures that a XML portion to be parsed is\n            of type xml.etree.ElementTree.Element.\n            If elt_data is a string, then it is converted to an\n            XML Element type.\n\n            :param elt_data: XML Element to be parsed or string\n            to be converted to a XML Element\n\n            :return: Element\n        \"\"\"\n        if isinstance(elt_data, str):\n            try:\n                xelement = ET.fromstring(elt_data)\n            except:\n                raise NmapParserException(\"Error while trying \"\n                                          \"to instanciate XML Element from \"\n                                          \"string {0}\".format(elt_data))\n        elif ET.iselement(elt_data):\n            xelement = elt_data\n        else:\n            raise NmapParserException(\"Error while trying to parse supplied \"\n                                      \"data: unsupported format\")\n        return xelement\n\n    @staticmethod\n    def __format_attributes(elt_data):\n        \"\"\"\n            Private method which converts a single XML tag to a python dict.\n            It also checks that the elt_data given as argument is of type\n            xml.etree.ElementTree.Element\n\n            :param elt_data: XML Element to be parsed or string\n            to be converted to a XML Element\n\n            :return: Element\n        \"\"\"\n\n        rval = {}\n        if not ET.iselement(elt_data):\n            raise NmapParserException(\"Error while trying to parse supplied \"\n                                      \"data attributes: format is not XML or \"\n                                      \"XML tag is empty\")\n        try:\n            for dkey in elt_data.keys():\n                rval[dkey] = elt_data.get(dkey)\n                if rval[dkey] is None:\n                    raise NmapParserException(\"Error while trying to build-up \"\n                                              \"element attributes: empty \"\n                                              \"attribute {0}\".format(dkey))\n        except:\n            raise\n        return rval\n\n\nclass NmapParserException(Exception):\n    def __init__(self, msg):\n        self.msg = msg\n\n    def __str__(self):\n        return self.msg\n"
  },
  {
    "path": "libnmap/plugins/__init__.py",
    "content": ""
  },
  {
    "path": "libnmap/plugins/backend_host.py",
    "content": "#!/usr/bin/env python\nfrom sqlalchemy import create_engine\nfrom sqlalchemy.schema import Column\nfrom sqlalchemy.types import Integer, DateTime, LargeBinary, Text, String, Boolean\nfrom sqlalchemy.ext.declarative import declarative_base\nfrom sqlalchemy.orm import sessionmaker\n\nfrom libnmap.plugins.backendplugin import NmapBackendPlugin\nfrom libnmap.reportjson import ReportEncoder, ReportDecoder\n\nimport json\nfrom datetime import datetime\n\nBase = declarative_base()\n\nclass NmapSqlPlugin(NmapBackendPlugin):\n    \"\"\"\n        This class handle the persistence of NmapRepport object in SQL backend\n        Implementation is made using sqlalchemy(0.8.1)\n        usage :\n\n        #get a nmapReport object\n        from libnmap.parser import NmapParser\n        from libnmap.reportjson import ReportDecoder, ReportEncoder\n        import json\n        nmap_report_obj = NmapParser.parse_fromfile(\n               '/home/vagrant/python-nmap-lib/libnmap/test/files/1_hosts.xml')\n\n         #get a backend with in memory sqlite\n         from libnmap.plugins.backendpluginFactory import BackendPluginFactory\n         mybackend_mem = BackendPluginFactory.create(plugin_name='sql',\n                                                     url='sqlite://',\n                                                     echo=True)\n\n         mybackend_mysql = BackendPluginFactory.create(plugin_name='sql',\n                            url='mysql+mysqldb://scott:tiger@localhost/foo',\n                            echo=True)\n         mybackend = BackendPluginFactory.create(plugin_name='sql',\n                                        url='sqlite:////tmp/reportdb.sql',\n                                        echo=True)\n         #lets save!!\n         nmap_report_obj.save(mybackend)\n         mybackend.getall()\n         mybackend.get(1)\n    \"\"\"\n    class Reports(Base):\n        \"\"\"\n            Embeded class for ORM map NmapReport to a\n            simple three column table\n        \"\"\"\n        __tablename__ = 'result_ip'\n\n        id = Column('id', Integer, primary_key=True)\n        taskid = Column('taskid', Integer)\n        inserted = Column('inserted', DateTime(), default='now')\n        domain = Column('domain', String(256))\n        address = Column('address', String(32))\n        is_up = Column('is_up', Boolean)\n        os = Column('os', String(256))\n    \n        def __init__(self, obj_NmapReport):\n            self.inserted = datetime.fromtimestamp(int(obj_NmapReport.endtime))\n            self.taskid = obj_NmapReport.taskid\n            self.is_up = obj_NmapReport.is_up()\n\n            if len(obj_NmapReport.hostnames) > 0:\n                self.domain = obj_NmapReport.hostnames[0]\n            else:\n                self.domain = None\n\n            self.address = obj_NmapReport.address\n            \n            if len(obj_NmapReport.os.osmatch()) > 0:\n                self.os = obj_NmapReport.os.osmatch()[0]\n            else:\n                self.os = None\n\n        def decode(self):\n            json_decoded = self.report_json.decode('utf-8')\n            nmap_report_obj = json.loads(json_decoded,\n                                         cls=ReportDecoder)\n            return nmap_report_obj\n\n    def __init__(self, **kwargs):\n        \"\"\"\n            constructor receive a **kwargs as the **kwargs in the sqlalchemy\n            create_engine() method (see sqlalchemy docs)\n            You must add to this **kwargs an 'url' key with the url to your\n            database\n            This constructor will :\n            - create all the necessary obj to discuss with the DB\n            - create all the mapping(ORM)\n\n            todo : suport the : sqlalchemy.engine_from_config\n\n            :param **kwargs:\n            :raises: ValueError if no url is given,\n                    all exception sqlalchemy can throw\n            ie sqlite in memory url='sqlite://' echo=True\n            ie sqlite file on hd url='sqlite:////tmp/reportdb.sql' echo=True\n            ie mysql url='mysql+mysqldb://scott:tiger@localhost/foo'\n        \"\"\"\n        NmapBackendPlugin.__init__(self)\n        self.engine = None\n        self.url = None\n        self.Session = sessionmaker()\n\n        if 'url' not in kwargs:\n            raise ValueError\n        self.url = kwargs['url']\n        del kwargs['url']\n        try:\n            self.engine = create_engine(self.url, **kwargs)\n            Base.metadata.create_all(bind=self.engine, checkfirst=True)\n            self.Session.configure(bind=self.engine)\n        except:\n            raise\n\n    def insert(self, nmap_report):\n        \"\"\"\n           insert NmapReport in the backend\n\n           :param NmapReport:\n\n           :returns: the ident of the object in the backend for future usage \\\n           or None\n        \"\"\"\n        sess = self.Session()\n        report = NmapSqlPlugin.Reports(nmap_report)\n        sess.add(report)\n        sess.commit()\n        reportid = report.id\n        sess.close()\n        return reportid if reportid else None\n\n    def get(self, report_id=None):\n        \"\"\"\n            retreive a NmapReport from the backend\n\n            :param id: str\n\n            :returns: NmapReport\n        \"\"\"\n        if report_id is None:\n            raise ValueError\n        sess = self.Session()\n        our_report = (\n            sess.query(NmapSqlPlugin.Reports).filter_by(id=report_id).first())\n        sess.close()\n        return our_report.decode() if our_report else None\n\n    def getall(self):\n        \"\"\"\n            :param filter: Nice to have implement a filter capability\n\n            :returns: collection of tuple (id,NmapReport)\n        \"\"\"\n        sess = self.Session()\n        nmapreportList = []\n        for report in (\n                sess.query(NmapSqlPlugin.Reports).\n                order_by(NmapSqlPlugin.Reports.inserted)):\n            nmapreportList.append((report.id, report.decode()))\n        sess.close()\n        return nmapreportList\n\n    def delete(self, report_id=None):\n        \"\"\"\n            Remove a report from the backend\n\n            :param id: str\n\n            :returns: The number of rows deleted\n        \"\"\"\n        if report_id is None:\n            raise ValueError\n        nb_line = 0\n        sess = self.Session()\n        nb_line = sess.query(NmapSqlPlugin.Reports).\\\n            filter_by(id=report_id).delete()\n        sess.commit()\n        sess.close()\n        return nb_line\n"
  },
  {
    "path": "libnmap/plugins/backend_service.py",
    "content": "#!/usr/bin/env python\nfrom sqlalchemy import create_engine\nfrom sqlalchemy.schema import Column\nfrom sqlalchemy.types import Integer, DateTime, LargeBinary, Text, String\nfrom sqlalchemy.ext.declarative import declarative_base\nfrom sqlalchemy.orm import sessionmaker\n\nfrom libnmap.plugins.backendplugin import NmapBackendPlugin\nfrom libnmap.reportjson import ReportEncoder, ReportDecoder\n\nimport json\nfrom datetime import datetime\nimport binascii\n\nBase = declarative_base()\n\nclass NmapSqlPlugin(NmapBackendPlugin):\n    \"\"\"\n        This class handle the persistence of NmapRepport object in SQL backend\n        Implementation is made using sqlalchemy(0.8.1)\n        usage :\n\n        #get a nmapReport object\n        from libnmap.parser import NmapParser\n        from libnmap.reportjson import ReportDecoder, ReportEncoder\n        import json\n        nmap_report_obj = NmapParser.parse_fromfile(\n               '/home/vagrant/python-nmap-lib/libnmap/test/files/1_hosts.xml')\n\n         #get a backend with in memory sqlite\n         from libnmap.plugins.backendpluginFactory import BackendPluginFactory\n         mybackend_mem = BackendPluginFactory.create(plugin_name='sql',\n                                                     url='sqlite://',\n                                                     echo=True)\n\n         mybackend_mysql = BackendPluginFactory.create(plugin_name='sql',\n                            url='mysql+mysqldb://scott:tiger@localhost/foo',\n                            echo=True)\n         mybackend = BackendPluginFactory.create(plugin_name='sql',\n                                        url='sqlite:////tmp/reportdb.sql',\n                                        echo=True)\n         #lets save!!\n         nmap_report_obj.save(mybackend)\n         mybackend.getall()\n         mybackend.get(1)\n    \"\"\"\n    class Reports(Base):\n        \"\"\"\n            Embeded class for ORM map NmapReport to a\n            simple three column table\n        \"\"\"\n        __tablename__ = 'result_ports'\n\n        id = Column('id', Integer, primary_key=True)\n        taskid = Column('taskid', Integer)\n        inserted = Column('inserted', DateTime(), default='now')\n        address = Column('address', String(256))\n        port = Column('port', Integer)\n        service = Column('service', String(256))\n        state = Column('state', String(12))\n        protocol = Column('protocol', String(12))\n        product = Column('product', String(64))\n        product_version = Column('product_version', String(64))\n        product_extrainfo = Column('product_extrainfo', String(128))\n        # banner = Column('banner', String(256))\n        scripts_results = Column('scripts_results', Text)\n\n        def __init__(self, obj_NmapReport):\n            self.inserted = datetime.fromtimestamp(int(obj_NmapReport.endtime))\n            self.taskid = obj_NmapReport.taskid\n            self.address = obj_NmapReport.address\n            self.port = obj_NmapReport.port\n            self.service = obj_NmapReport.service\n            self.state = obj_NmapReport.state\n            self.protocol = str(obj_NmapReport.protocol)\n            self.product = str(obj_NmapReport.product)\n            self.product_version = str(obj_NmapReport.product_version)\n            self.product_extrainfo = str(obj_NmapReport.product_extrainfo)\n            # self.banner = str(obj_NmapReport.banner)\n            # self.scripts_results = binascii.b2a_hex(str(obj_NmapReport.scripts_results))\n\n            if len(obj_NmapReport.scripts_results) > 0:                \n                self.scripts_results = obj_NmapReport.scripts_results[0]['output']\n            else:\n                self.scripts_results = None\n\n\n        def decode(self):\n            json_decoded = self.report_json.decode('utf-8')\n            nmap_report_obj = json.loads(json_decoded,\n                                         cls=ReportDecoder)\n            return nmap_report_obj\n\n    def __init__(self, **kwargs):\n        \"\"\"\n            constructor receive a **kwargs as the **kwargs in the sqlalchemy\n            create_engine() method (see sqlalchemy docs)\n            You must add to this **kwargs an 'url' key with the url to your\n            database\n            This constructor will :\n            - create all the necessary obj to discuss with the DB\n            - create all the mapping(ORM)\n\n            todo : suport the : sqlalchemy.engine_from_config\n\n            :param **kwargs:\n            :raises: ValueError if no url is given,\n                    all exception sqlalchemy can throw\n            ie sqlite in memory url='sqlite://' echo=True\n            ie sqlite file on hd url='sqlite:////tmp/reportdb.sql' echo=True\n            ie mysql url='mysql+mysqldb://scott:tiger@localhost/foo'\n        \"\"\"\n        NmapBackendPlugin.__init__(self)\n        self.engine = None\n        self.url = None\n        self.Session = sessionmaker()\n\n        if 'url' not in kwargs:\n            raise ValueError\n        self.url = kwargs['url']\n        del kwargs['url']\n        try:\n            self.engine = create_engine(self.url, **kwargs)\n            Base.metadata.create_all(bind=self.engine, checkfirst=True)\n            self.Session.configure(bind=self.engine)\n        except:\n            raise\n\n    def insert(self, nmap_report):\n        \"\"\"\n           insert NmapReport in the backend\n\n           :param NmapReport:\n\n           :returns: the ident of the object in the backend for future usage \\\n           or None\n        \"\"\"\n        sess = self.Session()\n        report = NmapSqlPlugin.Reports(nmap_report)\n        sess.add(report)\n        sess.commit()\n        reportid = report.id\n        sess.close()\n        return reportid if reportid else None\n\n    def get(self, report_id=None):\n        \"\"\"\n            retreive a NmapReport from the backend\n\n            :param id: str\n\n            :returns: NmapReport\n        \"\"\"\n        if report_id is None:\n            raise ValueError\n        sess = self.Session()\n        our_report = (\n            sess.query(NmapSqlPlugin.Reports).filter_by(id=report_id).first())\n        sess.close()\n        return our_report.decode() if our_report else None\n\n    def getall(self):\n        \"\"\"\n            :param filter: Nice to have implement a filter capability\n\n            :returns: collection of tuple (id,NmapReport)\n        \"\"\"\n        sess = self.Session()\n        nmapreportList = []\n        for report in (\n                sess.query(NmapSqlPlugin.Reports).\n                order_by(NmapSqlPlugin.Reports.inserted)):\n            nmapreportList.append((report.id, report.decode()))\n        sess.close()\n        return nmapreportList\n\n    def delete(self, report_id=None):\n        \"\"\"\n            Remove a report from the backend\n\n            :param id: str\n\n            :returns: The number of rows deleted\n        \"\"\"\n        if report_id is None:\n            raise ValueError\n        nb_line = 0\n        sess = self.Session()\n        nb_line = sess.query(NmapSqlPlugin.Reports).\\\n            filter_by(id=report_id).delete()\n        sess.commit()\n        sess.close()\n        return nb_line\n"
  },
  {
    "path": "libnmap/plugins/backendplugin.py",
    "content": "#!/usr/bin/env python\n\n\nclass NmapBackendPlugin(object):\n    \"\"\"\n        Abstract class showing to the minimal implementation for a plugin\n        All subclass MUST at least implement the following methods\n    \"\"\"\n    def __init__(self):\n        self.dbname = 'nmapdb'\n        self.store = 'reports'\n\n    def insert(self, NmapReport):\n        \"\"\"\n            insert NmapReport in the backend\n            :param NmapReport:\n            :return: str the ident of the object in the backend for\n            future usage\n            or None\n        \"\"\"\n        raise NotImplementedError\n\n    def delete(self, id):\n        \"\"\"\n            delete NmapReport if the backend\n            :param id: str\n        \"\"\"\n        raise NotImplementedError\n\n    def get(self, id):\n        \"\"\"\n            retreive a NmapReport from the backend\n            :param id: str\n            :return: NmapReport\n        \"\"\"\n        raise NotImplementedError\n\n    def getall(self, filter):\n        \"\"\"\n            :return: collection of tuple (id,NmapReport)\n            :param filter: Nice to have implement a filter capability\n        \"\"\"\n        raise NotImplementedError\n"
  },
  {
    "path": "libnmap/plugins/backendpluginFactory.py",
    "content": "#!/usr/bin/env python\nimport sys\nimport inspect\n\n\nclass BackendPluginFactory(object):\n    \"\"\"\n        This is a backend plugin factory a backend instance MUST be\n        created via the static method create()\n        ie : mybackend = BackendPluginFactory.create()\n    \"\"\"\n    @classmethod\n    def create(cls, plugin_name=\"mongodb\", **kwargs):\n        \"\"\"Import the needed lib and return an object NmapBackendPlugin\n           representing the backend of your desire.\n           NmapBackendPlugin is an abstract class, to know what argument\n           need to be given, review the code of the subclass you need\n           :param plugin_name: str : name of the py file without .py\n           :return: NmapBackend (abstract class on top of all plugin)\n        \"\"\"\n        backendplugin = None\n        plugin_path = \"libnmap.plugins.{0}\".format(plugin_name)\n        __import__(plugin_path)\n        pluginobj = sys.modules[plugin_path]\n        pluginclasses = inspect.getmembers(pluginobj, inspect.isclass)\n        for classname, classobj in pluginclasses:\n            if inspect.getmodule(classobj).__name__.find(plugin_path) == 0:\n                try:\n                    backendplugin = classobj(**kwargs)\n                except Exception as error:\n                    raise Exception(\"Cannot create Backend: {0}\".format(error))\n        return backendplugin\n"
  },
  {
    "path": "libnmap/plugins/es.py",
    "content": "# -*- coding: utf-8 -*-\n\nimport json\nfrom libnmap.reportjson import ReportEncoder\nfrom libnmap.plugins.backendplugin import NmapBackendPlugin\nfrom elasticsearch import Elasticsearch\nfrom datetime import datetime\n\n\nclass NmapElasticsearchPlugin(NmapBackendPlugin):\n    \"\"\"\n        This class enables the user to store and manipulate nmap reports \\\n        in a elastic search db.\n    \"\"\"\n    def __init__(self, index=None):\n        if index is None:\n            self.index = \"nmap.{0}\".format(datetime.now().strftime('%Y-%m-%d'))\n        else:\n            self.index = index\n        self._esapi = Elasticsearch()\n\n    def insert(self, report, doc_type=None):\n        \"\"\"\n            insert NmapReport in the backend\n            :param NmapReport:\n            :return: str the ident of the object in the backend for\n            future usage\n            or None\n        \"\"\"\n        if doc_type is None:\n            doc_type = 'NmapReport'\n        j = json.dumps(report, cls=ReportEncoder)\n        res = self._esapi.index(\n            index=self.index,\n            doc_type=doc_type,\n            body=json.loads(j))\n        rc = res['_id']\n        return rc\n\n    def delete(self, id):\n        \"\"\"\n            delete NmapReport if the backend\n            :param id: str\n        \"\"\"\n        raise NotImplementedError\n\n    def get(self, id):\n        \"\"\"\n            retreive a NmapReport from the backend\n            :param id: str\n            :return: NmapReport\n        \"\"\"\n        res = self._esapi.get(index=self.index,\n                              doc_type=\"NmapReport\",\n                              id=id)['_source']\n        return res\n\n    def getall(self, filter=None):\n        \"\"\"\n            :return: collection of tuple (id,NmapReport)\n            :param filter: Nice to have implement a filter capability\n        \"\"\"\n        rsearch = self._esapi.search(index=self.index,\n                                     body={\"query\": {\"match_all\": {}}})\n        print(\"--------------------\")\n        print(type(rsearch))\n        print(rsearch)\n        print(\"------------\")\n"
  },
  {
    "path": "libnmap/plugins/mongodb.py",
    "content": "#!/usr/bin/env python\nimport json\nfrom pymongo import MongoClient\nfrom bson.objectid import ObjectId\n\nfrom libnmap.reportjson import ReportEncoder\nfrom libnmap.parser import NmapParser\nfrom libnmap.plugins.backendplugin import NmapBackendPlugin\n\n\nclass NmapMongodbPlugin(NmapBackendPlugin):\n    \"\"\"\n        This class handle the persistence of NmapRepport object in mongodb\n        Implementation is made using pymongo\n        Object of this class must be create via the\n        BackendPluginFactory.create(**url) where url is a named dict like\n        {'plugin_name': \"mongodb\"} this dict may reeive all the param\n        MongoClient() support\n    \"\"\"\n    def __init__(self, dbname=None, store=None, **kwargs):\n        NmapBackendPlugin.__init__(self)\n        if dbname is not None:\n            self.dbname = dbname\n        if store is not None:\n            self.store = store\n        self.dbclient = MongoClient(**kwargs)\n        self.collection = self.dbclient[self.dbname][self.store]\n\n    def insert(self, report):\n        \"\"\"\n            create a json object from an NmapReport instance\n            :param NmapReport: obj to insert\n            :return: str id\n        \"\"\"\n        j = json.dumps(report, cls=ReportEncoder)\n        try:\n            oid = self.collection.insert(json.loads(j))\n        except:\n            raise Exception(\"Failed to insert nmap object in MongoDB\")\n        return str(oid)\n\n    def get(self, str_report_id=None):\n        \"\"\" select a NmapReport by Id\n            :param str: id\n            :return: NmapReport object\n        \"\"\"\n        rid = str_report_id\n        nmapreport = None\n        if str_report_id is not None and isinstance(str_report_id, str):\n            rid = ObjectId(str_report_id)\n\n        if isinstance(rid, ObjectId):\n            # get a specific report by mongo's id\n            resultset = self.collection.find({'_id': rid})\n            if resultset.count() == 1:\n                # search by id means only one in the iterator\n                record = resultset[0]\n                # remove mongo's id to recreate the NmapReport Obj\n                del record['_id']\n                nmapreport = NmapParser.parse_fromdict(record)\n        return nmapreport\n\n    def getall(self, dict_filter=None):\n        \"\"\"return a list of tuple (id,NmapReport) saved in the backend\n           TODO : add a filter capability\n        \"\"\"\n        nmapreportlist = []\n        resultset = self.collection.find()\n        for report in resultset:\n            oid = report['_id']\n            del report['_id']\n            nmapreport = NmapParser.parse_fromdict(report)\n            nmapreportlist.append((oid, nmapreport))\n        return nmapreportlist\n\n    def delete(self, report_id=None):\n        \"\"\"\n            delete an obj from the backend\n            :param str: id\n            :return: dict document with result or None\n        \"\"\"\n        if report_id is not None and isinstance(report_id, str):\n            return self.collection.remove({'_id': ObjectId(report_id)})\n        else:\n            return self.collection.remove({'_id': report_id})\n"
  },
  {
    "path": "libnmap/plugins/s3.py",
    "content": "#!/usr/bin/env python\n\"\"\"\n:mod:`libnmap.plugin.s3` -- S3 Backend Plugin\n=============================================\n\n.. module:: libnmap.plugin.s3\n\n:platform: Linux\n:synopsis: a plugin is representation of a S3 backend using boto\n\n.. moduleauthor:: Ronald Bister\n.. moduleauthor:: Mike Boutillier\n\"\"\"\nimport json\nfrom bson.objectid import ObjectId\nfrom boto.s3.connection import S3Connection, OrdinaryCallingFormat\nfrom boto.s3.key import Key\nfrom boto.s3.bucketlistresultset import bucket_lister\nfrom boto.exception import S3ResponseError\nfrom libnmap.reportjson import ReportEncoder\nfrom libnmap.parser import NmapParser\nfrom libnmap.plugins.backendplugin import NmapBackendPlugin\n\n\nclass NmapS3Plugin(NmapBackendPlugin):\n    \"\"\"\n        This plugin save the reports on S3 and compatible.\n    \"\"\"\n    def __init__(self, **kwargs):\n        \"\"\"\n            - create the conn object\n            - create the bucket (if it doesn't exist)\n                - if not given, awsaccessKey_nmapreport\n            - may raise exception (ie in case of conflict bucket name)\n            - sample :\n            To connect to walrus:\n            from libnmap.plugins.backendpluginFactory import\n                            BackendPluginFactory\n            walrusBackend =\n              BackendPluginFactory.create(\n                    plugin_name='s3',\n                    host=\"walrus.ecc.eucalyptus.com\",\n                    path=\"/services/Walrus\",port=8773,\n                    is_secure=False,\n                    aws_access_key_id='UU72FLVJCAYRATLXI70YH',\n                    aws_secret_access_key=\n                               'wFg7gP5YFHjVlxakw1g1uCC8UR2xVW5ax9ErZCut')\n           To connect to S3:\n           mybackend_S3 =\n             BackendPluginFactory.create(\n                plugin_name='s3',\n                is_secure=True,\n                aws_access_key_id='MYACCESSKEY',\n                aws_secret_access_key='MYSECRET')\n        \"\"\"\n        NmapBackendPlugin.__init__(self)\n        try:\n            calling_format = OrdinaryCallingFormat()\n            if 'bucket' not in kwargs:\n                self.bucket_name = ''.join(\n                    [kwargs['aws_access_key_id'].lower(),\n                     \"_nmapreport\"])\n            else:\n                self.bucket_name = kwargs['bucket']\n                del kwargs['bucket']\n            kwargs['calling_format'] = calling_format\n            self.conn = S3Connection(**kwargs)\n            self.bucket = self.conn.lookup(self.bucket_name)\n            if self.bucket is None:\n                self.bucket = self.conn.create_bucket(self.bucket_name)\n        except:\n            raise\n\n    def insert(self, report):\n        \"\"\"\n            create a json string from an NmapReport instance\n            and push it to S3 bucket.\n\n            :param NmapReport: obj to insert\n            :rtype: string\n            :return: str id\n            :todo: Add tagging option\n        \"\"\"\n        try:\n            oid = ObjectId()\n            mykey = Key(self.bucket)\n            mykey.key = str(oid)\n            strjsonnmapreport = json.dumps(report, cls=ReportEncoder)\n            mykey.set_contents_from_string(strjsonnmapreport)\n        except:\n            raise Exception(\"Failed to add nmap object in s3 bucket\")\n        return str(oid)\n\n    def get(self, str_report_id=None):\n        \"\"\"\n            select a NmapReport by Id.\n\n            :param str: id\n            :rtype: NmapReport\n            :return: NmapReport object\n        \"\"\"\n        nmapreport = None\n        if str_report_id is not None and isinstance(str_report_id, str):\n            try:\n                mykey = Key(self.bucket)\n                mykey.key = str_report_id\n                nmapreportjson = json.loads(mykey.get_contents_as_string())\n                nmapreport = NmapParser.parse_fromdict(nmapreportjson)\n            except S3ResponseError:\n                pass\n        return nmapreport\n\n    def getall(self, dict_filter=None):\n        \"\"\"\n            :rtype: List of tuple\n            :return: list of key/report\n            :todo: add a filter capability\n        \"\"\"\n        nmapreportlist = []\n        for key in bucket_lister(self.bucket):\n            if isinstance(key, Key):\n                nmapreportjson = json.loads(key.get_contents_as_string())\n                nmapreport = NmapParser.parse_fromdict(nmapreportjson)\n                nmapreportlist.append((key.key, nmapreport))\n        return nmapreportlist\n\n    def delete(self, report_id=None):\n        \"\"\"\n            delete an obj from the backend\n\n            :param str: id\n            :return: dict document with result or None\n        \"\"\"\n        rcode = None\n        if report_id is not None and isinstance(report_id, str):\n            rcode = self.bucket.delete_key(report_id)\n        return rcode\n"
  },
  {
    "path": "libnmap/plugins/sql.py",
    "content": "#!/usr/bin/env python\nfrom sqlalchemy import create_engine\nfrom sqlalchemy.schema import Column\nfrom sqlalchemy.types import Integer, DateTime, LargeBinary, Text, String\nfrom sqlalchemy.ext.declarative import declarative_base\nfrom sqlalchemy.orm import sessionmaker\n\nfrom libnmap.plugins.backendplugin import NmapBackendPlugin\nfrom libnmap.reportjson import ReportEncoder, ReportDecoder\n\nimport json\nfrom datetime import datetime\n\nBase = declarative_base()\n\nclass NmapSqlPlugin(NmapBackendPlugin):\n    \"\"\"\n        This class handle the persistence of NmapRepport object in SQL backend\n        Implementation is made using sqlalchemy(0.8.1)\n        usage :\n\n        #get a nmapReport object\n        from libnmap.parser import NmapParser\n        from libnmap.reportjson import ReportDecoder, ReportEncoder\n        import json\n        nmap_report_obj = NmapParser.parse_fromfile(\n               '/home/vagrant/python-nmap-lib/libnmap/test/files/1_hosts.xml')\n\n         #get a backend with in memory sqlite\n         from libnmap.plugins.backendpluginFactory import BackendPluginFactory\n         mybackend_mem = BackendPluginFactory.create(plugin_name='sql',\n                                                     url='sqlite://',\n                                                     echo=True)\n\n         mybackend_mysql = BackendPluginFactory.create(plugin_name='sql',\n                            url='mysql+mysqldb://scott:tiger@localhost/foo',\n                            echo=True)\n         mybackend = BackendPluginFactory.create(plugin_name='sql',\n                                        url='sqlite:////tmp/reportdb.sql',\n                                        echo=True)\n         #lets save!!\n         nmap_report_obj.save(mybackend)\n         mybackend.getall()\n         mybackend.get(1)\n    \"\"\"\n    class Reports(Base):\n        \"\"\"\n            Embeded class for ORM map NmapReport to a\n            simple three column table\n        \"\"\"\n        __tablename__ = 'reports'\n\n        id = Column('report_id', Integer, primary_key=True)\n        inserted = Column('inserted', DateTime(), default='now')\n        host = Column('host', String(256))\n        command_line = Column('command_line', String(256))\n        report_json = Column('report_json', LargeBinary())\n\n        def __init__(self, obj_NmapReport):\n            # self.inserted = datetime.fromtimestamp(obj_NmapReport.endtime)\n            self.command_line = str(obj_NmapReport.status)\n\n            self.host = obj_NmapReport.address\n            self.command_line = \n\n            dumped_json = json.dumps(obj_NmapReport,\n                                     cls=ReportEncoder)\n            self.report_json = bytes(dumped_json.encode('UTF-8'))\n\n        def decode(self):\n            json_decoded = self.report_json.decode('utf-8')\n            nmap_report_obj = json.loads(json_decoded,\n                                         cls=ReportDecoder)\n            return nmap_report_obj\n\n    def __init__(self, **kwargs):\n        \"\"\"\n            constructor receive a **kwargs as the **kwargs in the sqlalchemy\n            create_engine() method (see sqlalchemy docs)\n            You must add to this **kwargs an 'url' key with the url to your\n            database\n            This constructor will :\n            - create all the necessary obj to discuss with the DB\n            - create all the mapping(ORM)\n\n            todo : suport the : sqlalchemy.engine_from_config\n\n            :param **kwargs:\n            :raises: ValueError if no url is given,\n                    all exception sqlalchemy can throw\n            ie sqlite in memory url='sqlite://' echo=True\n            ie sqlite file on hd url='sqlite:////tmp/reportdb.sql' echo=True\n            ie mysql url='mysql+mysqldb://scott:tiger@localhost/foo'\n        \"\"\"\n        NmapBackendPlugin.__init__(self)\n        self.engine = None\n        self.url = None\n        self.Session = sessionmaker()\n\n        if 'url' not in kwargs:\n            raise ValueError\n        self.url = kwargs['url']\n        del kwargs['url']\n        try:\n            self.engine = create_engine(self.url, **kwargs)\n            Base.metadata.create_all(bind=self.engine, checkfirst=True)\n            self.Session.configure(bind=self.engine)\n        except:\n            raise\n\n    def insert(self, nmap_report):\n        \"\"\"\n           insert NmapReport in the backend\n\n           :param NmapReport:\n\n           :returns: the ident of the object in the backend for future usage \\\n           or None\n        \"\"\"\n        sess = self.Session()\n        report = NmapSqlPlugin.Reports(nmap_report)\n        sess.add(report)\n        sess.commit()\n        reportid = report.id\n        sess.close()\n        return reportid if reportid else None\n\n    def get(self, report_id=None):\n        \"\"\"\n            retreive a NmapReport from the backend\n\n            :param id: str\n\n            :returns: NmapReport\n        \"\"\"\n        if report_id is None:\n            raise ValueError\n        sess = self.Session()\n        our_report = (\n            sess.query(NmapSqlPlugin.Reports).filter_by(id=report_id).first())\n        sess.close()\n        return our_report.decode() if our_report else None\n\n    def getall(self):\n        \"\"\"\n            :param filter: Nice to have implement a filter capability\n\n            :returns: collection of tuple (id,NmapReport)\n        \"\"\"\n        sess = self.Session()\n        nmapreportList = []\n        for report in (\n                sess.query(NmapSqlPlugin.Reports).\n                order_by(NmapSqlPlugin.Reports.inserted)):\n            nmapreportList.append((report.id, report.decode()))\n        sess.close()\n        return nmapreportList\n\n    def delete(self, report_id=None):\n        \"\"\"\n            Remove a report from the backend\n\n            :param id: str\n\n            :returns: The number of rows deleted\n        \"\"\"\n        if report_id is None:\n            raise ValueError\n        nb_line = 0\n        sess = self.Session()\n        nb_line = sess.query(NmapSqlPlugin.Reports).\\\n            filter_by(id=report_id).delete()\n        sess.commit()\n        sess.close()\n        return nb_line\n"
  },
  {
    "path": "libnmap/process.py",
    "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\nimport os\nimport pwd\nimport shlex\nimport subprocess\nimport multiprocessing\nfrom threading import Thread\nfrom xml.dom import pulldom\nimport warnings\n\ntry:\n    from Queue import Empty, Full\nexcept ImportError:\n    from queue import Empty, Full\n\n__all__ = [\n    'NmapProcess'\n]\n\n\nclass NmapTask(object):\n    \"\"\"\n    NmapTask is a internal class used by process. Each time nmap\n    starts a new task during the scan, a new class will be instanciated.\n    Classes examples are: \"Ping Scan\", \"NSE script\", \"DNS Resolve\",..\n    To each class an estimated time to complete is assigned and updated\n    at least every second within the NmapProcess.\n    A property NmapProcess.current_task points to the running task at\n    time T and a dictionnary NmapProcess.tasks with \"task name\" as key\n    is built during scan execution\n    \"\"\"\n    def __init__(self, name, starttime=0, extrainfo=''):\n        self.name = name\n        self.etc = 0\n        self.progress = 0\n        self.percent = 0\n        self.remaining = 0\n        self.status = 'started'\n        self.starttime = starttime\n        self.endtime = 0\n        self.extrainfo = extrainfo\n        self.updated = 0\n\n\nclass NmapProcess(Thread):\n    \"\"\"\n    NmapProcess is a class which wraps around the nmap executable.\n\n    Consequently, in order to run an NmapProcess, nmap should be installed\n    on the host running the script. By default NmapProcess will produce\n    the output of the nmap scan in the nmap XML format. This could be then\n    parsed out via the NmapParser class from libnmap.parser module.\n    \"\"\"\n    def __init__(self, targets=\"127.0.0.1\",\n                 options=\"-sT\", event_callback=None, safe_mode=True, fqp=None):\n        \"\"\"\n        Constructor of NmapProcess class.\n\n        :param targets: hosts to be scanned. Could be a string of hosts \\\n        separated with a coma or a python list of hosts/ip.\n        :type targets: string or list\n\n        :param options: list of nmap options to be applied to scan. \\\n        These options are all documented in nmap's man pages.\n\n        :param event_callback: callable function which will be ran \\\n        each time nmap process outputs data. This function will receive \\\n        two parameters:\n\n            1. the nmap process object\n            2. the data produced by nmap process. See readme for examples.\n\n        :param safe_mode: parameter to protect unsafe options like -oN, -oG, \\\n        -iL, -oA,...\n\n        :param fqp: full qualified path, if None, nmap will be searched \\\n        in the PATH\n\n        :return: NmapProcess object\n\n        \"\"\"\n        Thread.__init__(self)\n        unsafe_opts = set(['-oG', '-oN', '-iL', '-oA', '-oS', '-oX',\n                           '--iflist', '--resume', '--stylesheet',\n                           '--datadir'])\n        if fqp:\n            if os.path.isfile(fqp) and os.access(fqp, os.X_OK):\n                self.__nmap_binary = fqp\n            else:\n                raise EnvironmentError(1, \"wrong path or not executable\", fqp)\n        else:\n            nmap_binary_name = \"nmap\"\n            self.__nmap_binary = self._whereis(nmap_binary_name)\n        # self.__nmap_fixed_options = \"-oX - -vvv --stats-every 1s\"\n        self.__nmap_fixed_options = \"-oX -\"\n\n        if self.__nmap_binary is None:\n            raise EnvironmentError(1, \"nmap is not installed or could \"\n                                      \"not be found in system path\")\n\n        if isinstance(targets, str):\n            self.__nmap_targets = targets.replace(\" \", \"\").split(',')\n        elif isinstance(targets, list):\n            self.__nmap_targets = targets\n        else:\n            raise Exception(\"Supplied target list should be either a \"\n                            \"string or a list\")\n\n        self._nmap_options = set(options.split())\n        if safe_mode and not self._nmap_options.isdisjoint(unsafe_opts):\n            raise Exception(\"unsafe options activated while safe_mode \"\n                            \"is set True\")\n        self.__nmap_dynamic_options = options\n        self.__sudo_run = ''\n        self.__nmap_command_line = self.get_command_line()\n\n        if event_callback and callable(event_callback):\n            self.__nmap_event_callback = event_callback\n        else:\n            self.__nmap_event_callback = None\n        (self.DONE, self.READY, self.RUNNING,\n         self.CANCELLED, self.FAILED) = range(5)\n        self._run_init()\n\n    def _run_init(self):\n        self.__process_killed = multiprocessing.Event()\n        self.__nmap_command_line = self.get_command_line()\n        # API usable in callback function\n        self.__nmap_proc = None\n        self.__qout = None\n        self.__nmap_rc = 0\n        self.__state = self.RUNNING\n        self.__starttime = 0\n        self.__endtime = 0\n        self.__version = ''\n        self.__elapsed = ''\n        self.__summary = ''\n        self.__stdout = ''\n        self.__stderr = ''\n        self.__current_task = ''\n        self.__nmap_tasks = {}\n\n    def _whereis(self, program):\n        \"\"\"\n        Protected method enabling the object to find the full path of a binary\n        from its PATH environment variable.\n\n        :param program: name of a binary for which the full path needs to\n        be discovered.\n\n        :return: the full path to the binary.\n\n        :todo: add a default path list in case PATH is empty.\n        \"\"\"\n        for path in os.environ.get('PATH', '').split(':'):\n            if (os.path.exists(os.path.join(path, program)) and not\n               os.path.isdir(os.path.join(path, program))):\n                return os.path.join(path, program)\n        return None\n\n    def get_command_line(self):\n        \"\"\"\n        Public method returning the reconstructed command line ran via the lib\n\n        :return: the full nmap command line to run\n        :rtype: string\n        \"\"\"\n        return (\"{0} {1} {2} {3} {4}\".format(self.__sudo_run,\n                                             self.__nmap_binary,\n                                             self.__nmap_fixed_options,\n                                             self.__nmap_dynamic_options,\n                                             \" \".join(self.__nmap_targets)))\n\n    def sudo_run(self, run_as='root'):\n        \"\"\"\n        Public method enabling the library's user to run the scan with\n        priviledges via sudo. The sudo configuration should be set manually\n        on the local system otherwise sudo will prompt for a password.\n        This method alters the command line by prefixing the sudo command to\n        nmap and will then call self.run()\n\n        :param run_as: user name to which the lib needs to sudo to run the scan\n\n        :return: return code from nmap execution\n        \"\"\"\n        sudo_user = run_as.split().pop()\n        try:\n            pwd.getpwnam(sudo_user).pw_uid\n        except KeyError:\n            _exmsg = (\"Username {0} does not exists. Please supply\"\n                      \" a valid username\".format(run_as))\n            raise EnvironmentError(_exmsg)\n\n        sudo_path = self._whereis(\"sudo\")\n        if sudo_path is None:\n            raise EnvironmentError(2, \"sudo is not installed or \"\n                                      \"could not be found in system path: \"\n                                      \"cannot run nmap with sudo\")\n\n        self.__sudo_run = \"{0} -u {1}\".format(sudo_path, sudo_user)\n        rc = self.run()\n        self.__sudo_run = \"\"\n\n        return rc\n\n    def sudo_run_background(self, run_as='root'):\n        \"\"\"\n        Public method enabling the library's user to run in background a\n        nmap scan with priviledges via sudo.\n        The sudo configuration should be set manually on the local system\n        otherwise sudo will prompt for a password.\n        This method alters the command line by prefixing the sudo command to\n        nmap and will then call self.run()\n\n        :param run_as: user name to which the lib needs to sudo to run the scan\n\n        :return: return code from nmap execution\n        \"\"\"\n        sudo_user = run_as.split().pop()\n        try:\n            pwd.getpwnam(sudo_user).pw_uid\n        except KeyError:\n            _exmsg = (\"Username {0} does not exists. Please supply\"\n                      \" a valid username\".format(run_as))\n            raise EnvironmentError(_exmsg)\n\n        sudo_path = self._whereis(\"sudo\")\n        if sudo_path is None:\n            raise EnvironmentError(2, \"sudo is not installed or \"\n                                      \"could not be found in system path: \"\n                                      \"cannot run nmap with sudo\")\n\n        self.__sudo_run = \"{0} -u {1}\".format(sudo_path, sudo_user)\n        super(NmapProcess, self).start()\n\n    def run(self):\n        \"\"\"\n        Public method which is usually called right after the constructor\n        of NmapProcess. This method starts the nmap executable's subprocess.\n        It will also bind a Process that will read from subprocess' stdout\n        and stderr and push the lines read in a python queue for futher\n        processing. This processing is waken-up each time data is pushed\n        from the nmap binary into the stdout reading routine. Processing\n        could be performed by a user-provided callback. The whole\n        NmapProcess object could be accessible asynchroneously.\n\n        return: return code from nmap execution\n        \"\"\"\n        def ioreader_routine(proc_stdout, io_queue, data_pushed, producing):\n            \"\"\"\n            local function that will read lines from a file descriptor\n            and put the data in a python queue for futher processing.\n\n            :param proc_stdout: file descriptor to read lines from.\n            :param io_queue: queue in which read lines will be pushed.\n            :param data_pushed: queue used to push data read from the\n            nmap stdout back into the parent process\n            :param producing: shared variable to notify the parent process\n            that processing is either running, either over.\n            \"\"\"\n            producing.value = 1\n            for streamline in iter(proc_stdout.readline, b''):\n                if self.__process_killed.is_set():\n                    break\n                if streamline is not None:\n                    try:\n                        io_queue.put(str(streamline.decode()))\n                    except Full:\n                        pass\n                    data_pushed.set()\n            producing.value = 0\n            data_pushed.set()\n\n        self._run_init()\n        producing = multiprocessing.Value('i', 1)\n        data_pushed = multiprocessing.Event()\n        self.__qout = multiprocessing.Queue()\n\n        _tmp_cmdline = shlex.split(self.__nmap_command_line)\n        try:\n            self.__nmap_proc = subprocess.Popen(args=_tmp_cmdline,\n                                                stdout=subprocess.PIPE,\n                                                stderr=subprocess.PIPE,\n                                                bufsize=0)\n            ioreader = multiprocessing.Process(target=ioreader_routine,\n                                               args=(self.__nmap_proc.stdout,\n                                                     self.__qout,\n                                                     data_pushed,\n                                                     producing))\n            # ioreader.daemon=False\n            ioreader.start()\n            self.__state = self.RUNNING\n        except OSError:\n            self.__state = self.FAILED\n            raise EnvironmentError(1, \"nmap is not installed or could \"\n                                      \"not be found in system path\")\n\n        thread_stream = ''\n        while(self.__nmap_proc.poll() is None or producing.value == 1):\n            if self.__process_killed.is_set():\n                break\n            if producing.value == 1 and self.__qout.empty():\n                try:\n                    data_pushed.wait()\n                except KeyboardInterrupt:\n                    break\n            try:\n                thread_stream = self.__qout.get_nowait()\n            except Empty:\n                pass\n            except KeyboardInterrupt:\n                break\n            else:\n                self.__stdout += thread_stream\n                evnt = self.__process_event(thread_stream)\n                if self.__nmap_event_callback and evnt:\n                    self.__nmap_event_callback(self)\n            data_pushed.clear()\n        ioreader.join()\n        # queue clean-up\n        while not self.__qout.empty():\n            self.__stdout += self.__qout.get_nowait()\n        self.__stderr += str(self.__nmap_proc.stderr.read().decode())\n\n        self.__nmap_rc = self.__nmap_proc.poll()\n        if self.rc is None:\n            self.__state = self.CANCELLED\n        elif self.rc == 0:\n            self.__state = self.DONE\n            if self.current_task:\n                self.__nmap_tasks[self.current_task.name].progress = 100\n        else:\n            self.__state = self.FAILED\n        return self.rc\n\n    def run_background(self):\n        \"\"\"\n        run nmap scan in background as a thread.\n        For privileged scans, consider NmapProcess.sudo_run_background()\n        \"\"\"\n        self.__state = self.RUNNING\n        super(NmapProcess, self).start()\n\n    def is_running(self):\n        \"\"\"\n        Checks if nmap is still running.\n\n        :return: True if nmap is still running\n        \"\"\"\n        return self.state == self.RUNNING\n\n    def has_terminated(self):\n        \"\"\"\n        Checks if nmap has terminated. Could have failed or succeeded\n\n        :return: True if nmap process is not running anymore.\n        \"\"\"\n        return (self.state == self.DONE or self.state == self.FAILED\n                or self.state == self.CANCELLED)\n\n    def has_failed(self):\n        \"\"\"\n        Checks if nmap has failed.\n\n        :return: True if nmap process errored.\n        \"\"\"\n        return self.state == self.FAILED\n\n    def is_successful(self):\n        \"\"\"\n        Checks if nmap terminated successfully.\n\n        :return: True if nmap terminated successfully.\n        \"\"\"\n        return self.state == self.DONE\n\n    def stop(self):\n        \"\"\"\n        Send KILL -15 to the nmap subprocess and gently ask the threads to\n        stop.\n        \"\"\"\n        self.__state = self.CANCELLED\n        if self.__nmap_proc.poll() is None:\n            self.__nmap_proc.kill()\n        self.__qout.cancel_join_thread()\n        self.__process_killed.set()\n\n    def __process_event(self, eventdata):\n        \"\"\"\n        Private method called while nmap process is running. It enables the\n        library to handle specific data/events produced by nmap process.\n        So far, the following events are supported:\n\n        1. task progress: updates estimated time to completion and percentage\n           done while scan is running. Could be used in combination with a\n           callback function which could then handle this data while scan is\n           running.\n        2. nmap run: header of the scan. Usually displayed when nmap is started\n        3. finished: when nmap scan ends.\n\n        :return: True is event is known.\n\n        :todo: handle parsing directly via NmapParser.parse()\n        \"\"\"\n        rval = False\n        try:\n            edomdoc = pulldom.parseString(eventdata)\n            for xlmnt, xmlnode in edomdoc:\n                if xlmnt is not None and xlmnt == pulldom.START_ELEMENT:\n                    if (xmlnode.nodeName == 'taskbegin' and\n                            xmlnode.attributes.keys()):\n                        xt = xmlnode.attributes\n                        taskname = xt['task'].value\n                        starttime = xt['time'].value\n                        xinfo = ''\n                        if 'extrainfo' in xt.keys():\n                            xinfo = xt['extrainfo'].value\n                        newtask = NmapTask(taskname, starttime, xinfo)\n                        self.__nmap_tasks[newtask.name] = newtask\n                        self.__current_task = newtask.name\n                        rval = True\n                    elif (xmlnode.nodeName == 'taskend' and\n                            xmlnode.attributes.keys()):\n                        xt = xmlnode.attributes\n                        tname = xt['task'].value\n                        xinfo = ''\n                        self.__nmap_tasks[tname].endtime = xt['time'].value\n                        if 'extrainfo' in xt.keys():\n                            xinfo = xt['extrainfo'].value\n                        self.__nmap_tasks[tname].extrainfo = xinfo\n                        self.__nmap_tasks[tname].status = \"ended\"\n                        rval = True\n                    elif (xmlnode.nodeName == 'taskprogress' and\n                            xmlnode.attributes.keys()):\n                        xt = xmlnode.attributes\n                        tname = xt['task'].value\n                        percent = xt['percent'].value\n                        etc = xt['etc'].value\n                        remaining = xt['remaining'].value\n                        updated = xt['time'].value\n                        self.__nmap_tasks[tname].percent = percent\n                        self.__nmap_tasks[tname].progress = percent\n                        self.__nmap_tasks[tname].etc = etc\n                        self.__nmap_tasks[tname].remaining = remaining\n                        self.__nmap_tasks[tname].updated = updated\n                        rval = True\n                    elif (xmlnode.nodeName == 'nmaprun' and\n                            xmlnode.attributes.keys()):\n                        self.__starttime = xmlnode.attributes['start'].value\n                        self.__version = xmlnode.attributes['version'].value\n                        rval = True\n                    elif (xmlnode.nodeName == 'finished' and\n                            xmlnode.attributes.keys()):\n                        self.__endtime = xmlnode.attributes['time'].value\n                        self.__elapsed = xmlnode.attributes['elapsed'].value\n                        self.__summary = xmlnode.attributes['summary'].value\n                        rval = True\n        except:\n            pass\n        return rval\n\n    @property\n    def command(self):\n        \"\"\"\n        return the constructed nmap command or empty string if not\n        constructed yet.\n\n        :return: string\n        \"\"\"\n        return self.__nmap_command_line or ''\n\n    @property\n    def targets(self):\n        \"\"\"\n        Provides the list of targets to scan\n\n        :return: list of string\n        \"\"\"\n        return self.__nmap_targets\n\n    @property\n    def options(self):\n        \"\"\"\n        Provides the list of options for that scan\n\n        :return: list of string (nmap options)\n        \"\"\"\n        return self._nmap_options\n\n    @property\n    def state(self):\n        \"\"\"\n        Accessor for nmap execution state. Possible states are:\n\n        - self.READY\n        - self.RUNNING\n        - self.FAILED\n        - self.CANCELLED\n        - self.DONE\n\n        :return: integer (from above documented enum)\n        \"\"\"\n        return self.__state\n\n    @property\n    def starttime(self):\n        \"\"\"\n        Accessor for time when scan started\n\n        :return: string. Unix timestamp\n        \"\"\"\n        return self.__starttime\n\n    @property\n    def endtime(self):\n        \"\"\"\n        Accessor for time when scan ended\n\n        :return: string. Unix timestamp\n        \"\"\"\n        warnings.warn(\"data collected from finished events are deprecated.\"\n                      \"Use NmapParser.parse()\", DeprecationWarning)\n        return self.__endtime\n\n    @property\n    def elapsed(self):\n        \"\"\"\n        Accessor returning for how long the scan ran (in seconds)\n\n        :return: string\n        \"\"\"\n        warnings.warn(\"data collected from finished events are deprecated.\"\n                      \"Use NmapParser.parse()\", DeprecationWarning)\n        return self.__elapsed\n\n    @property\n    def summary(self):\n        \"\"\"\n        Accessor returning a short summary of the scan's results\n\n        :return: string\n        \"\"\"\n        warnings.warn(\"data collected from finished events are deprecated.\"\n                      \"Use NmapParser.parse()\", DeprecationWarning)\n        return self.__summary\n\n    @property\n    def tasks(self):\n        \"\"\"\n        Accessor returning for the list of tasks ran during nmap scan\n\n        :return: dict of NmapTask object\n        \"\"\"\n        return self.__nmap_tasks\n\n    @property\n    def version(self):\n        \"\"\"\n        Accessor for nmap binary version number\n\n        :return: version number of nmap binary\n        :rtype: string\n        \"\"\"\n        return self.__version\n\n    @property\n    def current_task(self):\n        \"\"\"\n        Accessor for the current NmapTask beeing run\n\n        :return: NmapTask or None if no task started yet\n        \"\"\"\n        rval = None\n        if len(self.__current_task):\n            rval = self.tasks[self.__current_task]\n        return rval\n\n    @property\n    def etc(self):\n        \"\"\"\n        Accessor for estimated time to completion\n\n        :return:  estimated time to completion\n        \"\"\"\n        rval = 0\n        if self.current_task:\n            rval = self.current_task.etc\n        return rval\n\n    @property\n    def progress(self):\n        \"\"\"\n        Accessor for progress status in percentage\n\n        :return: percentage of job processed.\n        \"\"\"\n        rval = 0\n        if self.current_task:\n            rval = self.current_task.progress\n        return rval\n\n    @property\n    def rc(self):\n        \"\"\"\n        Accessor for nmap execution's return code\n\n        :return: nmap execution's return code\n        \"\"\"\n        return self.__nmap_rc\n\n    @property\n    def stdout(self):\n        \"\"\"\n        Accessor for nmap standart output\n\n        :return: output from nmap scan in XML\n        :rtype: string\n        \"\"\"\n        return self.__stdout\n\n    @property\n    def stderr(self):\n        \"\"\"\n        Accessor for nmap standart error\n\n        :return: output from nmap when errors occured.\n        :rtype: string\n        \"\"\"\n        return self.__stderr\n\n\ndef main():\n    def mycallback(nmapscan=None):\n        if nmapscan.is_running() and nmapscan.current_task:\n            ntask = nmapscan.current_task\n            print(\"Task {0} ({1}): ETC: {2} DONE: {3}%\".format(ntask.name,\n                                                               ntask.status,\n                                                               ntask.etc,\n                                                               ntask.progress))\n    nm = NmapProcess(\"scanme.nmap.org\",\n                     options=\"-A\",\n                     event_callback=mycallback)\n    rc = nm.run()\n    if rc == 0:\n        print(\"Scan started at {0} nmap version: {1}\").format(nm.starttime,\n                                                              nm.version)\n        print(\"state: {0} (rc: {1})\").format(nm.state, nm.rc)\n        print(\"results size: {0}\").format(len(nm.stdout))\n        print(\"Scan ended {0}: {1}\").format(nm.endtime, nm.summary)\n    else:\n        print(\"state: {0} (rc: {1})\").format(nm.state, nm.rc)\n        print(\"Error: {stderr}\").format(stderr=nm.stderr)\n        print(\"Result: {0}\").format(nm.stdout)\n\nif __name__ == '__main__':\n    main()\n"
  },
  {
    "path": "libnmap/reportjson.py",
    "content": "#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n\nimport json\nfrom libnmap.objects import NmapHost, NmapService, NmapReport\nfrom libnmap.objects.os import NmapOSFingerprint, NmapOSMatch, NmapOSClass\nfrom libnmap.objects.os import CPE, OSFPPortUsed\nfrom libnmap.parser import NmapParser\n\n\nclass ReportEncoder(json.JSONEncoder):\n    def default(self, obj):\n        otype = {'NmapHost': NmapHost,\n                 'NmapOSFingerprint': NmapOSFingerprint,\n                 'NmapOSMatch': NmapOSMatch,\n                 'NmapOSClass': NmapOSClass,\n                 'CPE': CPE,\n                 'OSFPPortUsed': OSFPPortUsed,\n                 'NmapService': NmapService,\n                 'NmapReport': NmapReport}\n        if isinstance(obj, tuple(otype.values())):\n            key = ('__{0}__').format(obj.__class__.__name__)\n            return {key: obj.__dict__}\n        return json.JSONEncoder.default(self, obj)\n\n\nclass ReportDecoder(json.JSONDecoder):\n    def decode(self, json_str):\n        r = NmapParser.parse_fromdict(json.loads(json_str))\n        return r\n"
  },
  {
    "path": "run.py",
    "content": "#!/usr/bin/env python\n# encoding: utf-8\n# tasks.py\n# email: ringzero@0x557.org\n\nimport sys\nfrom tasks import *\nfrom wyfunc import make_target_list\n\ndef start_nmap_dispath(targets, taskid=None):\n\tprint '-' * 50\n\ttarget_list = make_target_list(targets)\n\n\tfor target in target_list:\n\t\tprint '-' * 50\n\t\tprint '* push %s to Redis' % target\n\t\tprint '* AsyncResult:%s' % nmap_dispath.delay(target,taskid=taskid)\n\n\tprint '-' * 50\n\tprint '* Push nmapscan tasks complete.'\n\tprint '-' * 50\n\nif __name__ == \"__main__\":\n\tif len(sys.argv) == 2:\n\t\tstart_nmap_dispath(sys.argv[1])\n\t\tsys.exit(0)\n\telif len(sys.argv) == 3:\n\t\tstart_nmap_dispath(sys.argv[1], sys.argv[2])\n\telse:\n\t\tprint (\"usage: %s targets taskid\" % sys.argv[0])\n\t\tsys.exit(-1)"
  },
  {
    "path": "src/supervisord_client.conf",
    "content": "; Sample supervisor config file.\n;\n; For more information on the config file, please see:\n; http://supervisord.org/configuration.html\n;\n; Notes:\n;  - Shell expansion (\"~\" or \"$HOME\") is not supported.  Environment\n;    variables can be expanded using this syntax: \"%(ENV_HOME)s\".\n;  - Comments must have a leading space: \"a=b ;comment\" not \"a=b;comment\".\n\n[unix_http_server]\nfile=/tmp/supervisor.sock   ; (the path to the socket file)\n;chmod=0700                 ; socket file mode (default 0700)\n;chown=nobody:nogroup       ; socket file uid:gid owner\n;username=user              ; (default is no username (open server))\n;password=123               ; (default is no password (open server))\n\n[inet_http_server]         ; inet (TCP) server disabled by default\nport=0.0.0.0:9001        ; (ip_address:port specifier, *:port for all iface)\n;username=user              ; (default is no username (open server))\n;password=123               ; (default is no password (open server))\n\n[supervisord]\nlogfile=/tmp/supervisord.log ; (main log file;default $CWD/supervisord.log)\nlogfile_maxbytes=50MB        ; (max main logfile bytes b4 rotation;default 50MB)\nlogfile_backups=10           ; (num of main logfile rotation backups;default 10)\nloglevel=info                ; (log level;default info; others: debug,warn,trace)\npidfile=/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid)\nnodaemon=false               ; (start in foreground if true;default false)\nminfds=1024                  ; (min. avail startup file descriptors;default 1024)\nminprocs=200                 ; (min. avail process descriptors;default 200)\n;umask=022                   ; (process file creation umask;default 022)\n;user=chrism                 ; (default is current user, required if root)\n;identifier=supervisor       ; (supervisord identifier, default is 'supervisor')\n;directory=/tmp              ; (default is not to cd during start)\n;nocleanup=true              ; (don't clean up tempfiles at start;default false)\n;childlogdir=/tmp            ; ('AUTO' child log dir, default $TEMP)\n;environment=KEY=\"value\"     ; (key value pairs to add to environment)\n;strip_ansi=false            ; (strip ansi escape codes in logs; def. false)\n\n; the below section must remain in the config file for RPC\n; (supervisorctl/web interface) to work, additional interfaces may be\n; added by defining them in separate rpcinterface: sections\n[rpcinterface:supervisor]\nsupervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface\n\n[supervisorctl]\nserverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket\n;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket\n;username=chris              ; should be same as http_username if set\n;password=123                ; should be same as http_password if set\n;prompt=mysupervisor         ; cmd line prompt (default \"supervisor\")\n;history_file=~/.sc_history  ; use readline history if available\n\n; The below sample program section shows all possible program subsection values,\n; create one or more 'real' program: sections to be able to control them under\n; supervisor.\n\n;[program:theprogramname]\n;command=/bin/cat              ; the program (relative uses PATH, can take args)\n;process_name=%(program_name)s ; process_name expr (default %(program_name)s)\n;numprocs=1                    ; number of processes copies to start (def 1)\n;directory=/tmp                ; directory to cwd to before exec (def no cwd)\n;umask=022                     ; umask for process (default None)\n;priority=999                  ; the relative start priority (default 999)\n;autostart=true                ; start at supervisord start (default: true)\n;autorestart=unexpected        ; whether/when to restart (default: unexpected)\n;startsecs=1                   ; number of secs prog must stay running (def. 1)\n;startretries=3                ; max # of serial start failures (default 3)\n;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)\n;stopsignal=QUIT               ; signal used to kill process (default TERM)\n;stopwaitsecs=10               ; max num secs to wait b4 SIGKILL (default 10)\n;stopasgroup=false             ; send stop signal to the UNIX process group (default false)\n;killasgroup=false             ; SIGKILL the UNIX process group (def false)\n;user=chrism                   ; setuid to this UNIX account to run the program\n;redirect_stderr=true          ; redirect proc stderr to stdout (default false)\n;stdout_logfile=/a/path        ; stdout log path, NONE for none; default AUTO\n;stdout_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)\n;stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)\n;stdout_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)\n;stdout_events_enabled=false   ; emit events on stdout writes (default false)\n;stderr_logfile=/a/path        ; stderr log path, NONE for none; default AUTO\n;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)\n;stderr_logfile_backups=10     ; # of stderr logfile backups (default 10)\n;stderr_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)\n;stderr_events_enabled=false   ; emit events on stderr writes (default false)\n;environment=A=\"1\",B=\"2\"       ; process environment additions (def no adds)\n;serverurl=AUTO                ; override serverurl computation (childutils)\n\n; The below sample eventlistener section shows all possible\n; eventlistener subsection values, create one or more 'real'\n; eventlistener: sections to be able to handle event notifications\n; sent by supervisor.\n\n;[eventlistener:theeventlistenername]\n;command=/bin/eventlistener    ; the program (relative uses PATH, can take args)\n;process_name=%(program_name)s ; process_name expr (default %(program_name)s)\n;numprocs=1                    ; number of processes copies to start (def 1)\n;events=EVENT                  ; event notif. types to subscribe to (req'd)\n;buffer_size=10                ; event buffer queue size (default 10)\n;directory=/tmp                ; directory to cwd to before exec (def no cwd)\n;umask=022                     ; umask for process (default None)\n;priority=-1                   ; the relative start priority (default -1)\n;autostart=true                ; start at supervisord start (default: true)\n;autorestart=unexpected        ; whether/when to restart (default: unexpected)\n;startsecs=1                   ; number of secs prog must stay running (def. 1)\n;startretries=3                ; max # of serial start failures (default 3)\n;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)\n;stopsignal=QUIT               ; signal used to kill process (default TERM)\n;stopwaitsecs=10               ; max num secs to wait b4 SIGKILL (default 10)\n;stopasgroup=false             ; send stop signal to the UNIX process group (default false)\n;killasgroup=false             ; SIGKILL the UNIX process group (def false)\n;user=chrism                   ; setuid to this UNIX account to run the program\n;redirect_stderr=true          ; redirect proc stderr to stdout (default false)\n;stdout_logfile=/a/path        ; stdout log path, NONE for none; default AUTO\n;stdout_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)\n;stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)\n;stdout_events_enabled=false   ; emit events on stdout writes (default false)\n;stderr_logfile=/a/path        ; stderr log path, NONE for none; default AUTO\n;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)\n;stderr_logfile_backups        ; # of stderr logfile backups (default 10)\n;stderr_events_enabled=false   ; emit events on stderr writes (default false)\n;environment=A=\"1\",B=\"2\"       ; process environment additions\n;serverurl=AUTO                ; override serverurl computation (childutils)\n\n; The below sample group section shows all possible group values,\n; create one or more 'real' group: sections to create \"heterogeneous\"\n; process groups.\n\n;[group:thegroupname]\n;programs=progname1,progname2  ; each refers to 'x' in [program:x] definitions\n;priority=999                  ; the relative start priority (default 999)\n\n; The [include] section can just contain the \"files\" setting.  This\n; setting can list multiple files (separated by whitespace or\n; newlines).  It can also contain wildcards.  The filenames are\n; interpreted as relative to this file.  Included files *cannot*\n; include files themselves.\n\n[program:worker-ringzero]\ncommand=celery -A tasks worker --loglevel=info --workdir=/home/thorns --concurrency=10 --hostname=%(program_name)s%(process_num)02d\ndirectory=/home/thorns\nuser=root\nprocess_name=%(program_name)s_%(process_num)02d\nnumprocs=1\nautostart=true\nautorestart=true\nstartetries=10\nexitcodes=0\nstopsignal=KILL\nstopwaitsecs=10\nredirect_stderr=true\n\n;[include]\n;files = relative/directory/*.ini\n"
  },
  {
    "path": "src/supervisord_server.conf",
    "content": "; Sample supervisor config file.\n;\n; For more information on the config file, please see:\n; http://supervisord.org/configuration.html\n;\n; Notes:\n;  - Shell expansion (\"~\" or \"$HOME\") is not supported.  Environment\n;    variables can be expanded using this syntax: \"%(ENV_HOME)s\".\n;  - Comments must have a leading space: \"a=b ;comment\" not \"a=b;comment\".\n\n[unix_http_server]\nfile=/tmp/supervisor.sock   ; (the path to the socket file)\n;chmod=0700                 ; socket file mode (default 0700)\n;chown=nobody:nogroup       ; socket file uid:gid owner\n;username=user              ; (default is no username (open server))\n;password=123               ; (default is no password (open server))\n\n[inet_http_server]         ; inet (TCP) server disabled by default\nport=0.0.0.0:9001        ; (ip_address:port specifier, *:port for all iface)\n;username=user              ; (default is no username (open server))\n;password=123               ; (default is no password (open server))\n\n[supervisord]\nlogfile=/tmp/supervisord.log ; (main log file;default $CWD/supervisord.log)\nlogfile_maxbytes=50MB        ; (max main logfile bytes b4 rotation;default 50MB)\nlogfile_backups=10           ; (num of main logfile rotation backups;default 10)\nloglevel=info                ; (log level;default info; others: debug,warn,trace)\npidfile=/tmp/supervisord.pid ; (supervisord pidfile;default supervisord.pid)\nnodaemon=false               ; (start in foreground if true;default false)\nminfds=1024                  ; (min. avail startup file descriptors;default 1024)\nminprocs=200                 ; (min. avail process descriptors;default 200)\n;umask=022                   ; (process file creation umask;default 022)\n;user=chrism                 ; (default is current user, required if root)\n;identifier=supervisor       ; (supervisord identifier, default is 'supervisor')\n;directory=/tmp              ; (default is not to cd during start)\n;nocleanup=true              ; (don't clean up tempfiles at start;default false)\n;childlogdir=/tmp            ; ('AUTO' child log dir, default $TEMP)\n;environment=KEY=\"value\"     ; (key value pairs to add to environment)\n;strip_ansi=false            ; (strip ansi escape codes in logs; def. false)\n\n; the below section must remain in the config file for RPC\n; (supervisorctl/web interface) to work, additional interfaces may be\n; added by defining them in separate rpcinterface: sections\n[rpcinterface:supervisor]\nsupervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface\n\n[supervisorctl]\nserverurl=unix:///tmp/supervisor.sock ; use a unix:// URL  for a unix socket\n;serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket\n;username=chris              ; should be same as http_username if set\n;password=123                ; should be same as http_password if set\n;prompt=mysupervisor         ; cmd line prompt (default \"supervisor\")\n;history_file=~/.sc_history  ; use readline history if available\n\n; The below sample program section shows all possible program subsection values,\n; create one or more 'real' program: sections to be able to control them under\n; supervisor.\n\n;[program:theprogramname]\n;command=/bin/cat              ; the program (relative uses PATH, can take args)\n;process_name=%(program_name)s ; process_name expr (default %(program_name)s)\n;numprocs=1                    ; number of processes copies to start (def 1)\n;directory=/tmp                ; directory to cwd to before exec (def no cwd)\n;umask=022                     ; umask for process (default None)\n;priority=999                  ; the relative start priority (default 999)\n;autostart=true                ; start at supervisord start (default: true)\n;autorestart=unexpected        ; whether/when to restart (default: unexpected)\n;startsecs=1                   ; number of secs prog must stay running (def. 1)\n;startretries=3                ; max # of serial start failures (default 3)\n;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)\n;stopsignal=QUIT               ; signal used to kill process (default TERM)\n;stopwaitsecs=10               ; max num secs to wait b4 SIGKILL (default 10)\n;stopasgroup=false             ; send stop signal to the UNIX process group (default false)\n;killasgroup=false             ; SIGKILL the UNIX process group (def false)\n;user=chrism                   ; setuid to this UNIX account to run the program\n;redirect_stderr=true          ; redirect proc stderr to stdout (default false)\n;stdout_logfile=/a/path        ; stdout log path, NONE for none; default AUTO\n;stdout_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)\n;stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)\n;stdout_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)\n;stdout_events_enabled=false   ; emit events on stdout writes (default false)\n;stderr_logfile=/a/path        ; stderr log path, NONE for none; default AUTO\n;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)\n;stderr_logfile_backups=10     ; # of stderr logfile backups (default 10)\n;stderr_capture_maxbytes=1MB   ; number of bytes in 'capturemode' (default 0)\n;stderr_events_enabled=false   ; emit events on stderr writes (default false)\n;environment=A=\"1\",B=\"2\"       ; process environment additions (def no adds)\n;serverurl=AUTO                ; override serverurl computation (childutils)\n\n; The below sample eventlistener section shows all possible\n; eventlistener subsection values, create one or more 'real'\n; eventlistener: sections to be able to handle event notifications\n; sent by supervisor.\n\n;[eventlistener:theeventlistenername]\n;command=/bin/eventlistener    ; the program (relative uses PATH, can take args)\n;process_name=%(program_name)s ; process_name expr (default %(program_name)s)\n;numprocs=1                    ; number of processes copies to start (def 1)\n;events=EVENT                  ; event notif. types to subscribe to (req'd)\n;buffer_size=10                ; event buffer queue size (default 10)\n;directory=/tmp                ; directory to cwd to before exec (def no cwd)\n;umask=022                     ; umask for process (default None)\n;priority=-1                   ; the relative start priority (default -1)\n;autostart=true                ; start at supervisord start (default: true)\n;autorestart=unexpected        ; whether/when to restart (default: unexpected)\n;startsecs=1                   ; number of secs prog must stay running (def. 1)\n;startretries=3                ; max # of serial start failures (default 3)\n;exitcodes=0,2                 ; 'expected' exit codes for process (default 0,2)\n;stopsignal=QUIT               ; signal used to kill process (default TERM)\n;stopwaitsecs=10               ; max num secs to wait b4 SIGKILL (default 10)\n;stopasgroup=false             ; send stop signal to the UNIX process group (default false)\n;killasgroup=false             ; SIGKILL the UNIX process group (def false)\n;user=chrism                   ; setuid to this UNIX account to run the program\n;redirect_stderr=true          ; redirect proc stderr to stdout (default false)\n;stdout_logfile=/a/path        ; stdout log path, NONE for none; default AUTO\n;stdout_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)\n;stdout_logfile_backups=10     ; # of stdout logfile backups (default 10)\n;stdout_events_enabled=false   ; emit events on stdout writes (default false)\n;stderr_logfile=/a/path        ; stderr log path, NONE for none; default AUTO\n;stderr_logfile_maxbytes=1MB   ; max # logfile bytes b4 rotation (default 50MB)\n;stderr_logfile_backups        ; # of stderr logfile backups (default 10)\n;stderr_events_enabled=false   ; emit events on stderr writes (default false)\n;environment=A=\"1\",B=\"2\"       ; process environment additions\n;serverurl=AUTO                ; override serverurl computation (childutils)\n\n; The below sample group section shows all possible group values,\n; create one or more 'real' group: sections to create \"heterogeneous\"\n; process groups.\n\n;[group:thegroupname]\n;programs=progname1,progname2  ; each refers to 'x' in [program:x] definitions\n;priority=999                  ; the relative start priority (default 999)\n\n; The [include] section can just contain the \"files\" setting.  This\n; setting can list multiple files (separated by whitespace or\n; newlines).  It can also contain wildcards.  The filenames are\n; interpreted as relative to this file.  Included files *cannot*\n; include files themselves.\n\n[program:flower]\ncommand=celery flower --port=8080 --broker=redis://127.0.0.1:6379/0\ndirectory=/home/thorns\nprocess_name=%(program_name)s_%(process_num)02d\nnumprocs=1\nautostart=true\nautorestart=true\nstartetries=10\nexitcodes=0\nstopsignal=KILL\nstopwaitsecs=10\nredirect_stderr=true\n\n[program:worker-ringzero]\ncommand=celery -A tasks worker --loglevel=info --workdir=/home/thorns --concurrency=10 --hostname=%(program_name)s%(process_num)02d\ndirectory=/home/thorns\nuser=root\nprocess_name=%(program_name)s_%(process_num)02d\nnumprocs=1\nautostart=true\nautorestart=true\nstartetries=10\nexitcodes=0\nstopsignal=KILL\nstopwaitsecs=10\nredirect_stderr=true\n\n;[include]\n;files = relative/directory/*.ini\n"
  },
  {
    "path": "tasks.py",
    "content": "#!/usr/bin/env python\n# encoding: utf-8\n# tasks.py\n# email: ringzero@0x557.org\n\n'''\n\tThorns Project 分布式任务控制脚本\n\ttasks\n\t\t-- nmap_dispath\t\t\t# nmap 扫描调度函数\n\t\t-- hydra_dispath \t\t# hydra 暴力破解调度函数\n\t\t-- medusa_dispath \t\t# medusa 暴力破解调度函数\n\n\tworker run()\n\t\t--workdir=/home/thorns\n'''\n\nimport subprocess\nfrom celery import Celery, platforms \n\n# 初始化芹菜对象\napp = Celery()\n\n# 允许celery以root权限启动\nplatforms.C_FORCE_ROOT = True\n\n# 修改celery的全局配置\napp.conf.update(\n\tCELERY_IMPORTS = (\"tasks\", ),\n\tBROKER_URL = 'redis://120.132.54.90:6379/0',\n\tCELERY_RESULT_BACKEND = 'db+mysql://celery:celery1@3Wscan@42.62.52.62:443/wscan',\n\tCELERY_TASK_SERIALIZER='json',\n\tCELERY_RESULT_SERIALIZER='json',\n\tCELERY_TIMEZONE='Asia/Shanghai',\n\tCELERY_ENABLE_UTC=True,\n\tCELERY_REDIS_MAX_CONNECTIONS=5000, # Redis 最大连接数\n\tBROKER_TRANSPORT_OPTIONS = {'visibility_timeout': 3600}, # 如果任务没有在 可见性超时 内确认接收，任务会被重新委派给另一个Worker并执行  默认1 hour.\n\t# BROKER_TRANSPORT_OPTIONS = {'fanout_prefix': True},\t\t# 设置一个传输选项来给消息加上前缀\n)\n\n# 失败任务重启休眠时间300秒，最大重试次数5次\n# @app.task(bind=True, default_retry_delay=300, max_retries=5)\n\n@app.task\ndef nmap_dispath(targets, taskid=None):\n\t# nmap环境参数配置\n\trun_script_path = '/home/thorns'\n\tif taskid == None:\n\t\tcmdline = 'python wyportmap.py %s' % targets\n\telse: \n\t\tcmdline = 'python wyportmap.py %s %s' % (targets, taskid)\n\tnmap_proc = subprocess.Popen(cmdline,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)\n\tprocess_output = nmap_proc.stdout.readlines()\n\treturn process_output\n\n@app.task\ndef hydra_dispath(targets, protocol, userdic, passdic, taskid=None):\n\t# 命令执行环境参数配置\n\trun_script_path = '/home/thorns/script/hydra'\n\trun_env = '{\"LD_LIBRARY_PATH\": \"/home/thorns/libs/\"}'\n\n\tif taskid == None:\n\t\tcmdline = 'python hydra.py %s %s %s %s' % (target, protocol, userdic, passdic)\n\telse:\n\t\tcmdline = 'python hydra.py %s %s %s %s %s' % (target, protocol, userdic, passdic, taskid)\n\n\tnmap_proc = subprocess.Popen(cmdline,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd=run_script_path,env=run_env)\n\n\tprocess_output = nmap_proc.stdout.readlines()\n\treturn process_output\n\n@app.task\ndef medusa_dispath(targets, protocol, userdic, passdic, taskid=None):\n\t# 命令执行环境参数配置\n\trun_script_path = '/home/thorns/script/medusa'\n\trun_env = '{\"LD_LIBRARY_PATH\": \"/home/thorns/libs/\"}'\n\n\tif taskid == None:\n\t\tcmdline = 'python medusa.py %s %s %s %s' % (target, protocol, userdic, passdic)\n\telse:\n\t\tcmdline = 'python medusa.py %s %s %s %s %s' % (target, protocol, userdic, passdic, taskid)\n\n\tnmap_proc = subprocess.Popen(cmdline,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd=run_script_path,env=run_env)\n\n\tprocess_output = nmap_proc.stdout.readlines()\n\treturn process_output\n\n\n\n\n\n"
  },
  {
    "path": "wyfunc.py",
    "content": "#!/usr/bin/env python\n# encoding: utf-8\n# wyfunc.py\n# email: ringzero@0x557.org\n\ndef ip2num(ip):\n\tip = [int(x) for x in ip.split('.')]\n\treturn ip[0] << 24 | ip[1] << 16 | ip[2] << 8 | ip[3]\n\ndef num2ip(num):\n\treturn '%s.%s.%s.%s' % (\n\t\t(num & 0xff000000) >> 24,\n\t\t(num & 0x00ff0000) >> 16,\n\t\t(num & 0x0000ff00) >> 8,\n\t\tnum & 0x000000ff\n\t)\n \ndef gen_ips(start, end):\n\t\"\"\"生成IP地址\"\"\"\n\t# if num & 0xff 过滤掉 最后一段为 0 的IP\n\treturn [num2ip(num) for num in range(start, end + 1) if num & 0xff]\n\ndef make_ips_c_block(ipaddr):\n\taddress = {}\n\tipaddr = ipaddr.split('.')\n\tipaddr[3] = '0'\n\tipaddr = '.'.join(ipaddr)\n\n\taddress[ipaddr] = gen_ips(ip2num(ipaddr),ip2num(ipaddr) + 254)\n\treturn address\n\ndef ip_check(ip):\n\tq = ip.split('.')\n\treturn len(q) == 4 and len(filter(lambda x: x >= 0 and x <= 255, \\\n\t\tmap(int, filter(lambda x: x.isdigit(), q)))) == 4\n\ndef make_target_list(targets):\n\ttarget_list = []\n\tprocess_nmap_count = 1\n\tstartip, endip = targets.split('-')\n\tif not ip_check(startip):\n\t\tprint '* StartIP format error'\n\telse:\n\t\tendip_len = len(endip.split('.'))\n\t\tstartip_num = ip2num(startip)\n\t\tif endip_len == 1: # 结束IP为数字的场景\n\t\t\tstartip_endnum = startip.split('.')[3]\n\t\t\ttarget_count =  int(endip) - int(startip_endnum)\n\t\t\tif target_count >= 0:\n\t\t\t\ttarget_list.append(num2ip(startip_num))\n\t\t\t\t# 每10个IP分配到一个wyportmap子进程上\n\t\t\t\t# 更新：为每个独立的IP创建一个单独的任务，并删除掉开始IP，修改的时候注释掉上面那行\n\t\t\t\tfor i in xrange(0,(target_count+process_nmap_count-1)/process_nmap_count):\n\t\t\t\t\tip_count = (i * process_nmap_count) + process_nmap_count\n\t\t\t\t\tremaining_count = target_count - ip_count\n\t\t\t\t\tif remaining_count > 0:\n\t\t\t\t\t\tscan_startip = num2ip(startip_num)\n\t\t\t\t\t\tendip_num = startip_num + process_nmap_count\n\t\t\t\t\t\tstartip_num = endip_num\n\t\t\t\t\t\t# target_option = '%s-%s' % (scan_startip, num2ip(endip_num))\n\t\t\t\t\t\ttarget_option = num2ip(endip_num)\n\t\t\t\t\t\ttarget_list.append(target_option)\n\t\t\t\t\telse:\n\t\t\t\t\t\tscan_startip = num2ip(startip_num)\n\t\t\t\t\t\tendip_num = startip_num + process_nmap_count + remaining_count\n\t\t\t\t\t\tstartip_num = endip_num\n\t\t\t\t\t\t# target_option = '%s-%s' % (scan_startip, num2ip(endip_num))\n\t\t\t\t\t\ttarget_option = num2ip(endip_num)\n\t\t\t\t\t\ttarget_list.append(target_option)\n\t\t\telse:\n\t\t\t\tprint '* EndIP Less than StartIP'\n\t\telif endip_len == 4:\n\t\t\tif ip_check(endip):\n\t\t\t\tstartip_num = ip2num(startip)\n\t\t\t\tendip_num = ip2num(endip)\n\t\t\t\tif startip_num <= endip_num:\n\t\t\t\t\ttarget_count =  endip_num - startip_num\n\t\t\t\t\tif target_count >= 0:\n\t\t\t\t\t\ttarget_list.append(num2ip(startip_num))\n\t\t\t\t\t\t# 每10个IP分配到一个wyportmap子进程上\n\t\t\t\t\t\t# 更新：为每个独立的IP创建一个单独的任务，并删除掉开始IP，修改的时候注释掉上面那行\n\t\t\t\t\t\tfor i in xrange(0,(target_count+process_nmap_count-1)/process_nmap_count):\n\t\t\t\t\t\t\tip_count = (i * process_nmap_count) + process_nmap_count\n\t\t\t\t\t\t\tremaining_count = target_count - ip_count\n\t\t\t\t\t\t\tif remaining_count > 0:\n\t\t\t\t\t\t\t\tscan_startip = num2ip(startip_num)\n\t\t\t\t\t\t\t\tendip_num = startip_num + process_nmap_count\n\t\t\t\t\t\t\t\tstartip_num = endip_num\n\t\t\t\t\t\t\t\t# target_option = '%s-%s' % (scan_startip, num2ip(endip_num))\n\t\t\t\t\t\t\t\ttarget_option = num2ip(endip_num)\n\t\t\t\t\t\t\t\ttarget_list.append(target_option)\n\t\t\t\t\t\t\telse:\n\t\t\t\t\t\t\t\tscan_startip = num2ip(startip_num)\n\t\t\t\t\t\t\t\tendip_num = startip_num + process_nmap_count + remaining_count\n\t\t\t\t\t\t\t\tstartip_num = endip_num\n\t\t\t\t\t\t\t\t# target_option = '%s-%s' % (scan_startip, num2ip(endip_num))\n\t\t\t\t\t\t\t\ttarget_option = num2ip(endip_num)\n\t\t\t\t\t\t\t\ttarget_list.append(target_option)\n\t\t\t\telse:\n\t\t\t\t\tprint '* EndIP Less than StartIP'\n\t\t\telse:\n\t\t\t\tprint '* EndIP format error'\t\n\t\telse:\n\t\t\tprint '* EndIP format error'\n\n\treturn target_list\n\n\n"
  },
  {
    "path": "wyportmap.py",
    "content": "#!/usr/bin/env python\n# encoding: utf-8\n# mail: ringzero@0x557.org\n\nimport json\nimport sys\nfrom time import sleep\nfrom libnmap.process import NmapProcess\nfrom libnmap.reportjson import ReportDecoder, ReportEncoder\nfrom libnmap.parser import NmapParser, NmapParserException\nfrom libnmap.plugins.backendpluginFactory import BackendPluginFactory\n\n# 重试次数 & 超时时间(s)\nretrycnt = 3\ntimeout = 3600\n\n# 数据库连接 & 全局扫描参数\nglobal_dbcoon = 'mysql+mysqldb://celery:celery1@3Wscan@42.62.52.62:443/wscan'\n# global_dbcoon = 'mysql+mysqldb://用户名:密码@数据库服务器IP:数据库端口/数据库名称'\nglobal_options = '-sT -P0 -sV -O --script=banner -p T:21-25,80-89,110,143,443,513,873,1080,1433,1521,1158,3306-3308,3389,3690,5900,6379,7001,8000-8090,9000,9418,27017-27019,50060,111,11211,2049'\n\n# 处理端口状态\nglobal_log_states = ['open'] # open, filtered, closed, unfiltered\n\ndef do_nmap_scan(targets, options=global_options):\n\t# 运行次数初始化\n\ttrycnt = 0\n\n\twhile True:\n\t\t# 运行时间初始化\n\t\truntime = 0\n\n\t\tif trycnt >= retrycnt:\n\t\t\t# print '-' * 50\n\t\t\treturn 'retry overflow'\n\n\t\ttry:\n\t\t\tnmap_proc = NmapProcess(targets=targets, options=options, safe_mode=False)\n\t\t\tnmap_proc.run_background()\n\n\t\t\twhile nmap_proc.is_running():\n\t\t\t\tif runtime >= timeout:\t# 运行超时，结束掉任务，休息1分钟, 再重启这个nmap任务\n\t\t\t\t\t# print '-' * 50\n\t\t\t\t\t# print \"* timeout. terminate it...\"\n\t\t\t\t\tnmap_proc.stop()\n\t\t\t\t\t# 休眠时间\n\t\t\t\t\tsleep(60)\n\t\t\t\t\ttrycnt += 1\n\t\t\t\t\tbreak\n\t\t\t\telse:\n\t\t\t\t\t# print 'running[%ss]:%s' % (runtime, nmap_proc.command)\n\t\t\t\t\tsleep(5)\n\t\t\t\t\truntime += 5\n\t\t\tif nmap_proc.is_successful():\n\t\t\t\t# print '-' * 50\n\t\t\t\tprint nmap_proc.summary\n\t\t\t\treturn nmap_proc.stdout\n\n\t\texcept Exception, e:\n\t\t\t# raise e\n\t\t\tprint e\n\t\t\ttrycnt += 1\n\t\t\tif trycnt >= retrycnt:\n\t\t\t\t# print '-' * 50\n\t\t\t\t# print '* retry overflow'\n\t\t\t\treturn e\n\ndef parse_nmap_report(nmap_stdout, taskid=None):\n\ttry:\n\t\t# 处理结果并写入后台数据库\n\t\tnmap_report = NmapParser.parse(nmap_stdout)\n\n\t\t# 声明后台对应的ORM数据库处理模型\n\t\tmy_services_backend = BackendPluginFactory.create(plugin_name='backend_service', url=global_dbcoon, echo=False, encoding='utf-8', pool_timeout=3600)\n\t\tmy_hosts_backend = BackendPluginFactory.create(plugin_name='backend_host', url=global_dbcoon, echo=False, encoding='utf-8', pool_timeout=3600)\n\n\t\t# 开始处理扫描结果\n\t\tfor host in nmap_report.hosts:\n\n\t\t\t\t# print(\"Nmap scan : {0}\".format(host.address))\n\t\t\t\thost.taskid = taskid\n\n\t\t\t\t# 处理主机开放的服务和端口\n\t\t\t\tfor serv in host.services:\n\t\t\t\t\tserv.address = host.address\n\t\t\t\t\tserv.taskid = taskid\n\t\t\t\t\tserv.endtime = host.endtime\n\n\t\t\t\t\tif serv.state in global_log_states:\n\t\t\t\t\t\tserv.save(my_services_backend)\n\n\t\t\t\thost.save(my_hosts_backend)\n\n\t\treturn 'Scan finished'\n\n\texcept Exception, e:\n\t\t# 处理报表出错，返回错误结果\n\t\treturn e\n\ndef run_wyportmap(targets, taskid=None):\n\t# print '-' * 50\n\t# print '* Starting id:(%s) [%s] portmap scan' % (taskid, targets)\n\t# print '-' * 50\n\tnmap_result = do_nmap_scan(targets)\n\t# print '-' * 50\n\treturn parse_nmap_report(nmap_result,taskid)\n\nif __name__ == \"__main__\":\n\tif len(sys.argv) == 2:\n\t\tprint run_wyportmap(sys.argv[1])\n\t\tsys.exit(0)\n\telif len(sys.argv) == 3:\n\t\tprint run_wyportmap(sys.argv[1], sys.argv[2])\n\telse:\n\t\tprint (\"usage: %s targets taskid\" % sys.argv[0])\n\t\tsys.exit(-1)\n\n\n\n\n\n"
  }
]