Repository: smicallef/spiderfoot Branch: master Commit: 0f815a203afe Files: 795 Total size: 13.2 MB Directory structure: gitextract_l429po4y/ ├── .dockerignore ├── .github/ │ ├── ISSUE_TEMPLATE.md │ └── workflows/ │ ├── codeql-analysis.yml │ └── tests.yaml ├── .gitignore ├── .pylintrc ├── Dockerfile ├── Dockerfile.full ├── LICENSE ├── README.md ├── THANKYOU ├── VERSION ├── correlations/ │ ├── README.md │ ├── cert_expired.yaml │ ├── cloud_bucket_open.yaml │ ├── cloud_bucket_open_related.yaml │ ├── data_from_base64.yaml │ ├── data_from_docmeta.yaml │ ├── database_exposed.yaml │ ├── dev_or_test_system.yaml │ ├── dns_zone_transfer_possible.yaml │ ├── egress_ip_from_wikipedia.yaml │ ├── email_in_multiple_breaches.yaml │ ├── email_in_whois.yaml │ ├── email_only_from_pasteleak_site.yaml │ ├── host_only_from_bruteforce.yaml │ ├── host_only_from_certificatetransparency.yaml │ ├── http_errors.yaml │ ├── human_name_in_whois.yaml │ ├── internal_host.yaml │ ├── multiple_malicious.yaml │ ├── multiple_malicious_affiliate.yaml │ ├── multiple_malicious_cohost.yaml │ ├── name_only_from_pasteleak_site.yaml │ ├── open_port_version.yaml │ ├── outlier_cloud.yaml │ ├── outlier_country.yaml │ ├── outlier_email.yaml │ ├── outlier_hostname.yaml │ ├── outlier_ipaddress.yaml │ ├── outlier_registrar.yaml │ ├── outlier_webserver.yaml │ ├── remote_desktop_exposed.yaml │ ├── root_path_needs_auth.yaml │ ├── stale_host.yaml │ ├── strong_affiliate_certs.yaml │ ├── strong_similardomain_crossref.yaml │ ├── template.yaml │ ├── vulnerability_critical.yaml │ ├── vulnerability_high.yaml │ └── vulnerability_mediumlow.yaml ├── docker-compose-dev.yml ├── docker-compose-full.yml ├── docker-compose.yml ├── docs/ │ ├── Makefile │ ├── conf.py │ ├── index.rst │ ├── make.bat │ └── spiderfoot.rst ├── generate-certificate ├── modules/ │ ├── __init__.py │ ├── sfp__stor_db.py │ ├── sfp__stor_stdout.py │ ├── sfp_abstractapi.py │ ├── sfp_abusech.py │ ├── sfp_abuseipdb.py │ ├── sfp_abusix.py │ ├── sfp_accounts.py │ ├── sfp_adblock.py │ ├── sfp_adguard_dns.py │ ├── sfp_ahmia.py │ ├── sfp_alienvault.py │ ├── sfp_alienvaultiprep.py │ ├── sfp_apple_itunes.py │ ├── sfp_archiveorg.py │ ├── sfp_arin.py │ ├── sfp_azureblobstorage.py │ ├── sfp_base64.py │ ├── sfp_bgpview.py │ ├── sfp_binaryedge.py │ ├── sfp_bingsearch.py │ ├── sfp_bingsharedip.py │ ├── sfp_binstring.py │ ├── sfp_bitcoin.py │ ├── sfp_bitcoinabuse.py │ ├── sfp_bitcoinwhoswho.py │ ├── sfp_blockchain.py │ ├── sfp_blocklistde.py │ ├── sfp_botscout.py │ ├── sfp_botvrij.py │ ├── sfp_builtwith.py │ ├── sfp_c99.py │ ├── sfp_callername.py │ ├── sfp_censys.py │ ├── sfp_certspotter.py │ ├── sfp_cinsscore.py │ ├── sfp_circllu.py │ ├── sfp_citadel.py │ ├── sfp_cleanbrowsing.py │ ├── sfp_cleantalk.py │ ├── sfp_clearbit.py │ ├── sfp_cloudflaredns.py │ ├── sfp_coinblocker.py │ ├── sfp_commoncrawl.py │ ├── sfp_comodo.py │ ├── sfp_company.py │ ├── sfp_cookie.py │ ├── sfp_countryname.py │ ├── sfp_creditcard.py │ ├── sfp_crobat_api.py │ ├── sfp_crossref.py │ ├── sfp_crt.py │ ├── sfp_crxcavator.py │ ├── sfp_customfeed.py │ ├── sfp_cybercrimetracker.py │ ├── sfp_debounce.py │ ├── sfp_dehashed.py │ ├── sfp_digitaloceanspace.py │ ├── sfp_dns_for_family.py │ ├── sfp_dnsbrute.py │ ├── sfp_dnscommonsrv.py │ ├── sfp_dnsdb.py │ ├── sfp_dnsdumpster.py │ ├── sfp_dnsgrep.py │ ├── sfp_dnsneighbor.py │ ├── sfp_dnsraw.py │ ├── sfp_dnsresolve.py │ ├── sfp_dnszonexfer.py │ ├── sfp_dronebl.py │ ├── sfp_duckduckgo.py │ ├── sfp_email.py │ ├── sfp_emailcrawlr.py │ ├── sfp_emailformat.py │ ├── sfp_emailrep.py │ ├── sfp_emergingthreats.py │ ├── sfp_errors.py │ ├── sfp_ethereum.py │ ├── sfp_etherscan.py │ ├── sfp_filemeta.py │ ├── sfp_flickr.py │ ├── sfp_focsec.py │ ├── sfp_fortinet.py │ ├── sfp_fraudguard.py │ ├── sfp_fsecure_riddler.py │ ├── sfp_fullcontact.py │ ├── sfp_fullhunt.py │ ├── sfp_github.py │ ├── sfp_gleif.py │ ├── sfp_google_tag_manager.py │ ├── sfp_googlemaps.py │ ├── sfp_googleobjectstorage.py │ ├── sfp_googlesafebrowsing.py │ ├── sfp_googlesearch.py │ ├── sfp_gravatar.py │ ├── sfp_grayhatwarfare.py │ ├── sfp_greensnow.py │ ├── sfp_grep_app.py │ ├── sfp_greynoise.py │ ├── sfp_greynoise_community.py │ ├── sfp_h1nobbdde.py │ ├── sfp_hackertarget.py │ ├── sfp_hashes.py │ ├── sfp_haveibeenpwned.py │ ├── sfp_honeypot.py │ ├── sfp_hosting.py │ ├── sfp_hostio.py │ ├── sfp_hunter.py │ ├── sfp_hybrid_analysis.py │ ├── sfp_iban.py │ ├── sfp_iknowwhatyoudownload.py │ ├── sfp_intelx.py │ ├── sfp_intfiles.py │ ├── sfp_ipapico.py │ ├── sfp_ipapicom.py │ ├── sfp_ipinfo.py │ ├── sfp_ipqualityscore.py │ ├── sfp_ipregistry.py │ ├── sfp_ipstack.py │ ├── sfp_isc.py │ ├── sfp_jsonwhoiscom.py │ ├── sfp_junkfiles.py │ ├── sfp_keybase.py │ ├── sfp_koodous.py │ ├── sfp_leakix.py │ ├── sfp_maltiverse.py │ ├── sfp_malwarepatrol.py │ ├── sfp_metadefender.py │ ├── sfp_mnemonic.py │ ├── sfp_multiproxy.py │ ├── sfp_myspace.py │ ├── sfp_nameapi.py │ ├── sfp_names.py │ ├── sfp_networksdb.py │ ├── sfp_neutrinoapi.py │ ├── sfp_numverify.py │ ├── sfp_onioncity.py │ ├── sfp_onionsearchengine.py │ ├── sfp_onyphe.py │ ├── sfp_openbugbounty.py │ ├── sfp_opencorporates.py │ ├── sfp_opendns.py │ ├── sfp_opennic.py │ ├── sfp_openphish.py │ ├── sfp_openstreetmap.py │ ├── sfp_pageinfo.py │ ├── sfp_pastebin.py │ ├── sfp_pgp.py │ ├── sfp_phishstats.py │ ├── sfp_phishtank.py │ ├── sfp_phone.py │ ├── sfp_portscan_tcp.py │ ├── sfp_projectdiscovery.py │ ├── sfp_psbdmp.py │ ├── sfp_pulsedive.py │ ├── sfp_punkspider.py │ ├── sfp_quad9.py │ ├── sfp_reversewhois.py │ ├── sfp_ripe.py │ ├── sfp_riskiq.py │ ├── sfp_robtex.py │ ├── sfp_s3bucket.py │ ├── sfp_searchcode.py │ ├── sfp_securitytrails.py │ ├── sfp_seon.py │ ├── sfp_shodan.py │ ├── sfp_similar.py │ ├── sfp_skymem.py │ ├── sfp_slideshare.py │ ├── sfp_snov.py │ ├── sfp_social.py │ ├── sfp_sociallinks.py │ ├── sfp_socialprofiles.py │ ├── sfp_sorbs.py │ ├── sfp_spamcop.py │ ├── sfp_spamhaus.py │ ├── sfp_spider.py │ ├── sfp_spur.py │ ├── sfp_spyonweb.py │ ├── sfp_sslcert.py │ ├── sfp_stackoverflow.py │ ├── sfp_stevenblack_hosts.py │ ├── sfp_strangeheaders.py │ ├── sfp_subdomain_takeover.py │ ├── sfp_sublist3r.py │ ├── sfp_surbl.py │ ├── sfp_talosintel.py │ ├── sfp_template.py │ ├── sfp_textmagic.py │ ├── sfp_threatcrowd.py │ ├── sfp_threatfox.py │ ├── sfp_threatjammer.py │ ├── sfp_threatminer.py │ ├── sfp_tldsearch.py │ ├── sfp_tool_cmseek.py │ ├── sfp_tool_dnstwist.py │ ├── sfp_tool_nbtscan.py │ ├── sfp_tool_nmap.py │ ├── sfp_tool_nuclei.py │ ├── sfp_tool_onesixtyone.py │ ├── sfp_tool_retirejs.py │ ├── sfp_tool_snallygaster.py │ ├── sfp_tool_testsslsh.py │ ├── sfp_tool_trufflehog.py │ ├── sfp_tool_wafw00f.py │ ├── sfp_tool_wappalyzer.py │ ├── sfp_tool_whatweb.py │ ├── sfp_torch.py │ ├── sfp_torexits.py │ ├── sfp_trashpanda.py │ ├── sfp_trumail.py │ ├── sfp_twilio.py │ ├── sfp_twitter.py │ ├── sfp_uceprotect.py │ ├── sfp_urlscan.py │ ├── sfp_venmo.py │ ├── sfp_viewdns.py │ ├── sfp_virustotal.py │ ├── sfp_voipbl.py │ ├── sfp_vxvault.py │ ├── sfp_webanalytics.py │ ├── sfp_webframework.py │ ├── sfp_webserver.py │ ├── sfp_whatcms.py │ ├── sfp_whois.py │ ├── sfp_whoisology.py │ ├── sfp_whoxy.py │ ├── sfp_wigle.py │ ├── sfp_wikileaks.py │ ├── sfp_wikipediaedits.py │ ├── sfp_xforce.py │ ├── sfp_yandexdns.py │ ├── sfp_zetalytics.py │ ├── sfp_zonefiles.py │ └── sfp_zoneh.py ├── requirements.txt ├── setup.cfg ├── sf.py ├── sfcli.py ├── sflib.py ├── sfscan.py ├── sfwebui.py ├── spiderfoot/ │ ├── __init__.py │ ├── __version__.py │ ├── correlation.py │ ├── db.py │ ├── dicts/ │ │ ├── __init__.py │ │ ├── generic-usernames.txt │ │ ├── ispell/ │ │ │ ├── LICENSE │ │ │ ├── __init__.py │ │ │ ├── english.dict │ │ │ ├── french.dict │ │ │ ├── german.dict │ │ │ ├── names.dict │ │ │ └── spanish.dict │ │ ├── subdomains-10000.txt │ │ └── subdomains.txt │ ├── event.py │ ├── helpers.py │ ├── logger.py │ ├── plugin.py │ ├── static/ │ │ ├── css/ │ │ │ ├── dark.css │ │ │ └── spiderfoot.css │ │ ├── js/ │ │ │ ├── spiderfoot.js │ │ │ ├── spiderfoot.newscan.js │ │ │ ├── spiderfoot.opts.js │ │ │ ├── spiderfoot.scanlist.js │ │ │ └── viz.js │ │ └── package.json │ ├── target.py │ ├── templates/ │ │ ├── FOOTER.tmpl │ │ ├── HEADER.tmpl │ │ ├── error.tmpl │ │ ├── newscan.tmpl │ │ ├── opts.tmpl │ │ ├── scaninfo.tmpl │ │ └── scanlist.tmpl │ └── threadpool.py └── test/ ├── README.md ├── __init__.py ├── acceptance/ │ ├── requirements.txt │ ├── run │ └── scan.robot ├── bandit ├── conftest.py ├── integration/ │ ├── __init__.py │ ├── modules/ │ │ ├── __init__.py │ │ ├── test_sfp__stor_db.py │ │ ├── test_sfp__stor_stdout.py │ │ ├── test_sfp_abstractapi.py │ │ ├── test_sfp_abusech.py │ │ ├── test_sfp_abuseipdb.py │ │ ├── test_sfp_abusix.py │ │ ├── test_sfp_accounts.py │ │ ├── test_sfp_adblock.py │ │ ├── test_sfp_adguard_dns.py │ │ ├── test_sfp_ahmia.py │ │ ├── test_sfp_alienvault.py │ │ ├── test_sfp_alienvaultiprep.py │ │ ├── test_sfp_apple_itunes.py │ │ ├── test_sfp_archiveorg.py │ │ ├── test_sfp_arin.py │ │ ├── test_sfp_azureblobstorage.py │ │ ├── test_sfp_bgpview.py │ │ ├── test_sfp_binaryedge.py │ │ ├── test_sfp_bingsearch.py │ │ ├── test_sfp_bingsharedip.py │ │ ├── test_sfp_bitcoinabuse.py │ │ ├── test_sfp_bitcoinwhoswho.py │ │ ├── test_sfp_blockchain.py │ │ ├── test_sfp_blocklistde.py │ │ ├── test_sfp_botscout.py │ │ ├── test_sfp_botvrij.py │ │ ├── test_sfp_builtwith.py │ │ ├── test_sfp_c99.py │ │ ├── test_sfp_callername.py │ │ ├── test_sfp_censys.py │ │ ├── test_sfp_certspotter.py │ │ ├── test_sfp_cinsscore.py │ │ ├── test_sfp_circllu.py │ │ ├── test_sfp_citadel.py │ │ ├── test_sfp_cleanbrowsing.py │ │ ├── test_sfp_cleantalk.py │ │ ├── test_sfp_clearbit.py │ │ ├── test_sfp_cloudflaredns.py │ │ ├── test_sfp_coinblocker.py │ │ ├── test_sfp_commoncrawl.py │ │ ├── test_sfp_comodo.py │ │ ├── test_sfp_crobat_api.py │ │ ├── test_sfp_crossref.py │ │ ├── test_sfp_crt.py │ │ ├── test_sfp_crxcavator.py │ │ ├── test_sfp_customfeed.py │ │ ├── test_sfp_cybercrimetracker.py │ │ ├── test_sfp_debounce.py │ │ ├── test_sfp_dehashed.py │ │ ├── test_sfp_digitaloceanspace.py │ │ ├── test_sfp_dns_for_family.py │ │ ├── test_sfp_dnsbrute.py │ │ ├── test_sfp_dnscommonsrv.py │ │ ├── test_sfp_dnsdb.py │ │ ├── test_sfp_dnsdumpster.py │ │ ├── test_sfp_dnsgrep.py │ │ ├── test_sfp_dnsneighbor.py │ │ ├── test_sfp_dnsraw.py │ │ ├── test_sfp_dnsresolve.py │ │ ├── test_sfp_dnszonexfer.py │ │ ├── test_sfp_dronebl.py │ │ ├── test_sfp_duckduckgo.py │ │ ├── test_sfp_email.py │ │ ├── test_sfp_emailcrawlr.py │ │ ├── test_sfp_emailformat.py │ │ ├── test_sfp_emailrep.py │ │ ├── test_sfp_emergingthreats.py │ │ ├── test_sfp_etherscan.py │ │ ├── test_sfp_filemeta.py │ │ ├── test_sfp_flickr.py │ │ ├── test_sfp_focsec.py │ │ ├── test_sfp_fortinet.py │ │ ├── test_sfp_fraudguard.py │ │ ├── test_sfp_fsecure_riddler.py │ │ ├── test_sfp_fullcontact.py │ │ ├── test_sfp_fullhunt.py │ │ ├── test_sfp_github.py │ │ ├── test_sfp_gleif.py │ │ ├── test_sfp_google_tag_manager.py │ │ ├── test_sfp_googlemaps.py │ │ ├── test_sfp_googleobjectstorage.py │ │ ├── test_sfp_googlesafebrowsing.py │ │ ├── test_sfp_googlesearch.py │ │ ├── test_sfp_gravatar.py │ │ ├── test_sfp_grayhatwarfare.py │ │ ├── test_sfp_greensnow.py │ │ ├── test_sfp_grep_app.py │ │ ├── test_sfp_greynoise.py │ │ ├── test_sfp_h1nobbdde.py │ │ ├── test_sfp_hackertarget.py │ │ ├── test_sfp_haveibeenpwned.py │ │ ├── test_sfp_honeypot.py │ │ ├── test_sfp_hosting.py │ │ ├── test_sfp_hostio.py │ │ ├── test_sfp_hunter.py │ │ ├── test_sfp_hybrid_analysis.py │ │ ├── test_sfp_iknowwhatyoudownload.py │ │ ├── test_sfp_intelx.py │ │ ├── test_sfp_ipapico.py │ │ ├── test_sfp_ipapicom.py │ │ ├── test_sfp_ipinfo.py │ │ ├── test_sfp_ipqualityscore.py │ │ ├── test_sfp_ipregistry.py │ │ ├── test_sfp_ipstack.py │ │ ├── test_sfp_isc.py │ │ ├── test_sfp_jsonwhoiscom.py │ │ ├── test_sfp_junkfiles.py │ │ ├── test_sfp_keybase.py │ │ ├── test_sfp_koodous.py │ │ ├── test_sfp_leakix.py │ │ ├── test_sfp_maltiverse.py │ │ ├── test_sfp_malwarepatrol.py │ │ ├── test_sfp_metadefender.py │ │ ├── test_sfp_mnemonic.py │ │ ├── test_sfp_multiproxy.py │ │ ├── test_sfp_myspace.py │ │ ├── test_sfp_nameapi.py │ │ ├── test_sfp_networksdb.py │ │ ├── test_sfp_neutrinoapi.py │ │ ├── test_sfp_numverify.py │ │ ├── test_sfp_onioncity.py │ │ ├── test_sfp_onionsearchengine.py │ │ ├── test_sfp_onyphe.py │ │ ├── test_sfp_openbugbounty.py │ │ ├── test_sfp_opencorporates.py │ │ ├── test_sfp_opendns.py │ │ ├── test_sfp_opennic.py │ │ ├── test_sfp_openphish.py │ │ ├── test_sfp_openstreetmap.py │ │ ├── test_sfp_pageinfo.py │ │ ├── test_sfp_pastebin.py │ │ ├── test_sfp_pgp.py │ │ ├── test_sfp_phishstats.py │ │ ├── test_sfp_phishtank.py │ │ ├── test_sfp_portscan_tcp.py │ │ ├── test_sfp_projectdiscovery.py │ │ ├── test_sfp_psbdmp.py │ │ ├── test_sfp_pulsedive.py │ │ ├── test_sfp_punkspider.py │ │ ├── test_sfp_quad9.py │ │ ├── test_sfp_reversewhois.py │ │ ├── test_sfp_ripe.py │ │ ├── test_sfp_riskiq.py │ │ ├── test_sfp_robtex.py │ │ ├── test_sfp_s3bucket.py │ │ ├── test_sfp_searchcode.py │ │ ├── test_sfp_securitytrails.py │ │ ├── test_sfp_seon.py │ │ ├── test_sfp_shodan.py │ │ ├── test_sfp_similar.py │ │ ├── test_sfp_skymem.py │ │ ├── test_sfp_slideshare.py │ │ ├── test_sfp_snov.py │ │ ├── test_sfp_sociallinks.py │ │ ├── test_sfp_socialprofiles.py │ │ ├── test_sfp_sorbs.py │ │ ├── test_sfp_spamcop.py │ │ ├── test_sfp_spamhaus.py │ │ ├── test_sfp_spider.py │ │ ├── test_sfp_spur.py │ │ ├── test_sfp_spyonweb.py │ │ ├── test_sfp_sslcert.py │ │ ├── test_sfp_stackoverflow.py │ │ ├── test_sfp_stevenblack_hosts.py │ │ ├── test_sfp_subdomain_takeover.py │ │ ├── test_sfp_sublist3r.py │ │ ├── test_sfp_surbl.py │ │ ├── test_sfp_talosintel.py │ │ ├── test_sfp_textmagic.py │ │ ├── test_sfp_threatcrowd.py │ │ ├── test_sfp_threatfox.py │ │ ├── test_sfp_threatminer.py │ │ ├── test_sfp_tldsearch.py │ │ ├── test_sfp_torch.py │ │ ├── test_sfp_torexits.py │ │ ├── test_sfp_trashpanda.py │ │ ├── test_sfp_trumail.py │ │ ├── test_sfp_twilio.py │ │ ├── test_sfp_twitter.py │ │ ├── test_sfp_uceprotect.py │ │ ├── test_sfp_urlscan.py │ │ ├── test_sfp_venmo.py │ │ ├── test_sfp_viewdns.py │ │ ├── test_sfp_virustotal.py │ │ ├── test_sfp_voipbl.py │ │ ├── test_sfp_vxvault.py │ │ ├── test_sfp_webserver.py │ │ ├── test_sfp_whatcms.py │ │ ├── test_sfp_whois.py │ │ ├── test_sfp_whoisology.py │ │ ├── test_sfp_whoxy.py │ │ ├── test_sfp_wigle.py │ │ ├── test_sfp_wikileaks.py │ │ ├── test_sfp_wikipediaedits.py │ │ ├── test_sfp_xforce.py │ │ ├── test_sfp_yandexdns.py │ │ ├── test_sfp_zetalytics.py │ │ └── test_sfp_zoneh.py │ ├── test_sf.py │ ├── test_sfcli.py │ └── test_sfwebui.py ├── requirements.txt ├── run ├── unit/ │ ├── __init__.py │ ├── modules/ │ │ ├── __init__.py │ │ ├── test_sfp__stor_db.py │ │ ├── test_sfp__stor_stdout.py │ │ ├── test_sfp_abstractapi.py │ │ ├── test_sfp_abusech.py │ │ ├── test_sfp_abuseipdb.py │ │ ├── test_sfp_abusix.py │ │ ├── test_sfp_accounts.py │ │ ├── test_sfp_adblock.py │ │ ├── test_sfp_adguard_dns.py │ │ ├── test_sfp_ahmia.py │ │ ├── test_sfp_alienvault.py │ │ ├── test_sfp_alienvaultiprep.py │ │ ├── test_sfp_apple_itunes.py │ │ ├── test_sfp_archiveorg.py │ │ ├── test_sfp_arin.py │ │ ├── test_sfp_azureblobstorage.py │ │ ├── test_sfp_base64.py │ │ ├── test_sfp_bgpview.py │ │ ├── test_sfp_binaryedge.py │ │ ├── test_sfp_bingsearch.py │ │ ├── test_sfp_bingsharedip.py │ │ ├── test_sfp_binstring.py │ │ ├── test_sfp_bitcoin.py │ │ ├── test_sfp_bitcoinabuse.py │ │ ├── test_sfp_bitcoinwhoswho.py │ │ ├── test_sfp_blockchain.py │ │ ├── test_sfp_blocklistde.py │ │ ├── test_sfp_botscout.py │ │ ├── test_sfp_botvrij.py │ │ ├── test_sfp_builtwith.py │ │ ├── test_sfp_c99.py │ │ ├── test_sfp_callername.py │ │ ├── test_sfp_censys.py │ │ ├── test_sfp_certspotter.py │ │ ├── test_sfp_cinsscore.py │ │ ├── test_sfp_circllu.py │ │ ├── test_sfp_citadel.py │ │ ├── test_sfp_cleanbrowsing.py │ │ ├── test_sfp_cleantalk.py │ │ ├── test_sfp_clearbit.py │ │ ├── test_sfp_cloudflaredns.py │ │ ├── test_sfp_coinblocker.py │ │ ├── test_sfp_commoncrawl.py │ │ ├── test_sfp_comodo.py │ │ ├── test_sfp_company.py │ │ ├── test_sfp_cookie.py │ │ ├── test_sfp_countryname.py │ │ ├── test_sfp_creditcard.py │ │ ├── test_sfp_crobat_api.py │ │ ├── test_sfp_crossref.py │ │ ├── test_sfp_crt.py │ │ ├── test_sfp_crxcavator.py │ │ ├── test_sfp_customfeed.py │ │ ├── test_sfp_cybercrimetracker.py │ │ ├── test_sfp_debounce.py │ │ ├── test_sfp_dehashed.py │ │ ├── test_sfp_digitaloceanspace.py │ │ ├── test_sfp_dns_for_family.py │ │ ├── test_sfp_dnsbrute.py │ │ ├── test_sfp_dnscommonsrv.py │ │ ├── test_sfp_dnsdb.py │ │ ├── test_sfp_dnsdumpster.py │ │ ├── test_sfp_dnsgrep.py │ │ ├── test_sfp_dnsneighbor.py │ │ ├── test_sfp_dnsraw.py │ │ ├── test_sfp_dnsresolve.py │ │ ├── test_sfp_dnszonexfer.py │ │ ├── test_sfp_dronebl.py │ │ ├── test_sfp_duckduckgo.py │ │ ├── test_sfp_email.py │ │ ├── test_sfp_emailcrawlr.py │ │ ├── test_sfp_emailformat.py │ │ ├── test_sfp_emailrep.py │ │ ├── test_sfp_emergingthreats.py │ │ ├── test_sfp_errors.py │ │ ├── test_sfp_ethereum.py │ │ ├── test_sfp_etherscan.py │ │ ├── test_sfp_filemeta.py │ │ ├── test_sfp_flickr.py │ │ ├── test_sfp_focsec.py │ │ ├── test_sfp_fortinet.py │ │ ├── test_sfp_fraudguard.py │ │ ├── test_sfp_fsecure_riddler.py │ │ ├── test_sfp_fullcontact.py │ │ ├── test_sfp_fullhunt.py │ │ ├── test_sfp_github.py │ │ ├── test_sfp_gleif.py │ │ ├── test_sfp_google_tag_manager.py │ │ ├── test_sfp_googlemaps.py │ │ ├── test_sfp_googleobjectstorage.py │ │ ├── test_sfp_googlesafebrowsing.py │ │ ├── test_sfp_googlesearch.py │ │ ├── test_sfp_gravatar.py │ │ ├── test_sfp_grayhatwarfare.py │ │ ├── test_sfp_greensnow.py │ │ ├── test_sfp_grep_app.py │ │ ├── test_sfp_greynoise.py │ │ ├── test_sfp_h1nobbdde.py │ │ ├── test_sfp_hackertarget.py │ │ ├── test_sfp_hashes.py │ │ ├── test_sfp_haveibeenpwned.py │ │ ├── test_sfp_honeypot.py │ │ ├── test_sfp_hosting.py │ │ ├── test_sfp_hostio.py │ │ ├── test_sfp_hunter.py │ │ ├── test_sfp_hybrid_analysis.py │ │ ├── test_sfp_iban.py │ │ ├── test_sfp_iknowwhatyoudownload.py │ │ ├── test_sfp_intelx.py │ │ ├── test_sfp_intfiles.py │ │ ├── test_sfp_ipapico.py │ │ ├── test_sfp_ipapicom.py │ │ ├── test_sfp_ipinfo.py │ │ ├── test_sfp_ipqualityscore.py │ │ ├── test_sfp_ipregistry.py │ │ ├── test_sfp_ipstack.py │ │ ├── test_sfp_isc.py │ │ ├── test_sfp_jsonwhoiscom.py │ │ ├── test_sfp_junkfiles.py │ │ ├── test_sfp_keybase.py │ │ ├── test_sfp_koodous.py │ │ ├── test_sfp_leakix.py │ │ ├── test_sfp_maltiverse.py │ │ ├── test_sfp_malwarepatrol.py │ │ ├── test_sfp_metadefender.py │ │ ├── test_sfp_mnemonic.py │ │ ├── test_sfp_multiproxy.py │ │ ├── test_sfp_myspace.py │ │ ├── test_sfp_nameapi.py │ │ ├── test_sfp_names.py │ │ ├── test_sfp_networksdb.py │ │ ├── test_sfp_neutrinoapi.py │ │ ├── test_sfp_numverify.py │ │ ├── test_sfp_onioncity.py │ │ ├── test_sfp_onionsearchengine.py │ │ ├── test_sfp_onyphe.py │ │ ├── test_sfp_openbugbounty.py │ │ ├── test_sfp_opencorporates.py │ │ ├── test_sfp_opendns.py │ │ ├── test_sfp_opennic.py │ │ ├── test_sfp_openphish.py │ │ ├── test_sfp_openstreetmap.py │ │ ├── test_sfp_pageinfo.py │ │ ├── test_sfp_pastebin.py │ │ ├── test_sfp_pgp.py │ │ ├── test_sfp_phishstats.py │ │ ├── test_sfp_phishtank.py │ │ ├── test_sfp_phone.py │ │ ├── test_sfp_portscan_tcp.py │ │ ├── test_sfp_projectdiscovery.py │ │ ├── test_sfp_psbdmp.py │ │ ├── test_sfp_pulsedive.py │ │ ├── test_sfp_punkspider.py │ │ ├── test_sfp_quad9.py │ │ ├── test_sfp_reversewhois.py │ │ ├── test_sfp_ripe.py │ │ ├── test_sfp_riskiq.py │ │ ├── test_sfp_robtex.py │ │ ├── test_sfp_s3bucket.py │ │ ├── test_sfp_searchcode.py │ │ ├── test_sfp_securitytrails.py │ │ ├── test_sfp_seon.py │ │ ├── test_sfp_shodan.py │ │ ├── test_sfp_similar.py │ │ ├── test_sfp_skymem.py │ │ ├── test_sfp_slideshare.py │ │ ├── test_sfp_snov.py │ │ ├── test_sfp_social.py │ │ ├── test_sfp_sociallinks.py │ │ ├── test_sfp_socialprofiles.py │ │ ├── test_sfp_sorbs.py │ │ ├── test_sfp_spamcop.py │ │ ├── test_sfp_spamhaus.py │ │ ├── test_sfp_spider.py │ │ ├── test_sfp_spur.py │ │ ├── test_sfp_spyonweb.py │ │ ├── test_sfp_sslcert.py │ │ ├── test_sfp_stackoverflow.py │ │ ├── test_sfp_stevenblack_hosts.py │ │ ├── test_sfp_strangeheaders.py │ │ ├── test_sfp_subdomain_takeover.py │ │ ├── test_sfp_sublist3r.py │ │ ├── test_sfp_surbl.py │ │ ├── test_sfp_talosintel.py │ │ ├── test_sfp_template.py │ │ ├── test_sfp_textmagic.py │ │ ├── test_sfp_threatcrowd.py │ │ ├── test_sfp_threatfox.py │ │ ├── test_sfp_threatminer.py │ │ ├── test_sfp_tldsearch.py │ │ ├── test_sfp_tool_cmseek.py │ │ ├── test_sfp_tool_dnstwist.py │ │ ├── test_sfp_tool_nbtscan.py │ │ ├── test_sfp_tool_nmap.py │ │ ├── test_sfp_tool_nuclei.py │ │ ├── test_sfp_tool_onesixtyone.py │ │ ├── test_sfp_tool_retirejs.py │ │ ├── test_sfp_tool_snallygaster.py │ │ ├── test_sfp_tool_testsslsh.py │ │ ├── test_sfp_tool_trufflehog.py │ │ ├── test_sfp_tool_wafw00f.py │ │ ├── test_sfp_tool_wappalyzer.py │ │ ├── test_sfp_tool_whatweb.py │ │ ├── test_sfp_torch.py │ │ ├── test_sfp_torexits.py │ │ ├── test_sfp_trashpanda.py │ │ ├── test_sfp_trumail.py │ │ ├── test_sfp_twilio.py │ │ ├── test_sfp_twitter.py │ │ ├── test_sfp_uceprotect.py │ │ ├── test_sfp_urlscan.py │ │ ├── test_sfp_venmo.py │ │ ├── test_sfp_viewdns.py │ │ ├── test_sfp_virustotal.py │ │ ├── test_sfp_voipbl.py │ │ ├── test_sfp_vxvault.py │ │ ├── test_sfp_webanalytics.py │ │ ├── test_sfp_webframework.py │ │ ├── test_sfp_webserver.py │ │ ├── test_sfp_whatcms.py │ │ ├── test_sfp_whois.py │ │ ├── test_sfp_whoisology.py │ │ ├── test_sfp_whoxy.py │ │ ├── test_sfp_wigle.py │ │ ├── test_sfp_wikileaks.py │ │ ├── test_sfp_wikipediaedits.py │ │ ├── test_sfp_xforce.py │ │ ├── test_sfp_yandexdns.py │ │ ├── test_sfp_zetalytics.py │ │ ├── test_sfp_zonefiles.py │ │ └── test_sfp_zoneh.py │ ├── spiderfoot/ │ │ ├── test_spiderfootcorrelator.py │ │ ├── test_spiderfootdb.py │ │ ├── test_spiderfootevent.py │ │ ├── test_spiderfoothelpers.py │ │ ├── test_spiderfootplugin.py │ │ ├── test_spiderfoottarget.py │ │ └── test_spiderfootthreadpool.py │ ├── test_modules.py │ ├── test_spiderfoot.py │ ├── test_spiderfootcli.py │ ├── test_spiderfootscanner.py │ └── test_spiderfootwebui.py └── update-requirements ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ .git/ build dist *.egg-info *.egg/ *.pyc *.swp venv/ cache/ log/ spiderfoot.db* spiderfoot.test.db __pycache__ htmlcov/ docs/ ================================================ FILE: .github/ISSUE_TEMPLATE.md ================================================ Please provide a description of the issue and any relevant error messages. If you can provide an application stack trace that is even better. What version of Python are you using? What version of SpiderFoot are you using (stable release or Git `master` branch)? You may also wish to check if your issue has been posted previously: * https://github.com/smicallef/spiderfoot/issues ================================================ FILE: .github/workflows/codeql-analysis.yml ================================================ name: "CodeQL" on: push: branches: - master schedule: - cron: "26 16 * * 0" jobs: analyze: name: Analyze runs-on: ubuntu-latest strategy: fail-fast: true matrix: language: ["python"] steps: - name: Checkout repository uses: actions/checkout@v2 - name: Initialize CodeQL uses: github/codeql-action/init@v1 with: languages: ${{ matrix.language }} - name: Autobuild uses: github/codeql-action/autobuild@v1 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v1 ================================================ FILE: .github/workflows/tests.yaml ================================================ name: Tests on: push: branches: master pull_request: branches: master jobs: build: runs-on: ${{ matrix.os }} strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] os: [ubuntu-latest, macos-latest] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install flake8 pytest pip install -r test/requirements.txt - name: Lint with flake8 run: | # stop the build if there are Python Flake8 violations flake8 . --count --show-source --statistics - name: Test with pytest run: | python -m pytest -n auto --dist loadfile --ignore=test/integration/modules/ --durations=5 --cov-report term --cov=. . - uses: codecov/codecov-action@v1 with: fail_ci_if_error: false verbose: true ================================================ FILE: .gitignore ================================================ # SpiderFoot specific # ####################### /cache/ /log/ /spiderfoot.db* passwd spiderfoot.crt spiderfoot.key spiderfoot/static/node_modules/* spiderfoot/static/node_modules/alertifyjs/* spiderfoot/static/node_modules/alertifyjs/build/* spiderfoot/static/node_modules/alertifyjs/build/css/* !spiderfoot/static/node_modules/alertifyjs/ !spiderfoot/static/node_modules/alertifyjs/build/ !spiderfoot/static/node_modules/alertifyjs/build/alertify.min.js !spiderfoot/static/node_modules/alertifyjs/build/css/ !spiderfoot/static/node_modules/alertifyjs/build/css/alertify.min.css spiderfoot/static/node_modules/bootstrap/* spiderfoot/static/node_modules/bootstrap/dist/* spiderfoot/static/node_modules/bootstrap/dist/js/* spiderfoot/static/node_modules/bootstrap/dist/css/* spiderfoot/static/node_modules/bootstrap/dist/fonts/* !spiderfoot/static/node_modules/bootstrap/ !spiderfoot/static/node_modules/bootstrap/dist/ !spiderfoot/static/node_modules/bootstrap/dist/js/ !spiderfoot/static/node_modules/bootstrap/dist/js/bootstrap.min.js !spiderfoot/static/node_modules/bootstrap/dist/css/ !spiderfoot/static/node_modules/bootstrap/dist/css/bootstrap.min.css !spiderfoot/static/node_modules/bootstrap/dist/fonts/ !spiderfoot/static/node_modules/bootstrap/dist/fonts/glyphicons-halflings-regular.woff !spiderfoot/static/node_modules/bootstrap/dist/fonts/glyphicons-halflings-regular.woff2 spiderfoot/static/node_modules/jquery/* spiderfoot/static/node_modules/jquery/dist/* !spiderfoot/static/node_modules/jquery/ !spiderfoot/static/node_modules/jquery/dist/ !spiderfoot/static/node_modules/jquery/dist/jquery.min.js spiderfoot/static/node_modules/sigma/* spiderfoot/static/node_modules/sigma/build/* spiderfoot/static/node_modules/sigma/build/plugins/* !spiderfoot/static/node_modules/sigma/ !spiderfoot/static/node_modules/sigma/build/ !spiderfoot/static/node_modules/sigma/build/sigma.min.js !spiderfoot/static/node_modules/sigma/build/plugins/ !spiderfoot/static/node_modules/sigma/build/plugins/sigma.parsers.json.min.js !spiderfoot/static/node_modules/sigma/build/plugins/sigma.plugins.dragNodes.min.js !spiderfoot/static/node_modules/sigma/build/plugins/sigma.layout.forceAtlas2.min.js !spiderfoot/static/node_modules/sigma/build/plugins/sigma.renderers.snapshot.min.js spiderfoot/static/node_modules/tablesorter/* spiderfoot/static/node_modules/tablesorter/dist/* spiderfoot/static/node_modules/tablesorter/dist/js/* spiderfoot/static/node_modules/tablesorter/dist/css/* spiderfoot/static/node_modules/tablesorter/dist/js/extras/* !spiderfoot/static/node_modules/tablesorter/ !spiderfoot/static/node_modules/tablesorter/dist/ !spiderfoot/static/node_modules/tablesorter/dist/css/ !spiderfoot/static/node_modules/tablesorter/dist/css/jquery.tablesorter.pager.min.css !spiderfoot/static/node_modules/tablesorter/dist/css/theme.default.min.css !spiderfoot/static/node_modules/tablesorter/dist/js/ !spiderfoot/static/node_modules/tablesorter/dist/js/jquery.tablesorter.min.js !spiderfoot/static/node_modules/tablesorter/dist/js/jquery.tablesorter.widgets.min.js !spiderfoot/static/node_modules/tablesorter/dist/js/extras/ !spiderfoot/static/node_modules/tablesorter/dist/js/extras/jquery.tablesorter.pager.min.js spiderfoot/static/node_modules/d3/* !spiderfoot/static/node_modules/d3/ !spiderfoot/static/node_modules/d3/d3.min.js # Compiled source # ################### __pycache__/ *.com *.class *.dll *.exe *.o *.so *.py[cod] *$py.class # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # Packages # ############ # it's better to unpack these files and commit the raw source # git has its own built in compression methods *.7z *.dmg *.gz *.iso *.jar *.rar *.tar *.zip # Logs and databases # ###################### *.log *.sqlite *.db # OS generated files # ###################### .DS_Store .DS_Store? ._* .Spotlight-V100 .Trashes .*~ Icon? ehthumbs.db Thumbs.db # Virtual environments # ######################## venv/ venv3/ # IDE # ####### .idea .vscode # Test & Coverage Reports # ########################### .coverage coverage/ test/acceptance/results/ htmlcov/ report/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Documentation # ################# docs/_build/ ================================================ FILE: .pylintrc ================================================ [MASTER] # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code extension-pkg-whitelist= # Add files or directories to the blacklist. They should be base names, not # paths. ignore=CVS # Add files or directories matching the regex patterns to the blacklist. The # regex matches against base names, not paths. ignore-patterns= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= # Use multiple processes to speed up Pylint. jobs=4 # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. load-plugins= # Pickle collected data for later comparisons. persistent=yes # Specify a configuration file. #rcfile= # When enabled, pylint would attempt to guess common misconfiguration and emit # user-friendly hints instead of false-positive error messages suggestion-mode=yes # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no [MESSAGES CONTROL] # Only show warnings with the listed confidence levels. Leave empty to show # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED confidence= # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once).You can also use "--disable=all" to # disable everything first and then reenable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" disable=invalid-name, no-self-use, too-few-public-methods, too-many-ancestors, too-many-arguments, too-many-boolean-expressions, too-many-branches, too-many-instance-attributes, too-many-locals, too-many-nested-blocks, too-many-public-methods, too-many-return-statements, too-many-statements, # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where # it should appear only once). See also the "--disable" option for examples. enable=c-extension-no-member [REPORTS] # Python expression which should return a note less than 10 (10 is the highest # note). You have access to the variables errors warning, statement which # respectively contain the number of errors / warnings messages and the total # number of statements analyzed. This is used by the global evaluation report # (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) # Template used to display messages. This is a python new-style format string # used to format the message information. See doc for all details #msg-template= # Set the output format. Available formats are text, parseable, colorized, json # and msvs (visual studio).You can also give a reporter class, eg # mypackage.mymodule.MyReporterClass. #output-format=text output-format=parseable # Tells whether to display a full report or only the messages reports=no # Activate the evaluation score. score=yes [REFACTORING] # Maximum number of nested blocks for function / method body max-nested-blocks=5 # Complete name of functions that never returns. When checking for # inconsistent-return-statements if a never returning function is called then # it will be considered as an explicit return statement and no message will be # printed. never-returning-functions=optparse.Values,sys.exit [BASIC] # Regular expression matching correct argument names. Overrides argument- # naming-style argument-rgx=^[a-z][a-z0-9_]*$ # Regular expression matching correct attribute names. Overrides attr-naming- # style attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ # Bad variable names which should always be refused, separated by a comma bad-names= # Regular expression matching correct class attribute names. Overrides class- # attribute-naming-style class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ # Regular expression matching correct class names. Overrides class-naming-style class-rgx=^_?[A-Z][a-zA-Z0-9]*$ # Naming style matching correct constant names const-naming-style=UPPER_CASE # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. docstring-min-length=-1 # Regular expression matching correct function names. Overrides function- # naming-style function-rgx=^(?:(?PsetUp|tearDown|setUpModule|tearDownModule)|(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ # Good variable names which should always be accepted, separated by a comma good-names=main, sf, _ # Include a hint for the correct naming format with invalid-name include-naming-hint=no # Regular expression matching correct inline iteration names. Overrides # inlinevar-naming-style inlinevar-rgx=^[a-z][a-z0-9_]*$ # Regular expression matching correct method names. Overrides method-naming- # style method-rgx=(?x)^(?:(?P_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ # Regular expression matching correct module names. Overrides module-naming- # style module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$ # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= # Regular expression which should only match function or class names that do # not require a docstring. no-docstring-rgx=^_ # List of decorators that produce properties, such as abc.abstractproperty. Add # to this list to register other decorators that produce valid properties. property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl # Regular expression matching correct variable names. Overrides variable- # naming-style variable-rgx=^[a-z][a-z0-9_]*$ [SIMILARITIES] # Ignore comments when computing similarities. ignore-comments=yes # Ignore docstrings when computing similarities. ignore-docstrings=yes # Ignore imports when computing similarities. ignore-imports=no # Minimum lines number of a similarity. min-similarity-lines=4 [SPELLING] # Limits count of emitted suggestions for spelling mistakes max-spelling-suggestions=4 # Spelling dictionary name. Available dictionaries: en_US (myspell), en # (aspell), en_AU (aspell), en_CA (aspell), en_GB (aspell). spelling-dict= # List of comma separated words that should not be checked. spelling-ignore-words= # A path to a file that contains private dictionary; one word per line. spelling-private-dict-file= # Tells whether to store unknown words to indicated private dictionary in # --spelling-private-dict-file option instead of raising a message. spelling-store-unknown-words=no [LOGGING] # Logging modules to check that the string format arguments are in logging # function parameter format logging-modules=logging [FORMAT] # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. expected-line-ending-format= # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=(?x)( ^\s*(\#\ )??$| ^\s*(from\s+\S+\s+)?import\s+.+$) # Number of spaces of indent required inside a hanging or continued line. indent-after-paren=4 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). indent-string=' ' # Maximum number of characters on a single line. max-line-length=120 # Maximum number of lines in a module max-module-lines=1000 # List of optional constructs for which whitespace checking is disabled. `dict- # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. # `trailing-comma` allows a space between comma and closing bracket: (a, ). # `empty-line` allows space-only lines. no-space-check=trailing-comma # Allow the body of a class to be on the same line as the declaration if body # contains single statement. single-line-class-stmt=no # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. notes=FIXME, XXX, TODO [TYPECHECK] # List of decorators that produce context managers, such as # contextlib.contextmanager. Add to this list to register other decorators that # produce valid context managers. contextmanager-decorators=contextlib.contextmanager # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E1101 when accessed. Python regular # expressions are accepted. generated-members= # Tells whether missing members accessed in mixin class should be ignored. A # mixin class is detected if its name ends with "mixin" (case insensitive). ignore-mixin-members=yes # This flag controls whether pylint should warn about no-member and similar # checks whenever an opaque object is returned when inferring. The inference # can return multiple potential results while evaluating a Python object, but # some branches might not be evaluated, which results in partial inference. In # that case, it might be useful to still emit no-member and other checks for # the rest of the inferred objects. ignore-on-opaque-inference=yes # List of class names for which member attributes should not be checked (useful # for classes with dynamically set attributes). This supports the use of # qualified names. ignored-classes=optparse.Values,thread._local,_thread._local # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis. It # supports qualified module names, as well as Unix pattern matching. ignored-modules= # Show a hint with possible names when a member name was not found. The aspect # of finding the hint is based on edit distance. missing-member-hint=yes # The minimum edit distance a name should have in order to be considered a # similar match for a missing member name. missing-member-hint-distance=1 # The total number of similar names that should be taken in consideration when # showing a hint for a missing member. missing-member-max-choices=1 [VARIABLES] # List of additional names supposed to be defined in builtins. Remember that # you should avoid to define new builtins when possible. additional-builtins= # Tells whether unused global variables should be treated as a violation. allow-global-unused-variables=yes # List of strings which can identify a callback function by name. A callback # name must start or end with one of those strings. callbacks=cb_, _cb # A regular expression matching the name of dummy variables (i.e. expectedly # not used). dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ # Argument names that match this expression will be ignored. Default to name # with leading underscore ignored-argument-names=_.*|^ignored_|^unused_ # Tells whether we should check for unused import in __init__ files. init-import=no # List of qualified module names which can have objects that can redefine # builtins. redefining-builtins-modules=six.moves,past.builtins,future.builtins,io,builtins [DESIGN] # Maximum number of arguments for function / method max-args=5 # Maximum number of attributes for a class (see R0902). max-attributes=7 # Maximum number of boolean expressions in a if statement max-bool-expr=5 # Maximum number of branch for function / method body max-branches=12 # Maximum number of locals for function / method body max-locals=15 # Maximum number of parents for a class (see R0901). max-parents=7 # Maximum number of public methods for a class (see R0904). max-public-methods=20 # Maximum number of return / yield for function / method body max-returns=6 # Maximum number of statements in function / method body max-statements=50 # Minimum number of public methods for a class (see R0903). min-public-methods=2 [CLASSES] # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__, __new__, setUp # List of member names, which should be excluded from the protected access # warning. exclude-protected=_asdict, _fields, _replace, _source, _make # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls, class_ # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=mcs [IMPORTS] # Allow wildcard imports from modules that define __all__. allow-wildcard-with-all=no # Analyse import fallback blocks. This can be used to support both Python 2 and # 3 compatible code, which means that the block might have code that exists # only in one or another interpreter, leading to false positives when analysed. analyse-fallback-blocks=no # Deprecated modules which should not be used, separated by a comma deprecated-modules=regsub, TERMIOS, Bastion, rexec, sets # Create a graph of external dependencies in the given file (report RP0402 must # not be disabled) ext-import-graph= # Create a graph of every (i.e. internal and external) dependencies in the # given file (report RP0402 must not be disabled) import-graph= # Create a graph of internal dependencies in the given file (report RP0402 must # not be disabled) int-import-graph= # Force import order to recognize a module as part of the standard # compatibility libraries. known-standard-library= # Force import order to recognize a module as part of a third party library. known-third-party=enchant [EXCEPTIONS] # Exceptions that will emit a warning when being caught. Defaults to # "Exception" overgeneral-exceptions=Exception, BaseException, ================================================ FILE: Dockerfile ================================================ # # Spiderfoot Dockerfile # # http://www.spiderfoot.net # # Written by: Michael Pellon # Updated by: Chandrapal # Updated by: Steve Micallef # Updated by: Steve Bate # -> Inspired by https://github.com/combro2k/dockerfiles/tree/master/alpine-spiderfoot # # Usage: # # sudo docker build -t spiderfoot . # sudo docker run -p 5001:5001 --security-opt no-new-privileges spiderfoot # # Using Docker volume for spiderfoot data # # sudo docker run -p 5001:5001 -v /mydir/spiderfoot:/var/lib/spiderfoot spiderfoot # # Using SpiderFoot remote command line with web server # # docker run --rm -it spiderfoot sfcli.py -s http://my.spiderfoot.host:5001/ # # Running spiderfoot commands without web server (can optionally specify volume) # # sudo docker run --rm spiderfoot sf.py -h # # Running a shell in the container for maintenance # sudo docker run -it --entrypoint /bin/sh spiderfoot # # Running spiderfoot unit tests in container # # sudo docker build -t spiderfoot-test --build-arg REQUIREMENTS=test/requirements.txt . # sudo docker run --rm spiderfoot-test -m pytest --flake8 . FROM alpine:3.12.4 AS build ARG REQUIREMENTS=requirements.txt RUN apk add --no-cache gcc git curl python3 python3-dev py3-pip swig tinyxml-dev \ python3-dev musl-dev openssl-dev libffi-dev libxslt-dev libxml2-dev jpeg-dev \ openjpeg-dev zlib-dev cargo rust RUN python3 -m venv /opt/venv ENV PATH="/opt/venv/bin":$PATH COPY $REQUIREMENTS requirements.txt ./ RUN ls RUN echo "$REQUIREMENTS" RUN pip3 install -U pip RUN pip3 install -r "$REQUIREMENTS" FROM alpine:3.13.0 WORKDIR /home/spiderfoot # Place database and logs outside installation directory ENV SPIDERFOOT_DATA /var/lib/spiderfoot ENV SPIDERFOOT_LOGS /var/lib/spiderfoot/log ENV SPIDERFOOT_CACHE /var/lib/spiderfoot/cache # Run everything as one command so that only one layer is created RUN apk --update --no-cache add python3 musl openssl libxslt tinyxml libxml2 jpeg zlib openjpeg \ && addgroup spiderfoot \ && adduser -G spiderfoot -h /home/spiderfoot -s /sbin/nologin \ -g "SpiderFoot User" -D spiderfoot \ && rm -rf /var/cache/apk/* \ && rm -rf /lib/apk/db \ && rm -rf /root/.cache \ && mkdir -p $SPIDERFOOT_DATA || true \ && mkdir -p $SPIDERFOOT_LOGS || true \ && mkdir -p $SPIDERFOOT_CACHE || true \ && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA \ && chown spiderfoot:spiderfoot $SPIDERFOOT_LOGS \ && chown spiderfoot:spiderfoot $SPIDERFOOT_CACHE COPY . . COPY --from=build /opt/venv /opt/venv ENV PATH="/opt/venv/bin:$PATH" USER spiderfoot EXPOSE 5001 # Run the application. ENTRYPOINT ["/opt/venv/bin/python"] CMD ["sf.py", "-l", "0.0.0.0:5001"] ================================================ FILE: Dockerfile.full ================================================ # # Spiderfoot Dockerfile (Full - includes all CLI tools, etc.) # # http://www.spiderfoot.net # # Written by: TheTechromancer # FROM python:3 # Install tools/dependencies from apt RUN apt-get -y update && apt-get -y install nbtscan onesixtyone nmap # Compile other tools from source RUN mkdir /tools || true WORKDIR /tools # Install Golang tools RUN apt-get -y update && apt-get -y install golang ENV GOPATH="/go" ENV PATH="$GOPATH/bin:$PATH" RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" # Install Ruby tools for WhatWeb RUN apt-get -y update && apt-get -y install ruby ruby-dev bundler # Install WhatWeb RUN git clone https://github.com/urbanadventurer/WhatWeb \ && gem install rchardet mongo json && cd /tools/WhatWeb \ && bundle install && cd /tools RUN groupadd spiderfoot \ && useradd -m -g spiderfoot -d /home/spiderfoot -s /sbin/nologin \ -c "SpiderFoot User" spiderfoot # Install RetireJS RUN apt remove -y cmdtest \ && apt remove -y yarn \ && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ && echo 'deb https://dl.yarnpkg.com/debian/ stable main' |tee /etc/apt/sources.list.d/yarn.list \ && apt-get update \ && apt-get install yarn -y \ && yarn install \ && curl -fsSL https://deb.nodesource.com/setup_17.x | bash - \ && apt-get install -y nodejs \ && npm install -g retire # Install Google Chrome the New Way (Not via apt-key) RUN wget -qO - https://dl.google.com/linux/linux_signing_key.pub | gpg --dearmor -o /usr/share/keyrings/googlechrome-linux-keyring.gpg \ && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/googlechrome-linux-keyring.gpg] http://dl.google.com/linux/chrome/deb/ stable main" | tee /etc/apt/sources.list.d/google-chrome.list \ && apt -y update && apt install --allow-unauthenticated -y google-chrome-stable # Install Wappalyzer RUN git clone https://github.com/AliasIO/wappalyzer.git \ && cd wappalyzer \ && yarn install && yarn run link # Install Nuclei RUN wget https://github.com/projectdiscovery/nuclei/releases/download/v2.6.5/nuclei_2.6.5_linux_amd64.zip \ && unzip nuclei_2.6.5_linux_amd64.zip \ && git clone https://github.com/projectdiscovery/nuclei-templates.git # Install testssl.sh RUN apt-get install -y bsdmainutils dnsutils coreutils RUN git clone https://github.com/drwetter/testssl.sh.git # Install Snallygaster and TruffleHog RUN pip3 install snallygaster trufflehog # Place database and logs outside installation directory ENV SPIDERFOOT_DATA /var/lib/spiderfoot ENV SPIDERFOOT_LOGS /var/lib/spiderfoot/log ENV SPIDERFOOT_CACHE /var/lib/spiderfoot/cache RUN mkdir -p $SPIDERFOOT_DATA || true \ && mkdir -p $SPIDERFOOT_LOGS || true \ && mkdir -p $SPIDERFOOT_CACHE || true \ && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA \ && chown spiderfoot:spiderfoot $SPIDERFOOT_LOGS \ && chown spiderfoot:spiderfoot $SPIDERFOOT_CACHE WORKDIR /home/spiderfoot COPY . . ENV VIRTUAL_ENV=/opt/venv RUN mkdir -p "$VIRTUAL_ENV" || true ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN python -m venv "$VIRTUAL_ENV" ARG REQUIREMENTS=requirements.txt COPY "$REQUIREMENTS" requirements.txt RUN chown -R spiderfoot:spiderfoot /tools RUN chown -R spiderfoot:spiderfoot "$VIRTUAL_ENV" RUN chown -R spiderfoot:spiderfoot "/home/spiderfoot" USER spiderfoot RUN pip install -U pip RUN pip install -r "$REQUIREMENTS" # Install Python tools RUN pip install dnstwist # CMSeeK WORKDIR /tools RUN git clone https://github.com/Tuhinshubhra/CMSeeK && cd CMSeeK \ && pip install -r requirements.txt && mkdir Results # Install wafw00f RUN git clone https://github.com/EnableSecurity/wafw00f \ && cd wafw00f \ && python3 setup.py install WORKDIR /home/spiderfoot EXPOSE 5001 # Run the application CMD python -c 'from spiderfoot import SpiderFootDb; \ db = SpiderFootDb({"__database": "/var/lib/spiderfoot/spiderfoot.db"}); \ db.configSet({ \ "sfp_tool_dnstwist:dnstwistpath": "/opt/venv/bin/dnstwist", \ "sfp_tool_cmseek:cmseekpath": "/tools/CMSeeK/cmseek.py", \ "sfp_tool_whatweb:whatweb_path": "/tools/WhatWeb/whatweb", \ "sfp_tool_wafw00f:wafw00f_path": "/opt/venv/bin/wafw00f", \ "sfp_tool_onesixtyone:onesixtyone_path": "/usr/bin/onesixtyone", \ "sfp_tool_retirejs:retirejs_path": "/usr/bin/retire", \ "sfp_tool_testsslsh:testsslsh_path": "/tools/testssl.sh/testssl.sh", \ "sfp_tool_snallygaster:snallygaster_path": "/usr/local/bin/snallygaster", \ "sfp_tool_trufflehog:trufflehog_path": "/usr/local/bin/trufflehog", \ "sfp_tool_nuclei:nuclei_path": "/tools/nuclei", \ "sfp_tool_nuclei:template_path": "/tools/nuclei-templates", \ "sfp_tool_wappalyzer:wappalyzer_path": "/tools/wappalyzer/src/drivers/npm/cli.js", \ "sfp_tool_nbtscan:nbtscan_path": "/usr/bin/nbtscan", \ "sfp_tool_nmap:nmappath": "DISABLED_BECAUSE_NMAP_REQUIRES_ROOT_TO_WORK" \ })' || true && ./sf.py -l 0.0.0.0:5001 ================================================ FILE: LICENSE ================================================ Copyright 2022 Steve Micallef Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/smicallef/spiderfoot/master/LICENSE) [![Python Version](https://img.shields.io/badge/python-3.7+-green)](https://www.python.org) [![Stable Release](https://img.shields.io/badge/version-4.0-blue.svg)](https://github.com/smicallef/spiderfoot/releases/tag/v4.0) [![CI status](https://github.com/smicallef/spiderfoot/workflows/Tests/badge.svg)](https://github.com/smicallef/spiderfoot/actions?query=workflow%3A"Tests") [![Last Commit](https://img.shields.io/github/last-commit/smicallef/spiderfoot)](https://github.com/smicallef/spiderfoot/commits/master) [![Codecov](https://codecov.io/github/smicallef/spiderfoot/coverage.svg)](https://codecov.io/github/smicallef/spiderfoot) [![Twitter Follow](https://img.shields.io/twitter/follow/spiderfoot?label=follow&style=social)](https://twitter.com/spiderfoot) [![Discord](https://img.shields.io/discord/770524432464216074)](https://discord.gg/vyvztrG) **SpiderFoot** is an open source intelligence (OSINT) automation tool. It integrates with just about every data source available and utilises a range of methods for data analysis, making that data easy to navigate. SpiderFoot has an embedded web-server for providing a clean and intuitive web-based interface but can also be used completely via the command-line. It's written in **Python 3** and **MIT-licensed**. ### FEATURES - Web based UI or CLI - Over 200 modules (see below) - Python 3.7+ - YAML-configurable [correlation engine](/correlations/README.md) with [37 pre-defined rules](/correlations) - CSV/JSON/GEXF export - API key export/import - SQLite back-end for custom querying - Highly configurable - Fully documented - Visualisations - TOR integration for dark web searching - Dockerfile for Docker-based deployments - Can call other tools like DNSTwist, Whatweb, Nmap and CMSeeK - [Actively developed since 2012!](https://medium.com/@micallst/lessons-learned-from-my-10-year-open-source-project-4a4c8c2b4f64) ### WANT MORE? Need more from SpiderFoot? Check out [SpiderFoot HX](https://www.spiderfoot.net/hx) for: - 100% Cloud-based and managed for you - Attack Surface Monitoring with change notifications by email, REST and Slack - Multiple targets per scan - Multi-user collaboration - Authenticated and 2FA - Investigations - Customer support - Third party tools pre-installed & configured - Drive it with a fully RESTful API - TOR integration built-in - Screenshotting - Bring your own Python SpiderFoot modules - Feed scan data to Splunk, ElasticSearch and REST endpoints See the full set of differences between SpiderFoot HX and the open source version [here](https://www.spiderfoot.net/open-source-vs-hx/). ### USES SpiderFoot can be used offensively (e.g. in a red team exercise or penetration test) for reconnaissance of your target or defensively to gather information about what you or your organisation might have exposed over the Internet. You can target the following entities in a SpiderFoot scan: - IP address - Domain/sub-domain name - Hostname - Network subnet (CIDR) - ASN - E-mail address - Phone number - Username - Person's name - Bitcoin address SpiderFoot's 200+ modules feed each other in a publisher/subscriber model to ensure maximum data extraction to do things like: - [Host/sub-domain/TLD enumeration/extraction](https://asciinema.org/a/295912) - [Email address, phone number and human name extraction](https://asciinema.org/a/295947) - [Bitcoin and Ethereum address extraction](https://asciinema.org/a/295957) - [Check for susceptibility to sub-domain hijacking](https://asciinema.org/a/344377) - DNS zone transfers - [Threat intelligence and Blacklist queries](https://asciinema.org/a/295949) - API integration with [SHODAN](https://asciinema.org/a/127601), [HaveIBeenPwned](https://asciinema.org/a/128731), [GreyNoise](https://asciinema.org/a/295943), AlienVault, SecurityTrails, etc. - [Social media account enumeration](https://asciinema.org/a/295923) - [S3/Azure/Digitalocean bucket enumeration/scraping](https://asciinema.org/a/295941) - IP geo-location - Web scraping, web content analysis - [Image, document and binary file meta data analysis](https://asciinema.org/a/296274) - Dark web searches - [Port scanning and banner grabbing](https://asciinema.org/a/295939) - [Data breach searches](https://asciinema.org/a/296145) - So much more... ### INSTALLING & RUNNING To install and run SpiderFoot, you need at least Python 3.7 and a number of Python libraries which you can install with `pip`. We recommend you install a packaged release since master will often have bleeding edge features and modules that aren't fully tested. #### Stable build (packaged release): ``` wget https://github.com/smicallef/spiderfoot/archive/v4.0.tar.gz tar zxvf v4.0.tar.gz cd spiderfoot-4.0 pip3 install -r requirements.txt python3 ./sf.py -l 127.0.0.1:5001 ``` #### Development build (cloning git master branch): ``` git clone https://github.com/smicallef/spiderfoot.git cd spiderfoot pip3 install -r requirements.txt python3 ./sf.py -l 127.0.0.1:5001 ``` Check out the [documentation](https://www.spiderfoot.net/documentation) and our [asciinema videos](https://asciinema.org/~spiderfoot) for more tutorials. ### COMMUNITY Whether you're a contributor, user or just curious about SpiderFoot and OSINT in general, we'd love to have you join our community! SpiderFoot now has a [Discord server](https://discord.gg/vyvztrG) for seeking help from the community, requesting features or just general OSINT chit-chat. ### WRITING CORRELATION RULES We have a comprehensive write-up and reference of the correlation rule-set introduced in SpiderFoot 4.0 [here](/correlations/README.md). Also take a look at the [template.yaml](/correlations/template.yaml) file for a walk through. The existing [37 rules](/correlations) are also quite readable and good as starting points for additional rules. ### MODULES / INTEGRATIONS SpiderFoot has over 200 modules, most of which *don't require API keys*, and many of those that do require API keys *have a free tier*. | Name | Description | Type | |:---------| :-----------|:-------| [AbstractAPI](https://app.abstractapi.com/)|Look up domain, phone and IP address information from AbstractAPI.|Tiered API [abuse.ch](https://www.abuse.ch)|Check if a host/domain, IP address or netblock is malicious according to Abuse.ch.|Free API [AbuseIPDB](https://www.abuseipdb.com)|Check if an IP address is malicious according to AbuseIPDB.com blacklist.|Tiered API [Abusix Mail Intelligence](https://abusix.org/)|Check if a netblock or IP address is in the Abusix Mail Intelligence blacklist.|Tiered API Account Finder|Look for possible associated accounts on over 500 social and other websites such as Instagram, Reddit, etc.|Internal [AdBlock Check](https://adblockplus.org/)|Check if linked pages would be blocked by AdBlock Plus.|Tiered API [AdGuard DNS](https://adguard.com/)|Check if a host would be blocked by AdGuard DNS.|Free API [Ahmia](https://ahmia.fi/)|Search Tor 'Ahmia' search engine for mentions of the target.|Free API [AlienVault IP Reputation](https://cybersecurity.att.com/)|Check if an IP or netblock is malicious according to the AlienVault IP Reputation database.|Free API [AlienVault OTX](https://otx.alienvault.com/)|Obtain information from AlienVault Open Threat Exchange (OTX)|Tiered API [Amazon S3 Bucket Finder](https://aws.amazon.com/s3/)|Search for potential Amazon S3 buckets associated with the target and attempt to list their contents.|Free API [Apple iTunes](https://itunes.apple.com/)|Search Apple iTunes for mobile apps.|Free API [Archive.org](https://archive.org/)|Identifies historic versions of interesting files/pages from the Wayback Machine.|Free API [ARIN](https://www.arin.net/)|Queries ARIN registry for contact information.|Free API [Azure Blob Finder](https://azure.microsoft.com/en-in/services/storage/blobs/)|Search for potential Azure blobs associated with the target and attempt to list their contents.|Free API Base64 Decoder|Identify Base64-encoded strings in URLs, often revealing interesting hidden information.|Internal [BGPView](https://bgpview.io/)|Obtain network information from BGPView API.|Free API Binary String Extractor|Attempt to identify strings in binary content.|Internal [BinaryEdge](https://www.binaryedge.io/)|Obtain information from BinaryEdge.io Internet scanning systems, including breaches, vulnerabilities, torrents and passive DNS.|Tiered API [Bing (Shared IPs)](https://www.bing.com/)|Search Bing for hosts sharing the same IP.|Tiered API [Bing](https://www.bing.com/)|Obtain information from bing to identify sub-domains and links.|Tiered API Bitcoin Finder|Identify bitcoin addresses in scraped webpages.|Internal [Bitcoin Who's Who](https://bitcoinwhoswho.com/)|Check for Bitcoin addresses against the Bitcoin Who's Who database of suspect/malicious addresses.|Tiered API [BitcoinAbuse](https://www.bitcoinabuse.com/)|Check Bitcoin addresses against the bitcoinabuse.com database of suspect/malicious addresses.|Free API [Blockchain](https://www.blockchain.com/)|Queries blockchain.info to find the balance of identified bitcoin wallet addresses.|Free API [blocklist.de](http://www.blocklist.de/en/index.html)|Check if a netblock or IP is malicious according to blocklist.de.|Free API [BotScout](https://botscout.com/)|Searches BotScout.com's database of spam-bot IP addresses and e-mail addresses.|Tiered API [botvrij.eu](https://botvrij.eu/)|Check if a domain is malicious according to botvrij.eu.|Free API [BuiltWith](https://builtwith.com/)|Query BuiltWith.com's Domain API for information about your target's web technology stack, e-mail addresses and more.|Tiered API [C99](https://api.c99.nl/)|Queries the C99 API which offers various data (geo location, proxy detection, phone lookup, etc).|Commercial API [CallerName](http://callername.com/)|Lookup US phone number location and reputation information.|Free API [Censys](https://censys.io/)|Obtain host information from Censys.io.|Tiered API [Certificate Transparency](https://crt.sh/)|Gather hostnames from historical certificates in crt.sh.|Free API [CertSpotter](https://sslmate.com/certspotter/)|Gather information about SSL certificates from SSLMate CertSpotter API.|Tiered API [CINS Army List](https://cinsscore.com/)|Check if a netblock or IP address is malicious according to Collective Intelligence Network Security (CINS) Army list.|Free API [CIRCL.LU](https://www.circl.lu/)|Obtain information from CIRCL.LU's Passive DNS and Passive SSL databases.|Free API [CleanBrowsing.org](https://cleanbrowsing.org/)|Check if a host would be blocked by CleanBrowsing.org DNS content filters.|Free API [CleanTalk Spam List](https://cleantalk.org)|Check if a netblock or IP address is on CleanTalk.org's spam IP list.|Free API [Clearbit](https://clearbit.com/)|Check for names, addresses, domains and more based on lookups of e-mail addresses on clearbit.com.|Tiered API [CloudFlare DNS](https://www.cloudflare.com/)|Check if a host would be blocked by CloudFlare DNS.|Free API [CoinBlocker Lists](https://zerodot1.gitlab.io/CoinBlockerListsWeb/)|Check if a domain appears on CoinBlocker lists.|Free API [CommonCrawl](http://commoncrawl.org/)|Searches for URLs found through CommonCrawl.org.|Free API [Comodo Secure DNS](https://www.comodo.com/secure-dns/)|Check if a host would be blocked by Comodo Secure DNS.|Tiered API Company Name Extractor|Identify company names in any obtained data.|Internal Cookie Extractor|Extract Cookies from HTTP headers.|Internal Country Name Extractor|Identify country names in any obtained data.|Internal Credit Card Number Extractor|Identify Credit Card Numbers in any data|Internal [Crobat API](https://sonar.omnisint.io/)|Search Crobat API for subdomains.|Free API Cross-Referencer|Identify whether other domains are associated ('Affiliates') of the target by looking for links back to the target site(s).|Internal [CRXcavator](https://crxcavator.io/)|Search CRXcavator for Chrome extensions.|Free API Custom Threat Feed|Check if a host/domain, netblock, ASN or IP is malicious according to your custom feed.|Internal [CyberCrime-Tracker.net](https://cybercrime-tracker.net/)|Check if a host/domain or IP address is malicious according to CyberCrime-Tracker.net.|Free API [Debounce](https://debounce.io/)|Check whether an email is disposable|Free API [Dehashed](https://www.dehashed.com/)|Gather breach data from Dehashed API.|Commercial API [Digital Ocean Space Finder](https://www.digitalocean.com/products/spaces/)|Search for potential Digital Ocean Spaces associated with the target and attempt to list their contents.|Free API DNS Brute-forcer|Attempts to identify hostnames through brute-forcing common names and iterations.|Internal DNS Common SRV|Attempts to identify hostnames through brute-forcing common DNS SRV records.|Internal [DNS for Family](https://dnsforfamily.com/)|Check if a host would be blocked by DNS for Family.|Free API DNS Look-aside|Attempt to reverse-resolve the IP addresses next to your target to see if they are related.|Internal DNS Raw Records|Retrieves raw DNS records such as MX, TXT and others.|Internal DNS Resolver|Resolves hosts and IP addresses identified, also extracted from raw content.|Internal DNS Zone Transfer|Attempts to perform a full DNS zone transfer.|Internal [DNSDB](https://www.farsightsecurity.com)|Query FarSight's DNSDB for historical and passive DNS data.|Tiered API [DNSDumpster](https://dnsdumpster.com/)|Passive subdomain enumeration using HackerTarget's DNSDumpster|Free API [DNSGrep](https://opendata.rapid7.com/)|Obtain Passive DNS information from Rapid7 Sonar Project using DNSGrep API.|Free API [DroneBL](https://dronebl.org/)|Query the DroneBL database for open relays, open proxies, vulnerable servers, etc.|Free API [DuckDuckGo](https://duckduckgo.com/)|Query DuckDuckGo's API for descriptive information about your target.|Free API E-Mail Address Extractor|Identify e-mail addresses in any obtained data.|Internal [EmailCrawlr](https://emailcrawlr.com/)|Search EmailCrawlr for email addresses and phone numbers associated with a domain.|Tiered API [EmailFormat](https://www.email-format.com/)|Look up e-mail addresses on email-format.com.|Free API [EmailRep](https://emailrep.io/)|Search EmailRep.io for email address reputation.|Tiered API [Emerging Threats](https://rules.emergingthreats.net/)|Check if a netblock or IP address is malicious according to EmergingThreats.net.|Free API Error String Extractor|Identify common error messages in content like SQL errors, etc.|Internal Ethereum Address Extractor|Identify ethereum addresses in scraped webpages.|Internal [Etherscan](https://etherscan.io)|Queries etherscan.io to find the balance of identified ethereum wallet addresses.|Free API File Metadata Extractor|Extracts meta data from documents and images.|Internal [Flickr](https://www.flickr.com/)|Search Flickr for domains, URLs and emails related to the specified domain.|Free API [Focsec](https://focsec.com/)|Look up IP address information from Focsec.|Tiered API [FortiGuard Antispam](https://www.fortiguard.com/)|Check if an IP address is malicious according to FortiGuard Antispam.|Free API [Fraudguard](https://fraudguard.io/)|Obtain threat information from Fraudguard.io|Tiered API [F-Secure Riddler.io](https://riddler.io/)|Obtain network information from F-Secure Riddler.io API.|Commercial API [FullContact](https://www.fullcontact.com)|Gather domain and e-mail information from FullContact.com API.|Tiered API [FullHunt](https://fullhunt.io/)|Identify domain attack surface using FullHunt API.|Tiered API [Github](https://github.com/)|Identify associated public code repositories on Github.|Free API [GLEIF](https://search.gleif.org/)|Look up company information from Global Legal Entity Identifier Foundation (GLEIF).|Tiered API [Google Maps](https://cloud.google.com/maps-platform/)|Identifies potential physical addresses and latitude/longitude coordinates.|Tiered API [Google Object Storage Finder](https://cloud.google.com/storage)|Search for potential Google Object Storage buckets associated with the target and attempt to list their contents.|Free API [Google SafeBrowsing](https://developers.google.com/safe-browsing/v4/lookup-api)|Check if the URL is included on any of the Safe Browsing lists.|Free API [Google](https://developers.google.com/custom-search)|Obtain information from the Google Custom Search API to identify sub-domains and links.|Tiered API [Gravatar](https://secure.gravatar.com/)|Retrieve user information from Gravatar API.|Free API [Grayhat Warfare](https://buckets.grayhatwarfare.com/)|Find bucket names matching the keyword extracted from a domain from Grayhat API.|Tiered API [Greensnow](https://greensnow.co/)|Check if a netblock or IP address is malicious according to greensnow.co.|Free API [grep.app](https://grep.app/)|Search grep.app API for links and emails related to the specified domain.|Free API [GreyNoise Community](https://greynoise.io/)|Obtain IP enrichment data from GreyNoise Community API|Tiered API [GreyNoise](https://greynoise.io/)|Obtain IP enrichment data from GreyNoise|Tiered API [HackerOne (Unofficial)](http://www.nobbd.de/)|Check external vulnerability scanning/reporting service h1.nobbd.de to see if the target is listed.|Free API [HackerTarget](https://hackertarget.com/)|Search HackerTarget.com for hosts sharing the same IP.|Free API Hash Extractor|Identify MD5 and SHA hashes in web content, files and more.|Internal [HaveIBeenPwned](https://haveibeenpwned.com/)|Check HaveIBeenPwned.com for hacked e-mail addresses identified in breaches.|Commercial API Hosting Provider Identifier|Find out if any IP addresses identified fall within known 3rd party hosting ranges, e.g. Amazon, Azure, etc.|Internal [Host.io](https://host.io)|Obtain information about domain names from host.io.|Tiered API Human Name Extractor|Attempt to identify human names in fetched content.|Internal [Hunter.io](https://hunter.io/)|Check for e-mail addresses and names on hunter.io.|Tiered API [Hybrid Analysis](https://www.hybrid-analysis.com)|Search Hybrid Analysis for domains and URLs related to the target.|Free API IBAN Number Extractor|Identify International Bank Account Numbers (IBANs) in any data.|Internal [Iknowwhatyoudownload.com](https://iknowwhatyoudownload.com/en/peer/)|Check iknowwhatyoudownload.com for IP addresses that have been using torrents.|Tiered API [IntelligenceX](https://intelx.io/)|Obtain information from IntelligenceX about identified IP addresses, domains, e-mail addresses and phone numbers.|Tiered API Interesting File Finder|Identifies potential files of interest, e.g. office documents, zip files.|Internal [Internet Storm Center](https://isc.sans.edu)|Check if an IP address is malicious according to SANS ISC.|Free API [ipapi.co](https://ipapi.co/)|Queries ipapi.co to identify geolocation of IP Addresses using ipapi.co API|Tiered API [ipapi.com](https://ipapi.com/)|Queries ipapi.com to identify geolocation of IP Addresses using ipapi.com API|Tiered API [IPInfo.io](https://ipinfo.io)|Identifies the physical location of IP addresses identified using ipinfo.io.|Tiered API [IPQualityScore](https://www.ipqualityscore.com/)|Determine if target is malicious using IPQualityScore API|Tiered API [ipregistry](https://ipregistry.co/)|Query the ipregistry.co database for reputation and geo-location.|Tiered API [ipstack](https://ipstack.com/)|Identifies the physical location of IP addresses identified using ipstack.com.|Tiered API [JsonWHOIS.com](https://jsonwhois.com)|Search JsonWHOIS.com for WHOIS records associated with a domain.|Tiered API Junk File Finder|Looks for old/temporary and other similar files.|Internal [Keybase](https://keybase.io/)|Obtain additional information about domain names and identified usernames.|Free API [Koodous](https://koodous.com/apks/)|Search Koodous for mobile apps.|Tiered API [LeakIX](https://leakix.net/)|Search LeakIX for host data leaks, open ports, software and geoip.|Free API [Leak-Lookup](https://leak-lookup.com/)|Searches Leak-Lookup.com's database of breaches.|Free API [Maltiverse](https://maltiverse.com)|Obtain information about any malicious activities involving IP addresses|Free API [MalwarePatrol](https://www.malwarepatrol.net/)|Searches malwarepatrol.net's database of malicious URLs/IPs.|Tiered API [MetaDefender](https://metadefender.opswat.com/)|Search MetaDefender API for IP address and domain IP reputation.|Tiered API [Mnemonic PassiveDNS](https://www.mnemonic.no)|Obtain Passive DNS information from PassiveDNS.mnemonic.no.|Free API [multiproxy.org Open Proxies](https://multiproxy.org/)|Check if an IP address is an open proxy according to multiproxy.org open proxy list.|Free API [MySpace](https://myspace.com/)|Gather username and location from MySpace.com profiles.|Free API [NameAPI](https://www.nameapi.org/)|Check whether an email is disposable|Tiered API [NetworksDB](https://networksdb.io/)|Search NetworksDB.io API for IP address and domain information.|Tiered API [NeutrinoAPI](https://www.neutrinoapi.com/)|Search NeutrinoAPI for phone location information, IP address information, and host reputation.|Tiered API [numverify](http://numverify.com/)|Lookup phone number location and carrier information from numverify.com.|Tiered API [Onion.link](https://onion.link/)|Search Tor 'Onion City' search engine for mentions of the target domain using Google Custom Search.|Free API [Onionsearchengine.com](https://as.onionsearchengine.com)|Search Tor onionsearchengine.com for mentions of the target domain.|Free API [Onyphe](https://www.onyphe.io)|Check Onyphe data (threat list, geo-location, pastries, vulnerabilities) about a given IP.|Tiered API [Open Bug Bounty](https://www.openbugbounty.org/)|Check external vulnerability scanning/reporting service openbugbounty.org to see if the target is listed.|Free API [OpenCorporates](https://opencorporates.com)|Look up company information from OpenCorporates.|Tiered API [OpenDNS](https://www.opendns.com/)|Check if a host would be blocked by OpenDNS.|Free API [OpenNIC DNS](https://www.opennic.org/)|Resolves host names in the OpenNIC alternative DNS system.|Free API [OpenPhish](https://openphish.com/)|Check if a host/domain is malicious according to OpenPhish.com.|Free API [OpenStreetMap](https://www.openstreetmap.org/)|Retrieves latitude/longitude coordinates for physical addresses from OpenStreetMap API.|Free API Page Information|Obtain information about web pages (do they take passwords, do they contain forms, etc.)|Internal [PasteBin](https://pastebin.com/)|PasteBin search (via Google Search API) to identify related content.|Tiered API PGP Key Servers|Look up domains and e-mail addresses in PGP public key servers.|Internal [PhishStats](https://phishstats.info/)|Check if a netblock or IP address is malicious according to PhishStats.|Free API [PhishTank](https://phishtank.com/)|Check if a host/domain is malicious according to PhishTank.|Free API Phone Number Extractor|Identify phone numbers in scraped webpages.|Internal Port Scanner - TCP|Scans for commonly open TCP ports on Internet-facing systems.|Internal [Project Honey Pot](https://www.projecthoneypot.org/)|Query the Project Honey Pot database for IP addresses.|Free API [ProjectDiscovery Chaos](https://chaos.projectdiscovery.io)|Search for hosts/subdomains using chaos.projectdiscovery.io|Commercial API [Psbdmp](https://psbdmp.cc/)|Check psbdmp.cc (PasteBin Dump) for potentially hacked e-mails and domains.|Free API [Pulsedive](https://pulsedive.com/)|Obtain information from Pulsedive's API.|Tiered API [PunkSpider](https://punkspider.io/)|Check the QOMPLX punkspider.io service to see if the target is listed as vulnerable.|Free API [Quad9](https://quad9.net/)|Check if a host would be blocked by Quad9 DNS.|Free API [ReverseWhois](https://www.reversewhois.io/)|Reverse Whois lookups using reversewhois.io.|Free API [RIPE](https://www.ripe.net/)|Queries the RIPE registry (includes ARIN data) to identify netblocks and other info.|Free API [RiskIQ](https://community.riskiq.com/)|Obtain information from RiskIQ's (formerly PassiveTotal) Passive DNS and Passive SSL databases.|Tiered API [Robtex](https://www.robtex.com/)|Search Robtex.com for hosts sharing the same IP.|Free API [searchcode](https://searchcode.com/)|Search searchcode for code repositories mentioning the target domain.|Free API [SecurityTrails](https://securitytrails.com/)|Obtain Passive DNS and other information from SecurityTrails|Tiered API [Seon](https://seon.io/)|Queries seon.io to gather intelligence about IP Addresses, email addresses, and phone numbers|Commercial API [SHODAN](https://www.shodan.io/)|Obtain information from SHODAN about identified IP addresses.|Tiered API Similar Domain Finder|Search various sources to identify similar looking domain names, for instance squatted domains.|Internal [Skymem](http://www.skymem.info/)|Look up e-mail addresses on Skymem.|Free API [SlideShare](https://www.slideshare.net)|Gather name and location from SlideShare profiles.|Free API [Snov](https://snov.io/)|Gather available email IDs from identified domains|Tiered API [Social Links](https://sociallinks.io/)|Queries SocialLinks.io to gather intelligence from social media platforms and dark web.|Commercial API [Social Media Profile Finder](https://developers.google.com/custom-search)|Tries to discover the social media profiles for human names identified.|Tiered API Social Network Identifier|Identify presence on social media networks such as LinkedIn, Twitter and others.|Internal [SORBS](http://www.sorbs.net/)|Query the SORBS database for open relays, open proxies, vulnerable servers, etc.|Free API [SpamCop](https://www.spamcop.net/)|Check if a netblock or IP address is in the SpamCop database.|Free API [Spamhaus Zen](https://www.spamhaus.org/)|Check if a netblock or IP address is in the Spamhaus Zen database.|Free API [spur.us](https://spur.us/)|Obtain information about any malicious activities involving IP addresses found|Commercial API [SpyOnWeb](http://spyonweb.com/)|Search SpyOnWeb for hosts sharing the same IP address, Google Analytics code, or Google Adsense code.|Tiered API SSL Certificate Analyzer|Gather information about SSL certificates used by the target's HTTPS sites.|Internal [StackOverflow](https://www.stackexchange.com)|Search StackOverflow for any mentions of a target domain. Returns potentially related information.|Tiered API [Steven Black Hosts](https://github.com/StevenBlack/hosts)|Check if a domain is malicious (malware or adware) according to Steven Black Hosts list.|Free API Strange Header Identifier|Obtain non-standard HTTP headers returned by web servers.|Internal Subdomain Takeover Checker|Check if affiliated subdomains are vulnerable to takeover.|Internal [Sublist3r PassiveDNS](https://api.sublist3r.com)|Passive subdomain enumeration using Sublist3r's API|Free API [SURBL](http://www.surbl.org/)|Check if a netblock, IP address or domain is in the SURBL blacklist.|Free API [Talos Intelligence](https://talosintelligence.com/)|Check if a netblock or IP address is malicious according to TalosIntelligence.|Free API [TextMagic](https://www.textmagic.com/)|Obtain phone number type from TextMagic API|Tiered API [Threat Jammer](https://threatjammer.com)|Check if an IP address is malicious according to ThreatJammer.com|Tiered API [ThreatCrowd](https://www.threatcrowd.org)|Obtain information from ThreatCrowd about identified IP addresses, domains and e-mail addresses.|Free API [ThreatFox](https://threatfox.abuse.ch)|Check if an IP address is malicious according to ThreatFox.|Free API [ThreatMiner](https://www.threatminer.org/)|Obtain information from ThreatMiner's database for passive DNS and threat intelligence.|Free API TLD Searcher|Search all Internet TLDs for domains with the same name as the target (this can be very slow.)|Internal [Tool - CMSeeK]([https://github.com/Tuhinshubhra/CMSeeK](https://github.com/Tuhinshubhra/CMSeeK))|Identify what Content Management System (CMS) might be used.|Tool [Tool - DNSTwist]([https://github.com/elceef/dnstwist](https://github.com/elceef/dnstwist))|Identify bit-squatting, typo and other similar domains to the target using a local DNSTwist installation.|Tool [Tool - nbtscan]([http://www.unixwiz.net/tools/nbtscan.html](http://www.unixwiz.net/tools/nbtscan.html))|Scans for open NETBIOS nameservers on your target's network.|Tool [Tool - Nmap]([https://nmap.org/](https://nmap.org/))|Identify what Operating System might be used.|Tool [Tool - Nuclei]([https://nuclei.projectdiscovery.io/](https://nuclei.projectdiscovery.io/))|Fast and customisable vulnerability scanner.|Tool [Tool - onesixtyone]([https://github.com/trailofbits/onesixtyone](https://github.com/trailofbits/onesixtyone))|Fast scanner to find publicly exposed SNMP services.|Tool [Tool - Retire.js]([http://retirejs.github.io/retire.js/](http://retirejs.github.io/retire.js/))|Scanner detecting the use of JavaScript libraries with known vulnerabilities|Tool [Tool - snallygaster]([https://github.com/hannob/snallygaster](https://github.com/hannob/snallygaster))|Finds file leaks and other security problems on HTTP servers.|Tool [Tool - testssl.sh]([https://testssl.sh](https://testssl.sh))|Identify various TLS/SSL weaknesses, including Heartbleed, CRIME and ROBOT.|Tool [Tool - TruffleHog]([https://github.com/trufflesecurity/truffleHog](https://github.com/trufflesecurity/truffleHog))|Searches through git repositories for high entropy strings and secrets, digging deep into commit history.|Tool [Tool - WAFW00F]([https://github.com/EnableSecurity/wafw00f](https://github.com/EnableSecurity/wafw00f))|Identify what web application firewall (WAF) is in use on the specified website.|Tool [Tool - Wappalyzer]([https://www.wappalyzer.com/](https://www.wappalyzer.com/))|Wappalyzer indentifies technologies on websites.|Tool [Tool - WhatWeb]([https://github.com/urbanadventurer/whatweb](https://github.com/urbanadventurer/whatweb))|Identify what software is in use on the specified website.|Tool [TOR Exit Nodes](https://metrics.torproject.org/)|Check if an IP adddress or netblock appears on the Tor Metrics exit node list.|Free API [TORCH](https://torchsearch.wordpress.com/)|Search Tor 'TORCH' search engine for mentions of the target domain.|Free API [Trashpanda](https://got-hacked.wtf)|Queries Trashpanda to gather intelligence about mentions of target in pastesites|Tiered API [Trumail](https://trumail.io/)|Check whether an email is disposable|Free API [Twilio](https://www.twilio.com/)|Obtain information from Twilio about phone numbers. Ensure you have the Caller Name add-on installed in Twilio.|Tiered API [Twitter](https://twitter.com/)|Gather name and location from Twitter profiles.|Free API [UCEPROTECT](http://www.uceprotect.net/)|Check if a netblock or IP address is in the UCEPROTECT database.|Free API [URLScan.io](https://urlscan.io/)|Search URLScan.io cache for domain information.|Free API [Venmo](https://venmo.com/)|Gather user information from Venmo API.|Free API [ViewDNS.info](https://viewdns.info/)|Identify co-hosted websites and perform reverse Whois lookups using ViewDNS.info.|Tiered API [VirusTotal](https://www.virustotal.com/)|Obtain information from VirusTotal about identified IP addresses.|Tiered API [VoIP Blacklist (VoIPBL)](https://voipbl.org/)|Check if an IP address or netblock is malicious according to VoIP Blacklist (VoIPBL).|Free API [VXVault.net](http://vxvault.net/)|Check if a domain or IP address is malicious according to VXVault.net.|Free API Web Analytics Extractor|Identify web analytics IDs in scraped webpages and DNS TXT records.|Internal Web Framework Identifier|Identify the usage of popular web frameworks like jQuery, YUI and others.|Internal Web Server Identifier|Obtain web server banners to identify versions of web servers being used.|Internal Web Spider|Spidering of web-pages to extract content for searching.|Internal [WhatCMS](https://whatcms.org/)|Check web technology using WhatCMS.org API.|Tiered API [Whoisology](https://whoisology.com/)|Reverse Whois lookups using Whoisology.com.|Commercial API Whois|Perform a WHOIS look-up on domain names and owned netblocks.|Internal [Whoxy](https://www.whoxy.com/)|Reverse Whois lookups using Whoxy.com.|Commercial API [WiGLE](https://wigle.net/)|Query WiGLE to identify nearby WiFi access points.|Free API [Wikileaks](https://wikileaks.org/)|Search Wikileaks for mentions of domain names and e-mail addresses.|Free API [Wikipedia Edits](https://www.wikipedia.org/)|Identify edits to Wikipedia articles made from a given IP address or username.|Free API [XForce Exchange](https://exchange.xforce.ibmcloud.com/)|Obtain IP reputation and passive DNS information from IBM X-Force Exchange.|Tiered API [Yandex DNS](https://yandex.com/)|Check if a host would be blocked by Yandex DNS.|Free API [Zetalytics](https://zetalytics.com/)|Query the Zetalytics database for hosts on your target domain(s).|Tiered API [ZoneFile.io](https://zonefiles.io)|Search ZoneFiles.io Domain query API for domain information.|Tiered API [Zone-H Defacement Check](https://zone-h.org/)|Check if a hostname/domain appears on the zone-h.org 'special defacements' RSS feed.|Free API ### DOCUMENTATION Read more at the [project website](https://www.spiderfoot.net/r.php?u=aHR0cHM6Ly93d3cuc3BpZGVyZm9vdC5uZXQv&s=os_gh), including more complete documentation, blog posts with tutorials/guides, plus information about [SpiderFoot HX](https://www.spiderfoot.net/r.php?u=aHR0cHM6Ly93d3cuc3BpZGVyZm9vdC5uZXQvaHgvCg==&s=os_gh). Latest updates announced on [Twitter](https://twitter.com/spiderfoot). ================================================ FILE: THANKYOU ================================================ These people are owed some credit for their contributions to SpiderFoot, be it bug fixes or new functionality. Thank you! - Brendan Coles (https://github.com/bcoles) - Second highest contributor to the project - Tons of modules, fixes and established the whole testing framework - Too many contributions to list here, see https://github.com/smicallef/spiderfoot/pulls?q=is%3Apr+author%3Abcoles+is%3Aclosed - Krishnasis Mandal (https://github.com/krishnasism) - Many modules, see https://github.com/smicallef/spiderfoot/pulls?q=is%3Apr+author%3Akrishnasism+is%3Aclosed - Steve Bate (https://github.com/steve-bate) - Multiple bug fixes, performance and quality improvements - fallingcubes (https://github.com/fallingcubes) - Use of multiprocessing instead of threading for running multiple scans simultaneously - Counter Intrusive Technologies (https://github.com/counterintrusive) - sfp_citadel updates - Dhiraj Mishra (https://github.com/RootUp) - Created the sfp_h1nobbdde module - Chris Weber (http://www.casaba.com/blog/author/chris/) - Many recommendations for improvement - Henri Salo (https://github.com/fgeek) - Identified XSS issues (issue #29) - MarioVilas (https://github.com/MarioVilas) - Contributed a fix for CSV exports (issue #35) - johnnykv (https://github.com/johnnykv) - Contributed improved pip instructions - Russ McRee (@holisticinfosec) - Identified XSS/CSRF issues - Andrea De Pasquale (https://github.com/adepasquale) - Cache directory fix - Viyat Bhalodia (https://github.com/delta24) - Prettied up README.md - Gormogon (https://github.com/Gormogon) - Huge code clean up for PEP 8 compliance - Lin Zhemin (https://github.com/miaoski) - Added the ability to use a configurable document root - Micah Hoffman (https://github.com/WebBreacher) - Comprehensive list of URLs for identifying account usage in modules/sfp_accounts.py - Michael Pellon (https://github.com/mpellon) - Providing a Dockerfile - HackershubNL (https://github.com/HackershubNL) - Contributions towards authentication functionality - Mathew Woodyard (https://github.com/woodrad) - Bad Packets module improvements ================================================ FILE: VERSION ================================================ SpiderFoot 4.0.0 ================================================ FILE: correlations/README.md ================================================ ## Background SpiderFoot’s goal is to automate OSINT collection and analysis to the greatest extent possible. Since its inception, SpiderFoot has heavily focused on automating OSINT collection and entity extraction, but the automation of common analysis tasks -- beyond some reporting and visualisations -- has been left entirely to the user. The meant that the strength of SpiderFoot's data collection capabilities has sometimes been its weakness since with so much data collected, users have often needed to export it and use other tools to weed out data of interest. ## Introducing Correlations We started tackling this analysis gap with the launch of SpiderFoot HX in 2019 through the introduction of the "Correlations" feature. This feature was represented by some 30 "correlation rules" that ran with each scan, analyzing data and presenting results reflecting SpiderFoot's opinionated view on what may be important or interesting. Here are a few of those rules as examples: * Hosts/IPs reported as malicious by multiple data sources * Outlier web servers (can be an indication of shadow IT) * Databases exposed on the Internet * Open ports revealing software versions * and many more. With the release of SpiderFoot 4.0 we wanted to bring this capability from SpiderFoot HX to the community, but also re-imagine it at the same time so that the community might not simply run rules we provide, but also write their own correlation rules and contribute them back. We also hope that just as with modules, we see a long list of contributions made in the years ahead so that all may benefit. With that said, let's get into what these rules look like and how to write one. ## Key concepts ### YAML The rules themselves are written in YAML. Why YAML? It’s easy to read, write, allows for comments and is increasingly commonplace in many modern tools. ### Rule structure The simplest way to think of a SpiderFoot correlation rule is like a simple database query that consists of a few sections: 1. Defining the rule itself (`id`, `version` and `meta` sections). 2. Stating what you'd like to extract from the scan results (`collections` section). 3. Grouping that data in some way (`aggregation` section; optional). 3. Performing some analysis over that data in some way (`analysis` section; optional). 4. Presenting the results (`headline` section). ### Example rule Here's an example rule that looks at SpiderFoot scan results for data revealing open TCP ports where the banner (the data returned upon connecting to the port) reports a software version. It does so by applying some regular expressions to the content of `TCP_PORT_OPEN_BANNER` data elements, filtering out some false positives and then grouping the results by the banner itself so that one correlation result is created per banner revealing a version: ```yaml id: open_port_version version: 1 meta: name: Open TCP port reveals version description: > A possible software version has been revealed on an open port. Such information may reveal the use of old/unpatched software used by the target. risk: INFO collections: - collect: - method: exact field: type value: TCP_PORT_OPEN_BANNER - method: regex field: data value: .*[0-9]\.[0-9].* - method: regex field: data value: not .*Mime-Version.* - method: regex field: data value: not .*HTTP/1.* aggregation: field: data headline: "Software version revealed on open port: {data}" ``` ### The outcome To show this in practice, we can run a simple scan against a target, in this case focusing on performing a port scan: ``` -> # python3.9 ./sf.py -s www.binarypool.com -m sfp_dnsresolve,sfp_portscan_tcp 2022-04-06 08:14:58,476 [INFO] sflib : Scan [94EB5F0B] for 'www.binarypool.com' initiated. ... sfp_portscan_tcp Open TCP Port Banner SSH-2.0-OpenSSH_7.2p2 Ubuntu-4ubuntu2.10 ... 2022-04-06 08:15:23,110 [INFO] correlation : New correlation [open_port_version]: Software version revealed on open port: SSH-2.0-OpenSSH_7.2p2 Ubuntu-4ubuntu2.10 2022-04-06 08:15:23,244 [INFO] sflib : Scan [94EB5F0B] completed. ``` We can see above that a port was found to be open by the `sfp_portscan_tcp` module, and it happens to include a version. The correlation rule `open_port_version` picked this up and reported it. This is also visible in the web interface: **NOTE:** Rules will only succeed if relevant data exists in your scan results in the first place. In other words, correlation rules analyze scan data, they don't collect data from targets. ### How it works In short, SpiderFoot translates the YAML rules into a combination queries against the backend database of scan results and Python logic to filter and group the results, creating "correlation results" in the SpiderFoot database. These results can be viewed in the SpiderFot web interface or from the SpiderFoot CLI. You can also query them directly out of the SQLite database if you like (they are in the `tbl_scan_correlation_results` table, and the `tbl_scan_correlation_results_events` table maps the events (data elements) to the correlation result). ### The rules Each rule exists as a YAML file within the `/correlations` folder in the SpiderFoot installation path. Here you can see a list of rules in 4.0, which we hope to grow over time: ```sh cert_expired.yaml host_only_from_certificatetransparency.yaml outlier_ipaddress.yaml cloud_bucket_open.yaml http_errors.yaml outlier_registrar.yaml cloud_bucket_open_related.yaml human_name_in_whois.yaml outlier_webserver.yaml data_from_base64.yaml internal_host.yaml remote_desktop_exposed.yaml data_from_docmeta.yaml multiple_malicious.yaml root_path_needs_auth.yaml database_exposed.yaml multiple_malicious_affiliate.yaml stale_host.yaml dev_or_test_system.yaml multiple_malicious_cohost.yaml strong_affiliate_certs.yaml dns_zone_transfer_possible.yaml name_only_from_pasteleak_site.yaml strong_similardomain_crossref.yaml egress_ip_from_wikipedia.yaml open_port_version.yaml template.yaml email_in_multiple_breaches.yaml outlier_cloud.yaml vulnerability_critical.yaml email_in_whois.yaml outlier_country.yaml vulnerability_high.yaml email_only_from_pasteleak_site.yaml outlier_email.yaml vulnerability_mediumlow.yaml host_only_from_bruteforce.yaml outlier_hostname.yaml ``` ### Rule components The rules themselves are broken down into the following components: **Meta**: Describes the rule itself so that humans understand what the rule does and the risk level of any results. This information is used mostly in the web interface and CLI. **Collections**: A collection represents a set of data pulled from scan results, to be used in later aggregation and analysis stages. Each rule can have multiple collections. **Aggregations**: An aggregation buckets the collected data into groups for analysis in distinct groups of data elements. **Analysis**: Analysis performs (you guessed it) analysis on the data to whittle down the data elements to what ultimately gets reported. For example, the analysis stage may look only for cases where the data field is repeated in the data set, indicating it was found multiple times and therefore discarding any only appearing once. **Headline**: The headline represents the actual correlation title that summarizes what was found. You can think of this as equivalent to a meal name (beef stew), and all the data elements as being the ingredients (beef, tomatoes, onions, etc.). ### Creating a rule To create your own rule, simply copy the `template.yaml` file in the `correlations` folder to a meaningful name that matches the ID you intend to provide it, e.g. `aws_cloud_usage.yaml` and edit the rule to fit your needs. Save it and re-start SpiderFoot for the rule to be loaded. If there are any syntax errors, SpiderFoot will abort at startup and (hopefully) give you enough information to know where the error is. The `template.yaml` file is also a good next point of reference to better understand the structure of the rules and how to use them. We also recommend taking a look through the actual rules themselves to see the concepts in practice. ## Rule Reference **id**: The internal ID for this rule, which needs to match the filename. **version**: The rule syntax version. This must be `1` for now. **meta**: This section contains a few important fields used to describe the rule. * **name**: A short, human readable name for the rule. * **description**: A longer (can be multi-paragraph) description of the rule. * **risk**: The risk level represented by this rule's findings. Can be `INFO`, `LOW`, `MEDIUM`, `HIGH`. **collection**: A correlation rule contains one or more `collect` blocks. Each `collect` block contains one or more `method` blocks telling SpiderFoot what criteria to use for extracting data from the database and how to filter it down. * **collect**: Technically, the first `method` block in each `collect` block is what actually pulls data from the database, and each subsequent `method` refines that dataset down to what you’re seeking. You may have multiple `collect` blocks overall but the rule remains that within each `collect`, the first `method` pulls data from the database and subsequent `method` blocks within the `collect` refine that data. * **method**: Each `method` block tells SpiderFoot how to collect and refine data. Each `collect` must contain at least one `method` block. Valid methods are `exact` for performing an exact match of the chosen `field` to the supplied `value`, or `regex` to perform regular expression matching. * **field**: Each `method` block has a `field` upon which the matching should be performed. Valid fields are `type` (e.g. `INTERNET_NAME`), `module` (e.g. `sfp_whois`) and `data`, which would be the value of the data element (e.g. in case of an `INTERNET_NAME`, the `data` would be the hostname). After the first `method` block, you can also prefix the field with `source.`, `child.` or `entity.` to refer to the fields of the source, children or relevant entities of the collected data, respectively (see `multiple_malicious.yaml` and `data_from_docmeta.yaml` as examples of this approach). * **value**: Here you supply the value or values you wish to match against the field you supplied in `field`. If your `method` was `regex`, this would be a regular expression. **aggregation**: With all the data elements in their collections, you can now aggregate them into buckets for further analysis or immediately generate results from the rule. While the collection phase is about obtaining the data from the database and filtering down to data of interest, the aggregation phase is about grouping that data in different ways in order to support analysis and/or grouping reported results. Aggregation simply iterates through the data elements in each collection and places them into groups based on the `field` specified. For instance if you pick the `type` field, you’ll end up with data elements with the same `type` field grouped together. The purpose of this grouping is two-fold: to support the analysis stage, or if you don’t have the analysis stage, it’s how your correlation results will be grouped for the user. * **field**: The `field` defines how you'd like your data elements grouped together. Just like the `field` option in `method` blocks above, you may prefix the field with `source.`, `child.` or `entity.` to apply the aggregation on those fields of the data element instead. For example, if you intended to look for multiple occurrences of a hostname, you would specify `data` here as that field, since you want to count the number of times the value of the `data` field appears. **analysis** The analysis section applies (you guessed it) some analysis to the aggregated results or collections directly if you didn’t perform any aggregation, and drops candidate results if they fail this stage. Various analysis `method` types exist, and each takes different options, described below. * **method**: * **threshold**: Drop any collection/aggregation of data elements that do not meet the defined thresholds. You would use this analysis rule when wanting to generate a result only when a data element has appeared more or less than a limit specified, for instance reporting when an email address is reported just once, or more than 100 times. * **field**: The field you want to apply the threshold too. As per above, you can use `child.`, `source.` and `entity.` field prefixes here too. * **count_unique_only**: By default the threshold is applied to the `field` specified on all data elements, but by setting `count_unique_only` to `true`, you can limit the threshold to only unique values in the `field` specified, so as not to also count duplicates. * **minimum**: The minimum number of data elements that must appear within the collection or aggregation. * **maximum**: The maximum number of data elements that must appear within the collection or aggregation. * **outlier**: Only keep outliers within any collection/aggregation of data elements. * **maximum_percent**: The maximum percentage of the overall results that an aggregation can represent. This method requires that you have performed an aggregation on a certain field in order to function. For example, if you aggregate on the `data` field of your collections, and one of those buckets contains less than 10% of the overall volume, it will be reported as an outlier. * **noisy_percent**: By default this is `10`, meaning that if the average percentage every bucket is below 10%, don't report outliers since the dataset is anomalous. * **first_collection_only**: Only keep data elements that appeared in the first collection but not any others. For example, this is handy for finding cases where data was found from one or several data sources but not others. * **field**: The field you want to use for looking up between collections. * **match_all_to_first_collection**: Only keep data elements that have matched in some way to the first collection. This requires an aggregation to have been performed, as the field used for aggregation is what will be used for checking for a match. * **match_method**: How to match between all collections and the first collection. Options are `contains` (simple wildcard match), `exact` and `subnet` which reports a match if the expected field may contain an IP address that is within the first collection field containing a subnet. **headline** After all data elements have been collected, filtered down, aggregated and analyzed, if data elements are remaining, these are what we call "correlation results" -> the results of your correlation rule. These need a "headline" to summarize the findings, which you can define here. To place any value from your data into the headline, you must enclose the field in `{}`, e.g. `{entity.data}`. There are two ways to write a `headline` rule. The typical way is to simply have `headline: titletexthere`, or have it as a block, in which case you can be more granular about how the correlation results are published: * **text**: The headline text, as described above. * **publish_collections**: The collection you wish to have associated with the correlation result. This is not often needed, but more in combination with the `match_all_to_first_collection` analysis rule in case your first collection is only used as a reference point and not actually contain any data elements you wish to publish with this correlation result. Take a look at the `egress_ip_from_wikipedia.yaml` rule for an example of this used in practice. ### A note about `child.`, `source.` and `entity.` field prefixes Every data element pulled in the first `match` rule in a collection will also have any children (data resulting from that data element), the source (the data element that this data element was generated from) and entity (the source, or source of source, etc. that was an entity like IP address, domain, etc.). This enables you to prefix subsequent (and only subsequent!) match block field names with `child.`, `source.` and `entity.` if you wish to match based on those fields. These prefixes, as shown above, can also be used in the `aggregation`, `analysis` and `headline` sections too. It is vital to note that these prefixes **always** are in reference to the first `match` block within each `collect` block, since every subsequent `match` block is always a refinement of the first `match` block. This can be complicated, so let's use an example to illustrate. Let's say your scan has found a hostname (a data element type of `INTERNET_NAME`) of `foo`, and it found that within some webpage content (a data element type of `TARGET_WEB_CONTENT`) of "This is some web content: foo", which was from a URL (data element type of `LINKED_URL_INTERNAL`) of "https://bar/page.html", which was from another host named `bar`. Here's the data discovery path: `bar` [`INTERNET_NAME`] -> `https://bar/page.html` [`LINKED_URL_INTERNAL`] -> `This is some web content: foo` [`TARGET_WEB_CONTENT`] -> `foo` [`INTERNET_NAME`] If we were to look at `This is some web content: foo` in our rule, here are the `data` and `type` fields you would expect to exist (`module` would also exist but has been left out of this example for brevity): * `data`: `This is some web content: foo` * `type`: `TARGET_WEB_CONTENT` * `source.data`: `https://bar/page.html` * `source.type`: `LINKED_URL_INTERNAL` * `child.data`: `foo` * `child.type`: `INTERNET_NAME` * `entity.type`: `INTERNET_NAME` * `entity.data`: `bar` Notice how the `entity.type` and `entity.data` fields for "This is some web content: foo" is **not** the `LINKED_URL_INTERNAL` data element, but actually the `bar` `INTERNET_NAME` data element. This is because an `INTERNET_NAME` is an entity, but a `LINKED_URL_INTERNAL` is not. You can look in `spiderfoot/db.py` to see which data types are entities and which are not. ================================================ FILE: correlations/cert_expired.yaml ================================================ id: cert_expired version: 1 meta: name: Expired SSL certificate found description: > A host was found with an expired SSL certificate. This may pose a risk to the security of the service exposed and/or cause connecting services to fail due to being unable to verify the certificate. risk: MEDIUM collections: - collect: - method: exact field: type value: SSL_CERTIFICATE_EXPIRED aggregation: field: source.data headline: "Expired SSL certificate found: {source.data}" ================================================ FILE: correlations/cloud_bucket_open.yaml ================================================ id: cloud_bucket_open version: 1 meta: name: Cloud storage bucket open to the Internet description: > A cloud storage bucket (e.g. S3) referenced from the target website has been found to be open to the Internet. Such buckets should be restricted so that contents cannot be listed, even if needing to be publicly accessible. risk: HIGH collections: - collect: - method: exact field: type value: CLOUD_STORAGE_BUCKET - method: exact field: source.type value: LINKED_URL_EXTERNAL - method: exact field: child.type value: CLOUD_STORAGE_BUCKET_OPEN aggregation: field: data headline: "Cloud storage bucket found open: {data}" ================================================ FILE: correlations/cloud_bucket_open_related.yaml ================================================ id: cloud_bucket_open_related version: 1 meta: name: Possibly related cloud storage bucket open to the Internet description: > A cloud storage bucket (e.g. S3) potentially related to the target has been found to be open to the Internet. As the buckets in this case are based on name-matching, verification for actual association with the target is necessary. risk: LOW collections: - collect: - method: exact field: type value: CLOUD_STORAGE_BUCKET - method: exact field: source.type value: not LINKED_URL_EXTERNAL - method: exact field: child.type value: CLOUD_STORAGE_BUCKET_OPEN aggregation: field: data headline: "Potentially relevant cloud storage bucket found open: {data}" ================================================ FILE: correlations/data_from_base64.yaml ================================================ id: data_from_base64 version: 1 meta: name: Data was found within base64-encoded data description: > Possibly interesting data was found within base64-encoded data, such as software versions, names, email addresses and hostnames. risk: INFO collections: - collect: - method: exact field: type value: BASE64_DATA - method: regex field: child.data value: .* - method: exact field: child.type value: not HASH aggregation: field: child.data headline: "Interesting data was found within base64-encoded data: '{child.data}'" ================================================ FILE: correlations/data_from_docmeta.yaml ================================================ id: data_from_docmeta version: 1 meta: name: Data was found within document/image meta data description: > Possibly interesting data was found within document/image meta data, such as software versions, names, email addresses and hostnames. risk: INFO collections: - collect: - method: exact field: type value: RAW_FILE_META_DATA - method: regex field: child.data value: .* - method: exact field: child.type value: not HASH aggregation: field: child.data headline: "Interesting data was found within document meta data: '{child.data}'" ================================================ FILE: correlations/database_exposed.yaml ================================================ id: database_exposed version: 1 meta: name: Database server exposed to the Internet description: > A database technology (MySQL, Oracle, Postgres, Redis, Hadoop, MongoDB, Spark) was found to be accessible over the Internet. Even if authentication is required such systems should not be exposed over the Internet due to the risk of misconfiguration or unpatched vulnerabilities. risk: HIGH collections: - collect: - method: exact field: type value: TCP_PORT_OPEN - method: regex field: data value: # MySQL - .*:3306$ # Oracle - .*:1521$ # PostgreSQL - .*:5432$ # Redis - .*:6379$ - .*:6380$ # Hadoop - .*:50070$ - .*:50470$ - .*:50090$ - .*:500[12]0$ - .*:50475$ - .*:50075$ - .*:8020$ - .*:9000$ # Spark - .*:7077$ # MongoDB - .*:2701[789]$ aggregation: field: data headline: "Database server exposed to the Internet: {data}" ================================================ FILE: correlations/dev_or_test_system.yaml ================================================ id: dev_or_test_system version: 1 meta: name: A host appearing to be a test or development system was found description: > A host that has dev, test, staging, internal or uat in its name was found. This may indicate a system that is exposed over the Internet unintentionally, and/or may be less secure than other systems. risk: MEDIUM collections: - collect: - method: exact field: type value: INTERNET_NAME - method: regex field: data value: - .*dev.* - .*test.* - .*uat.* - .*internal.* - .*staging.* aggregation: field: data headline: "Development or internal system found: {data}" ================================================ FILE: correlations/dns_zone_transfer_possible.yaml ================================================ id: dns_zone_transfer_possible version: 1 meta: name: DNS Zone Transfer is possible description: > It was possible to perform a DNS Zone Transfer with the target's name server. Zone transfers can expose a significant amount of information about the target network and have no real need to be enabled in modern networks. risk: HIGH collections: - collect: - method: exact field: module value: sfp_dnszonexfer aggregation: field: source.data headline: "DNS Zone Transfer possible with {source.data}" ================================================ FILE: correlations/egress_ip_from_wikipedia.yaml ================================================ id: egress_ip_from_wikipedia version: 1 meta: name: Wikipedia page edit from target-owned network description: > A Wikipedia page edit was found to come from an IP address within a network owned by the target. That IP address is highly likely to be that of a VPN server or office egress proxy/gateway. risk: MEDIUM collections: - collect: - method: exact field: type value: NETBLOCK_OWNER - collect: - method: exact field: type value: IP_ADDRESS - method: exact field: child.type value: WIKIPEDIA_PAGE_EDIT aggregation: field: data analysis: - method: match_all_to_first_collection field: data # Could also be 'contains' or 'exact' match_method: subnet headline: text: "Wikipedia edit from IP within target-owned network: {data}" publish_collections: - 1 ================================================ FILE: correlations/email_in_multiple_breaches.yaml ================================================ id: email_in_multiple_breaches version: 1 meta: name: > An email address was reported to be in multiple breaches description: > An email address was reported to be in multiple breaches. The presence in multiple breaches may indicate that the password of the account is particularly weak, or that it was re-used across the sites involved in the breaches. Note that some breaches simply aggregate other breaches, or may be very old. risk: HIGH collections: - collect: - method: exact field: type value: EMAILADDR_COMPROMISED aggregation: field: source.data analysis: - method: threshold field: source.data minimum: 2 headline: "Email address reported in multiple breaches: {source.data}" ================================================ FILE: correlations/email_in_whois.yaml ================================================ id: email_in_whois version: 1 meta: name: A target-relevant email address was found in Whois data description: > An email address was found in Whois data that is directly related to the target (e.g. same domain). The email found may be that of a highly privileged person responsible for maintaining the infrastructure of the target. Email addresses are increasingly rarely found in Whois data due to GDPR. risk: INFO collections: - collect: - method: exact field: type value: EMAILADDR - method: regex field: source.type value: .*WHOIS.* aggregation: field: data headline: "Email address found in Whois record: {data}" ================================================ FILE: correlations/email_only_from_pasteleak_site.yaml ================================================ id: email_only_from_pasteleak_site version: 1 meta: name: Email address only from paste/leak site description: > An email address was found mentioned in a paste/leak site but nowhere else. Since the email address was not found anywhere else, this may indicate that the address is in some way special, perhaps not intended to be publicly exposed/used or targeted in an attack. risk: MEDIUM collections: - collect: - method: exact field: type value: EMAILADDR - method: exact field: source.type value: - LEAKSITE_CONTENT - collect: - method: exact field: type value: EMAILADDR - method: exact field: source.type value: - not LEAKSITE_CONTENT aggregation: field: data analysis: - method: first_collection_only field: data headline: "Email address found only in paste/leak site: {data}" ================================================ FILE: correlations/host_only_from_bruteforce.yaml ================================================ id: host_only_from_bruteforce version: 1 meta: name: Host only from bruteforcing description: > A hostname was found only by brute-forcing but nowhere else. Since the host was not found anywhere else, this may indicate that the host is in some way special, perhaps not intended to be publicly exposed/used. risk: LOW collections: - collect: - method: exact field: type value: INTERNET_NAME - method: exact field: module value: sfp_dnsbrute - collect: - method: exact field: type value: INTERNET_NAME - method: exact field: module value: not sfp_dnsbrute aggregation: field: data analysis: - method: first_collection_only field: data headline: "Host found only through bruteforcing: {data}" ================================================ FILE: correlations/host_only_from_certificatetransparency.yaml ================================================ id: host_only_from_certificatetransparency version: 1 meta: name: Hostname only from certificate transparency description: > A hostname was found from certificate transparency but nowhere else. Since the host was not found anywhere else, this may indicate that the host is in some way special, perhaps not intended to be publicly exposed/used. risk: LOW collections: - collect: - method: exact field: type value: INTERNET_NAME - method: exact field: module value: - sfp_crt - sfp_certspotter - collect: - method: exact field: type value: INTERNET_NAME - method: exact field: module value: - not sfp_crt - not sfp_certspotter - not sfp_dnsresolve aggregation: field: data analysis: - method: first_collection_only field: data headline: "Host found only in certificate transparency: {data}" ================================================ FILE: correlations/http_errors.yaml ================================================ id: http_errors version: 1 meta: name: Multiple HTTP errors found description: > Non-successful HTTP error codes were encountered. 401 and 403 are not included as these refer to authentication/authorization failures. risk: LOW collections: - collect: - method: exact field: type value: HTTP_CODE - method: regex field: data value: ^[4-9].*$ # Leave out authentication/authorization failures - method: regex field: data value: not 40[13] aggregation: field: entity.data analysis: - method: threshold minimum: 2 field: data headline: "Multiple failure HTTP codes found at {entity.data}" ================================================ FILE: correlations/human_name_in_whois.yaml ================================================ id: human_name_in_whois version: 1 meta: name: A human name was found in Whois data description: > A human name was found in Whois data. The name found may be that of a highly privileged person responsible for maintaining the infrastructure of the target. Names are increasingly rarely found in Whois data due to GDPR. risk: INFO collections: - collect: - method: exact field: type value: HUMAN_NAME - method: regex field: source.type value: .*WHOIS.* aggregation: field: data headline: "Human name found in Whois record: {data}" ================================================ FILE: correlations/internal_host.yaml ================================================ id: internal_host version: 1 meta: name: A host resolving to unroutable IPs was found description: > A host that resolves to an IP address on ranges reserved for internal use (10.0.0.0/8, etc.) was found to be publicly resolvable. This might reveal information about the internal infrastructure and workings of the target. risk: MEDIUM collections: - collect: - method: exact field: type value: INTERNAL_IP_ADDRESS - collect: - method: exact field: type value: IP_ADDRESS - method: regex field: data value: - ^192\\.168\\..* - ^10\\..* - method: exact field: module value: sfp_dnsresolve aggregation: field: data headline: "An internal host was found: {data}" ================================================ FILE: correlations/multiple_malicious.yaml ================================================ id: multiple_malicious version: 1 meta: name: > An IP, host, subnet or email address was considered malicious by multiple sources description: > An IP, host, subnet or email address was considered malicious by multiple sources. Such cases have a high likelihood of being genuinely malicious and should be urgently investigated. Even if the entity in question is not compromised, it's likely to be blocked across parts of the Internet due to its presence in these lists. risk: HIGH collections: - collect: - method: regex field: type value: - MALICIOUS_* - BLACKLIST_* # Filter out all subnets - method: regex field: source.data value: not .*/.* # Filter out affiliates and Co-hosts - method: regex field: type value: - not .*COHOST.* - not .*AFFILIATE.* aggregation: field: source.data analysis: - method: threshold field: source.data minimum: 2 headline: "Entity considered malicious by multiple sources: {source.data}" ================================================ FILE: correlations/multiple_malicious_affiliate.yaml ================================================ id: multiple_malicious_affiliate version: 1 meta: name: > An affiliated IP or host was considered malicious by multiple sources description: > An affiliated IP or host was considered malicious by multiple sources. Such cases have a high likelihood of being genuinely malicious and should be investigated depending on the nature of the relationship between the target and the affiliate. Even if the entity in question is not compromised, it's likely to be blocked across parts of the Internet due to its presence in these lists and may therefore have an impact on the target. risk: LOW collections: - collect: - method: regex field: type value: - MALICIOUS_* - BLACKLIST_* # Filter to only affiliated entities - method: regex field: type value: .*AFFILIATE.* aggregation: field: source.data analysis: - method: threshold field: source.data minimum: 2 headline: "Affiliated entity considered malicious by multiple sources: {source.data}" ================================================ FILE: correlations/multiple_malicious_cohost.yaml ================================================ id: multiple_malicious_cohost version: 1 meta: name: > A co-hosted site was considered malicious by multiple sources description: > A co-hosted site was considered malicious by multiple sources. Such cases have a high likelihood of being genuinely malicious and should be investigated depending on the nature of the relationship between the target and the co-host. Even if the entity in question is not compromised, it's likely to be blocked across parts of the Internet due to its presence in these lists and may therefore have an impact on the target. risk: LOW collections: - collect: - method: regex field: type value: - MALICIOUS_* - BLACKLIST_* # Filter to only co-hosted sites - method: regex field: type value: .*COHOST.* aggregation: field: source.data analysis: - method: threshold field: source.data minimum: 2 headline: "Co-hosted site considered malicious by multiple sources: {source.data}" ================================================ FILE: correlations/name_only_from_pasteleak_site.yaml ================================================ id: name_only_from_pasteleak_site version: 1 meta: name: Human name only from paste/leak site description: > A human name was found mentioned in a paste/leak site but nowhere else. Since the name was not found anywhere else, this may indicate that the name is in some way special, perhaps not intended to be publicly exposed/used or targeted in an attack. risk: MEDIUM collections: - collect: - method: exact field: type value: INTERNET_NAME - method: exact field: source.type value: - LEAKSITE_CONTENT - collect: - method: exact field: type value: INTERNET_NAME - method: exact field: source.type value: - not LEAKSITE_CONTENT aggregation: field: data analysis: - method: first_collection_only field: data headline: "Human name found only in paste/leak site: {data}" ================================================ FILE: correlations/open_port_version.yaml ================================================ id: open_port_version version: 1 meta: name: Open TCP port reveals version description: > A possible software version has been revealed on an open port. Such information may reveal the use of old/unpatched software used by the target. risk: INFO collections: - collect: - method: exact field: type value: TCP_PORT_OPEN_BANNER - method: regex field: data value: .*[0-9]\.[0-9].* - method: regex field: data value: not .*Mime-Version.* - method: regex field: data value: not .*HTTP/1.* aggregation: field: data headline: "Software version revealed on open port: {data}" ================================================ FILE: correlations/outlier_cloud.yaml ================================================ id: outlier_cloud version: 1 meta: name: Outlier Cloud provider description: > A cloud provider that appeared in 10% or less of the total providers found. Outliers can often reveal entities that are rare and therefore interesting. Particularly in the case of cloud providers, an outlier may be an indicator of Shadow IT or unmtaintained infrastructure. risk: MEDIUM collections: - collect: - method: exact field: type value: PROVIDER_HOSTING aggregation: field: data analysis: - method: outlier maximum_percent: 10 headline: "Outlier cloud provider found: {data}" ================================================ FILE: correlations/outlier_country.yaml ================================================ id: outlier_country version: 1 meta: name: Outlier country description: > A country that appeared in 10% or less of the total countries found. Outliers can often reveal entities that are rare and therefore interesting. risk: INFO collections: - collect: - method: exact field: type value: COUNTRY_NAME aggregation: field: data analysis: - method: outlier maximum_percent: 10 headline: "Outlier country found: {data}" ================================================ FILE: correlations/outlier_email.yaml ================================================ id: outlier_email version: 1 meta: name: Outlier email address found description: > An email address that appeared in 10% or less of the total email addresses found. Outliers can often reveal entities that are rare and therefore interesting. risk: INFO collections: - collect: - method: exact field: type value: EMAILADDR aggregation: field: data analysis: - method: outlier maximum_percent: 10 headline: "Outlier email address found: {data}" ================================================ FILE: correlations/outlier_hostname.yaml ================================================ id: outlier_hostname version: 1 meta: name: Outlier hostname description: > A hostname that appeared in 10% or less of the total hostnames found. Outliers can often reveal entities that are rare and therefore interesting. risk: INFO collections: - collect: - method: exact field: type value: INTERNET_NAME aggregation: field: data analysis: - method: outlier maximum_percent: 10 headline: "Outlier hostname found: {data}" ================================================ FILE: correlations/outlier_ipaddress.yaml ================================================ id: outlier_ipaddress version: 1 meta: name: Outlier IP address description: > An IP address that appeared in 10% or less of the total IP addresses found. Outliers can often reveal entities that are rare and therefore interesting. risk: INFO collections: - collect: - method: exact field: type value: - IP_ADDRESS - IPV6_ADDRESS aggregation: field: data analysis: - method: outlier maximum_percent: 10 headline: "Outlier IP address found: {data}" ================================================ FILE: correlations/outlier_registrar.yaml ================================================ id: outlier_registrar version: 1 meta: name: Outlier registrar description: > A registrar that appeared in 10% or less of the total registrars found. Outliers can often reveal entities that are rare and therefore interesting. Particularly in the case of registrars, an outlier may be an indicator of Shadow IT or unmtaintained infrastructure. risk: MEDIUM collections: - collect: - method: exact field: type value: DOMAIN_REGISTRAR aggregation: field: data analysis: - method: outlier maximum_percent: 10 headline: "Outlier registrar found: {data}" ================================================ FILE: correlations/outlier_webserver.yaml ================================================ id: outlier_webserver version: 1 meta: name: Outlier web server description: > A web server that appeared in 10% or less of the total web servers found. Outliers can often reveal entities that are rare and therefore interesting. Particularly in the case of web servers, an outlier may be an indicator of Shadow IT or unmtaintained infrastructure. risk: MEDIUM collections: - collect: - method: exact field: type value: WEBSERVER_BANNER aggregation: field: data analysis: - method: outlier maximum_percent: 10 headline: "Outlier web server found: {data}" ================================================ FILE: correlations/remote_desktop_exposed.yaml ================================================ id: remote_desktop_exposed version: 1 meta: name: Remote desktop technology found exposed to the Internet description: > A remote desktop technology (RDP, VNC, NoMachine) was found to be accessible over the Internet. Even if authentication is required such systems should not be exposed over the Internet due to the risk of misconfiguration or unpatched vulnerabilities. risk: HIGH collections: - collect: - method: exact field: type value: TCP_PORT_OPEN - method: regex field: data value: - .*:5900$ - .*:3389$ - .*:4000$ aggregation: field: source.data headline: "Remote desktop exposed to the Internet: {source.data}" ================================================ FILE: correlations/root_path_needs_auth.yaml ================================================ id: root_path_needs_auth version: 1 meta: name: Website root path needs authentication description: > The base path of a server needs authentication, indicating the server is likely sensitive or important. risk: INFO collections: - collect: - method: exact field: type value: HTTP_CODE - method: regex field: data value: 40[31] # The source might be a host or URL - method: regex field: source.data value: - ^https?://[a-zA-Z0-9\.\-]+/?#?$ - ^[a-zA-Z0-9\.\-]+$ aggregation: field: source.data headline: "Base URL requires authentication: {source.data}" ================================================ FILE: correlations/stale_host.yaml ================================================ id: stale_host version: 1 meta: name: Stale host found description: > A host appears to be stale based upon various indicators such as unsuccessful HTTP codes, expired SSL certificates, error messages, vulnerabilities and junk files found. Such hosts may be unmaintained, exposing the target to security risks. risk: HIGH collections: - collect: - method: exact field: type value: SSL_CERTIFICATE_EXPIRED - collect: - method: exact field: type value: HTTP_CODE - method: regex field: data value: not ^[2-3].*$ - method: regex field: data value: not 40[13] - collect: - method: exact field: type value: ERROR_MESSAGE - collect: - method: exact field: type value: JUNK_FILE - collect: - method: regex field: type value: VULNERABILITY_.* aggregation: field: entity.data analysis: - method: threshold field: type # Avoid multiple of the same type triggering. This means # a minimum of 3 of different types must appear in one # bucket. count_unique_only: true minimum: 3 headline: "Potentially stale/unmaintained host: {entity.data}" ================================================ FILE: correlations/strong_affiliate_certs.yaml ================================================ id: strong_affiliate_certs version: 1 meta: name: Affiliated host/domain with strong relationship to the target description: > A host or domain was found to be referenced multiple times from SSL certificates covering multiple target hosts. Strong affiliates indicate potential targets that should be included in the scope of scanning to ensure more complete coverage. risk: INFO collections: - collect: - method: exact field: module value: - sfp_sslcert - sfp_crt - sfp_certspotter - method: exact field: source.type value: - INTERNET_NAME - DOMAIN_NAME - method: exact field: type value: - AFFILIATE_INTERNET_NAME - AFFILIATE_DOMAIN_NAME - CO_HOSTED_SITE - CO_HOSTED_SITE_DOMAIN aggregation: field: data analysis: - method: threshold field: source.data count_unique_only: true minimum: 2 headline: "Affiliate with strong target relationship: {data}" ================================================ FILE: correlations/strong_similardomain_crossref.yaml ================================================ id: strong_similardomain_crossref version: 1 meta: name: Similar domain with strong relationship to the target description: > A similar domain was found to be referenced by the target, and references back, indicating a strong relationship. Strong affiliates indicate potential targets that should be included in the scope of scanning to ensure more complete coverage. risk: INFO collections: - collect: - method: exact field: module value: sfp_crossref - method: exact field: type value: AFFILIATE_INTERNET_NAME - method: exact field: source.type value: SIMILARDOMAIN aggregation: field: data headline: "Similar domain with strong target relationship: {data}" ================================================ FILE: correlations/template.yaml ================================================ # Set the ID for the rule. Ensure it matches the filename (without the .yaml # extension) and has no spaces or other special characters. id: set_a_meaningful_id_here # Use version 1 for now. version: 1 # The meta section sets meta attributes about the rule. meta: # The human-readable name for the rule name: This is a briefly descriptive name. # A longer description for the rule description: > This is a more detailed description about the rule and ideally includes some rationale explaining the risk posed, or other relevant information that helps the user better understand the value of the rule results. You can have multiple paragraphs here too. For this example rule, our goal is to find all hosts that match "foo" or "bar", creating a correlation result when such a host is found more than once. # The risk the results of this rule posed. They can be one of: # - HIGH (reserved for genuine cases requiring immediate action) # - MEDIUM (potential high risk, needs deeper examination) # - LOW (potential risk, but probably low risk or false positive) # - INFO (no risk posed but potential interesting information) risk: MEDIUM # Define the collections to perform here. Refer to the README.md # in the correlations folder for more details. collections: # Each 'collect' block represents a collection of data from # the SpiderFoot database. You need at least one 'collect' # block for the rule to actually do something. - collect: # Each 'collect' block may contain multiple blocks # for collecting and analyzing data. The first pulls # data from the SpiderFoot database, and subsequent # blocks further refine that data down to what you are # interested in. You need at least one for the 'collect' # block to have data. # # For example, this rule will extract all data elements # that exactly match the INTERNET_NAME type. - method: exact field: type value: INTERNET_NAME # This next rule will filter out all the INTERNET_NAME # results where data doesn't match any of the regular # expressions provided. - method: regex field: data value: - .*foo.* - .*bar.* # What we are now left with in this collection are all # INTERNET_NAME elements with data matching either .*foo.* # or .*bar.* # With the data from your one or more collections, you can # now use 'aggregation' to define how they should be grouped # (if at all) for analysis at the next stage (also optional). aggregation: # Each data element will be placed into a bucket according # to its data field, which is the hostname since we have # INTERNET_NAME data elements. field: data # Analysis is an optional step to perform analysis on the # (optionally aggegated) data gathered to apply some criteria # determining whether a correlation creates results or not. analysis: # Here we want to only generate a correlation result when # the value of the data element's data field (the hostname # in this case) appears at least two times. - method: threshold minimum: 2 field: data # With the collection, aggegation and analysis performed, # we can now generate the correlation result with a title # that represents something meaningful to the user. headline: "A foo or bar host was found more than once: {data}" ================================================ FILE: correlations/vulnerability_critical.yaml ================================================ id: vulnerability_critical version: 1 meta: name: At least one CRITICAL-rated vulnerability was found description: > A vulnerability rated as CRITICAL was found on a host. Such vulnerabilities are typically easily exploitable and should be urgently patched. risk: HIGH collections: - collect: - method: exact field: type value: VULNERABILITY_CVE_CRITICAL aggregation: field: entity.data headline: "Critical-rated vulnerability found on {entity.data}" ================================================ FILE: correlations/vulnerability_high.yaml ================================================ id: vulnerability_high version: 1 meta: name: At least one HIGH-rated vulnerability was found description: > A vulnerability rated as HIGH was found on a host. Such vulnerabilities are typically easily exploitable and should be urgently patched. risk: HIGH collections: - collect: - method: exact field: type value: VULNERABILITY_CVE_HIGH aggregation: field: entity.data headline: "High-rated vulnerability found on {entity.data}" ================================================ FILE: correlations/vulnerability_mediumlow.yaml ================================================ id: vulnerability_mediumlow version: 1 meta: name: Multiple MEDIUM or LOW-rated vulnerabilities were found description: > Multiple vulnerabilities rated as MEDIUM or LOW were found on a host. Such vulnerabilities pose a lower risk to the target, however in aggregate may be an indication that the host is not well maintained. risk: MEDIUM collections: - collect: - method: exact field: type value: - VULNERABILITY_CVE_MEDIUM - VULNERABILITY_CVE_LOW - VULNERABILITY_GENERAL aggregation: field: entity.data analysis: - method: threshold field: entity.data minimum: 2 headline: "Multiple medium/low vulnerabilities found on {entity.data}" ================================================ FILE: docker-compose-dev.yml ================================================ version: "3" services: spiderfoot: volumes: - .:/home/spiderfoot ================================================ FILE: docker-compose-full.yml ================================================ version: "3" services: spiderfoot: build: context: ./ dockerfile: ./Dockerfile.full ================================================ FILE: docker-compose.yml ================================================ version: "3" # Basic usage: # $ docker-compose up # # Dev environment (code directory mapped into container): # $ docker-compose -f docker-compose.yml -f docker-compose-dev.yml up # # Full image (all CLI tools installed): # $ docker-compose -f docker-compose.yml -f docker-compose-full.yml up # # Spiderfoot data resides in a Docker volume # # $ ls -lh /var/lib/docker/volumes/spiderfoot_spiderfoot-data/_data # total 104K # drwxr-xr-x 2 user user 4.0K Sep 22 09:51 cache # -rw-r--r-- 1 user user 100K Sep 22 15:19 spiderfoot.db services: spiderfoot: build: context: ./ volumes: - spiderfoot-data:/var/lib/spiderfoot image: spiderfoot container_name: spiderfoot ports: - "5001:5001" restart: unless-stopped volumes: spiderfoot-data: ================================================ FILE: docs/Makefile ================================================ # Minimal makefile for Sphinx documentation # # You can set these variables from the command line, and also # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) ================================================ FILE: docs/conf.py ================================================ # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html # -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # # import os # import sys # sys.path.insert(0, '/root/Desktop/spiderfoot/spiderfoot') # -- Project information ----------------------------------------------------- project = 'SpiderFoot' copyright = '2012, Steve Micallef' author = 'Steve Micallef' # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'sphinx.ext.viewcode', 'sphinx.ext.todo', ] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. language = 'en' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # html_theme = 'alabaster' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] # -- Extension configuration ------------------------------------------------- # -- Options for todo extension ---------------------------------------------- # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True ================================================ FILE: docs/index.rst ================================================ .. spiderfoot documentation master file, created by sphinx-quickstart on Sat Jun 26 01:55:34 2021. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. Welcome to spiderfoot's documentation! ====================================== .. toctree:: :maxdepth: 4 :caption: Contents: spiderfoot Indices and tables ================== * :ref:`genindex` * :ref:`modindex` * :ref:`search` ================================================ FILE: docs/make.bat ================================================ @ECHO OFF pushd %~dp0 REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) set SOURCEDIR=. set BUILDDIR=_build if "%1" == "" goto help %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. echo.The 'sphinx-build' command was not found. Make sure you have Sphinx echo.installed, then set the SPHINXBUILD environment variable to point echo.to the full path of the 'sphinx-build' executable. Alternatively you echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from echo.http://sphinx-doc.org/ exit /b 1 ) %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end popd ================================================ FILE: docs/spiderfoot.rst ================================================ spiderfoot package ================== Submodules ---------- spiderfoot.db module -------------------- .. automodule:: spiderfoot.db :members: :undoc-members: :show-inheritance: spiderfoot.event module ----------------------- .. automodule:: spiderfoot.event :members: :undoc-members: :show-inheritance: spiderfoot.helpers module ------------------------- .. automodule:: spiderfoot.helpers :members: :undoc-members: :show-inheritance: spiderfoot.plugin module ------------------------ .. automodule:: spiderfoot.plugin :members: :undoc-members: :show-inheritance: spiderfoot.target module ------------------------ .. automodule:: spiderfoot.target :members: :undoc-members: :show-inheritance: Module contents --------------- .. automodule:: spiderfoot :members: :undoc-members: :show-inheritance: ================================================ FILE: generate-certificate ================================================ #!/bin/sh if ! command -v openssl >/dev/null 2>&1 ; then echo "Error: Could not find openssl in \$PATH: $PATH" exit 1 fi if test -f spiderfoot.key; then echo "Error: spiderfoot.key already exists" exit 1 fi if test -f spiderfoot.crt; then echo "Error: spiderfoot.crt already exists" exit 1 fi openssl req -new -newkey rsa:4096 -sha256 -x509 -days 365 -nodes -out spiderfoot.crt -keyout spiderfoot.key -subj "/CN=localhost" chmod 600 spiderfoot.crt chmod 600 spiderfoot.key ================================================ FILE: modules/__init__.py ================================================ ================================================ FILE: modules/sfp__stor_db.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_stor_db # Purpose: SpiderFoot plug-in for storing events to the local SpiderFoot # SQLite database. # # Author: Steve Micallef # # Created: 14/05/2012 # Copyright: (c) Steve Micallef 2012 # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootPlugin class sfp__stor_db(SpiderFootPlugin): meta = { 'name': "Storage", 'summary': "Stores scan results into the back-end SpiderFoot database. You will need this." } _priority = 0 # Default options opts = { 'maxstorage': 1024, # max bytes for any piece of info stored (0 = unlimited) '_store': True } # Option descriptions optdescs = { 'maxstorage': "Maximum bytes to store for any piece of information retrieved (0 = unlimited.)" } def setup(self, sfc, userOpts=dict()): self.sf = sfc for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input # Because this is a storage plugin, we are interested in everything so we # can store all events for later analysis. def watchedEvents(self): return ["*"] # Handle events sent to this module def handleEvent(self, sfEvent): if not self.opts['_store']: return if self.opts['maxstorage'] != 0 and len(sfEvent.data) > self.opts['maxstorage']: self.debug("Storing an event: " + sfEvent.eventType) self.__sfdb__.scanEventStore(self.getScanId(), sfEvent, self.opts['maxstorage']) return self.debug("Storing an event: " + sfEvent.eventType) self.__sfdb__.scanEventStore(self.getScanId(), sfEvent) # End of sfp__stor_db class ================================================ FILE: modules/sfp__stor_stdout.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_stor_stdout # Purpose: SpiderFoot plug-in for dumping events to standard output. # # Author: Steve Micallef # # Created: 22/10/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootPlugin class sfp__stor_stdout(SpiderFootPlugin): meta = { 'name': "Command-line output", 'summary': "Dumps output to standard out. Used for when a SpiderFoot scan is run via the command-line." } _priority = 0 firstEvent = True # Default options opts = { "_format": "tab", # tab, csv, json "_requested": [], "_showonlyrequested": False, "_stripnewline": False, "_showsource": False, "_csvdelim": ",", "_maxlength": 0, "_eventtypes": dict() } # Option descriptions optdescs = { } def setup(self, sfc, userOpts=dict()): self.sf = sfc for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input # Because this is a storage plugin, we are interested in everything so we # can store all events for later analysis. def watchedEvents(self): return ["*"] def output(self, event): d = self.opts['_csvdelim'] if type(event.data) in [list, dict]: data = str(event.data) else: data = event.data if type(data) != str: data = str(event.data) if type(event.sourceEvent.data) in [list, dict]: srcdata = str(event.sourceEvent.data) else: srcdata = event.sourceEvent.data if type(srcdata) != str: srcdata = str(event.sourceEvent.data) if self.opts['_stripnewline']: data = data.replace("\n", " ").replace("\r", "") srcdata = srcdata.replace("\n", " ").replace("\r", "") if "" in data: data = data.replace("", "").replace("", "") if "" in srcdata: srcdata = srcdata.replace("", "").replace("", "") if self.opts['_maxlength'] > 0: data = data[0:self.opts['_maxlength']] srcdata = srcdata[0:self.opts['_maxlength']] if self.opts['_format'] == "tab": event_type = self.opts['_eventtypes'][event.eventType] if self.opts['_showsource']: print(f"{event.module.ljust(30)}\t{event_type.ljust(45)}\t{srcdata}\t{data}") else: print(f"{event.module.ljust(30)}\t{event_type.ljust(45)}\t{data}") if self.opts['_format'] == "csv": print((event.module + d + self.opts['_eventtypes'][event.eventType] + d + srcdata + d + data)) if self.opts['_format'] == "json": d = event.asDict() d['type'] = self.opts['_eventtypes'][event.eventType] if self.firstEvent: self.firstEvent = False else: print(",") print(json.dumps(d), end='') # Handle events sent to this module def handleEvent(self, sfEvent): if sfEvent.eventType == "ROOT": return if self.opts['_showonlyrequested']: if sfEvent.eventType in self.opts['_requested']: self.output(sfEvent) else: self.output(sfEvent) # End of sfp__stor_stdout class ================================================ FILE: modules/sfp_abstractapi.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_abstractapi # Purpose: Search AbstractAPI for domain, phone and IP address information. # # Author: Krishnasis Mandal # # Created: 29/07/2021 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_abstractapi(SpiderFootPlugin): meta = { 'name': "AbstractAPI", 'summary': "Look up domain, phone and IP address information from AbstractAPI.", 'flags': ["apikey"], 'useCases': ["Passive", "Footprint", "Investigate"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://app.abstractapi.com/", 'model': "FREE_NOAUTH_LIMITED", 'references': [ "https://app.abstractapi.com/", ], 'apiKeyInstructions': [ "Visit https://app.abstractapi.com/users/signup", "Register a free account", "Visit https://app.abstractapi.com/api/", "Visit each API page and click on 'Try it out'", "Your API Key will be listed under 'This is your private API key, specific to this API.'", ], 'favIcon': "https://app.abstractapi.com/favicon.ico", 'logo': "https://app.abstractapi.com/logo192.png", 'description': "Abstract provides powerful APIs to help you enrich any user experience or automate any workflow." } } opts = { "companyenrichment_api_key": "", "phonevalidation_api_key": "", "ipgeolocation_api_key": "", } optdescs = { "companyenrichment_api_key": "AbstractAPI Company Enrichment API key.", "phonevalidation_api_key": "AbstractAPI Phone Validation API key.", "ipgeolocation_api_key": "AbstractAPI IP Geolocation API key.", } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.errorState = False self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["DOMAIN_NAME", "PHONE_NUMBER", "IP_ADDRESS", "IPV6_ADDRESS"] def producedEvents(self): return ["COMPANY_NAME", "SOCIAL_MEDIA", "GEOINFO", "PHYSICAL_COORDINATES", "PROVIDER_TELCO", "RAW_RIR_DATA"] def parseApiResponse(self, res: dict): if not res: self.error("No response from Abstract API.") return None # Rate limited to one request per second if res['code'] == '429': self.error("You are being rate-limited by AbstractAPI.") return None if res['code'] == '401': self.error("Unauthorized. Invalid AbstractAPI API key.") self.errorState = True return None if res['code'] == '422': self.error("Usage quota reached. Insufficient API credit.") self.errorState = True return None if res['code'] == '500' or res['code'] == '502' or res['code'] == '503': self.error("Abstract API service is unavailable") self.errorState = True return None if res['code'] == '204': self.debug("No response data for target") return None if res['code'] != '200': self.error(f"Unexpected reply from AbstractAPI: {res['code']}") return None if res['content'] is None: return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None def queryCompanyEnrichment(self, qry): """Enrich domain with company information. Args: qry (str): domain name Returns: dict: company information """ api_key = self.opts['companyenrichment_api_key'] if not api_key: return None params = urllib.parse.urlencode({ 'api_key': api_key, 'domain': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), }) res = self.sf.fetchUrl( f"https://companyenrichment.abstractapi.com/v1/?{params}", useragent=self.opts['_useragent'] ) time.sleep(1) if not res: self.debug("No response from AbstractAPI Company Enrichment API endpoint") return None return self.parseApiResponse(res) def queryPhoneValidation(self, qry): """Verify phone number and enrich with carrier and location information. Args: qry (str): phone number Returns: dict: phone number information """ api_key = self.opts['phonevalidation_api_key'] if not api_key: return None params = urllib.parse.urlencode({ 'api_key': api_key, 'phone': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), }) res = self.sf.fetchUrl( f"https://phonevalidation.abstractapi.com/v1/?{params}", useragent=self.opts['_useragent'] ) time.sleep(1) if not res: self.debug("No response from AbstractAPI Phone Validation API endpoint") return None return self.parseApiResponse(res) def queryIpGeolocation(self, qry): """Enrich IP address with geolocation information. Args: qry (str): IPv4 address Returns: dict: location information """ api_key = self.opts['ipgeolocation_api_key'] if not api_key: return None params = urllib.parse.urlencode({ 'api_key': api_key, 'ip_address': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), }) res = self.sf.fetchUrl( f"https://ipgeolocation.abstractapi.com/v1/?{params}", useragent=self.opts['_useragent'] ) time.sleep(1) if not res: self.debug("No response from AbstractAPI Phone Validation API endpoint") return None return self.parseApiResponse(res) def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if self.opts["companyenrichment_api_key"] == "" and self.opts["phonevalidation_api_key"] == "" and self.opts["ipgeolocation_api_key"] == "": self.error( f"You enabled {self.__class__.__name__} but did not set any API keys!" ) self.errorState = True return if eventName not in self.watchedEvents(): return if eventName == "DOMAIN_NAME": if self.opts["companyenrichment_api_key"] == "": self.info( f"No API key set for Company Enrichment API endpoint. Ignoring {eventData}" ) return data = self.queryCompanyEnrichment(eventData) if not data: return name = data.get('name') if not name: return if name == 'To Be Confirmed': return e = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(e) e = SpiderFootEvent("COMPANY_NAME", name, self.__name__, event) self.notifyListeners(e) linkedin_url = data.get('linkedin_url') if linkedin_url: if linkedin_url.startswith('linkedin.com'): linkedin_url = f"https://{linkedin_url}" e = SpiderFootEvent("SOCIAL_MEDIA", f"LinkedIn (Company): {linkedin_url}", self.__name__, event) self.notifyListeners(e) locality = data.get('locality') country = data.get('country') geoinfo = ', '.join( filter(None, [locality, country]) ) if geoinfo: e = SpiderFootEvent("GEOINFO", geoinfo, self.__name__, event) self.notifyListeners(e) elif eventName == "PHONE_NUMBER": if self.opts["phonevalidation_api_key"] == "": self.info( f"No API key set for Phone Validation API endpoint. Ignoring {eventData}" ) return data = self.queryPhoneValidation(eventData) if not data: return valid = data.get('valid') if not valid: return e = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(e) carrier = data.get('carrier') if carrier: e = SpiderFootEvent("PROVIDER_TELCO", carrier, self.__name__, event) self.notifyListeners(e) location = data.get('location') country = data.get('country') country_name = None if country: country_name = country.get('name') geoinfo = ', '.join( filter(None, [location, country_name]) ) if geoinfo: e = SpiderFootEvent("GEOINFO", geoinfo, self.__name__, event) self.notifyListeners(e) elif eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']: if self.opts["ipgeolocation_api_key"] == "": self.info( f"No API key set for IP Geolocation API endpoint. Ignoring {eventData}" ) return data = self.queryIpGeolocation(eventData) if not data: return e = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(e) geoinfo = ', '.join( [ _f for _f in [ data.get('city'), data.get('region'), data.get('postal_code'), data.get('country'), data.get('continent'), ] if _f ] ) if geoinfo: e = SpiderFootEvent("GEOINFO", geoinfo, self.__name__, event) self.notifyListeners(e) latitude = data.get('latitude') longitude = data.get('longitude') if latitude and longitude: e = SpiderFootEvent("PHYSICAL_COORDINATES", f"{latitude}, {longitude}", self.__name__, event) self.notifyListeners(e) # End of sfp_abstractapi class ================================================ FILE: modules/sfp_abusech.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_abusech # Purpose: Check if a host/domain, IP address or netblock is malicious according # to Abuse.ch. # # Author: steve@binarypool.com # # Created: 14/12/2013 # Copyright: (c) Steve Micallef, 2013 # Licence: MIT # ------------------------------------------------------------------------------- from netaddr import IPAddress, IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_abusech(SpiderFootPlugin): meta = { 'name': "abuse.ch", 'summary': "Check if a host/domain, IP address or netblock is malicious according to Abuse.ch.", 'flags': [], 'useCases': ["Passive", "Investigate"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://www.abuse.ch", 'model': "FREE_AUTH_UNLIMITED", 'references': [ "https://feodotracker.abuse.ch/", "https://sslbl.abuse.ch/", "https://urlhaus.abuse.ch/", ], 'apiKeyInstructions': [ "Visit https://bazaar.abuse.ch/api#api_key", "Login using a Twitter Account", "Navigate to 'Account Settings'", "The API key is listed under 'Your API Key'", "Visit https://urlhaus.abuse.ch/api/", "Login using a Twitter Account at https://urlhaus.abuse.ch/login/", "Navigate to https://urlhaus.abuse.ch/api/#account", "The API Key is listed under 'API-Key'" ], 'favIcon': "https://abuse.ch/favicon.ico", 'logo': "https://abuse.ch/images/abusech.svg", 'description': "abuse.ch is operated by a random swiss guy fighting malware for non-profit, " "running a couple of projects helping internet service providers and " "network operators protecting their infrastructure from malware.\n" "IT-Security researchers, vendors and law enforcement agencies rely on data from abuse.ch," "trying to make the internet a safer place.", } } # Default options opts = { 'abusefeodoip': True, 'abusesslblip': True, 'abuseurlhaus': True, 'checkaffiliates': True, 'checkcohosts': True, 'cacheperiod': 18, 'checknetblocks': True, 'checksubnets': True } # Option descriptions optdescs = { 'abusefeodoip': "Enable abuse.ch Feodo IP check?", 'abusesslblip': "Enable abuse.ch SSL Backlist IP check?", 'abuseurlhaus': "Enable abuse.ch URLhaus check?", 'checkaffiliates': "Apply checks to affiliates?", 'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?", 'cacheperiod': "Hours to cache list data before re-fetching.", 'checknetblocks': "Report if any malicious IPs are found within owned netblocks?", 'checksubnets': "Check if any malicious IPs are found within the same subnet of the target?" } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return [ "INTERNET_NAME", "IP_ADDRESS", "NETBLOCK_MEMBER", "AFFILIATE_INTERNET_NAME", "AFFILIATE_IPADDR", "CO_HOSTED_SITE", "NETBLOCK_OWNER" ] # What events this module produces def producedEvents(self): return [ "MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_SUBNET", "MALICIOUS_COHOST", "MALICIOUS_NETBLOCK" ] def queryFeodoTrackerBlacklist(self, target, targetType): blacklist = self.retrieveFeodoTrackerBlacklist() if not blacklist: return False if targetType == "ip": if target in blacklist: self.debug(f"IP address {target} found in Abuse.ch Feodo Tracker.") return True elif targetType == "netblock": netblock = IPNetwork(target) for ip in blacklist: if IPAddress(ip) in netblock: self.debug(f"IP address {ip} found within netblock/subnet {target} in Abuse.ch Feodo Tracker.") return True return False def retrieveFeodoTrackerBlacklist(self): blacklist = self.sf.cacheGet('abusech_feodo', 24) if blacklist is not None: return self.parseFeodoTrackerBlacklist(blacklist) res = self.sf.fetchUrl( "https://feodotracker.abuse.ch/downloads/ipblocklist.txt", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from Abuse.ch Abuse.ch Feodo Tracker.") self.errorState = True return None if res['content'] is None: self.error("Received no content from Abuse.ch Feodo Tracker") self.errorState = True return None self.sf.cachePut("abusech_feodo", res['content']) return self.parseFeodoTrackerBlacklist(res['content']) def parseFeodoTrackerBlacklist(self, blacklist): """Parse plaintext blacklist Args: blacklist (str): plaintext blacklist from Abuse.ch Feodo Tracker Returns: list: list of blacklisted IP addresses """ ips = list() if not blacklist: return ips for ip in blacklist.split('\n'): ip = ip.strip() if not ip: continue if ip.startswith('#'): continue if not self.sf.validIP(ip): continue ips.append(ip) return ips def querySslBlacklist(self, target, targetType): blacklist = self.retrieveSslBlacklist() if not blacklist: return False if targetType == "ip": if target in blacklist: self.debug(f"IP address {target} found in Abuse.ch SSL Blacklist.") return True elif targetType == "netblock": netblock = IPNetwork(target) for ip in blacklist: if IPAddress(ip) in netblock: self.debug(f"IP address {ip} found within netblock/subnet {target} in Abuse.ch SSL Blacklist.") return True return False def retrieveSslBlacklist(self): blacklist = self.sf.cacheGet('abusech_ssl', 24) if blacklist is not None: return self.parseSslBlacklist(blacklist) res = self.sf.fetchUrl( "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from Abuse.ch Abuse.ch Feodo Tracker.") self.errorState = True return None if res['content'] is None: self.error("Received no content from Abuse.ch Feodo Tracker") self.errorState = True return None self.sf.cachePut("abusech_ssl", res['content']) return self.parseSslBlacklist(res['content']) def parseSslBlacklist(self, blacklist): """Parse plaintext blacklist Args: blacklist (str): CSV blacklist from Abuse.ch SSL Blacklist Returns: list: list of blacklisted IP addresses """ ips = list() if not blacklist: return ips for line in blacklist.split('\n'): line = line.strip() if not line: continue if line.startswith('#'): continue csv = line.split(',') if len(csv) < 2: continue ip = csv[1] if not self.sf.validIP(ip): continue ips.append(ip) return ips def queryUrlHausBlacklist(self, target, targetType): blacklist = self.retrieveUrlHausBlacklist() if not blacklist: return False if targetType == "ip": if target in blacklist: self.debug(f"IP address {target} found in Abuse.ch URL Haus Blacklist.") return True elif targetType == "netblock": netblock = IPNetwork(target) for ip in blacklist: if IPAddress(ip) in netblock: self.debug(f"IP address {ip} found within netblock/subnet {target} in Abuse.ch URL Haus Blacklist.") return True elif targetType == "domain": if target.lower() in blacklist: self.debug(f"Host name {target} found in Abuse.ch URL Haus Blacklist.") return True return False def retrieveUrlHausBlacklist(self): blacklist = self.sf.cacheGet('abusech_urlhaus', 24) if blacklist is not None: return self.parseUrlHausBlacklist(blacklist) res = self.sf.fetchUrl( "https://urlhaus.abuse.ch/downloads/csv_recent/", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from Abuse.ch URL Haus.") self.errorState = True return None if res['content'] is None: self.error("Received no content from Abuse.ch URL Haus") self.errorState = True return None self.sf.cachePut("abusech_urlhaus", res['content']) return self.parseUrlHausBlacklist(res['content']) def parseUrlHausBlacklist(self, blacklist): """Parse plaintext blacklist Args: blacklist (str): plaintext blacklist from Abuse.ch URL Haus Returns: list: list of blacklisted hosts """ hosts = list() if not blacklist: return hosts for line in blacklist.split('\n'): if not line: continue if line.startswith('#'): continue # Note: URL parsing and validation with sf.validHost() is too slow to use here url = line.strip().lower() if len(url.split("/")) < 3: continue host = url.split("/")[2].split(':')[0] if not host: continue if "." not in host: continue hosts.append(host) return hosts def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName == 'IP_ADDRESS': targetType = 'ip' evtType = 'MALICIOUS_IPADDR' elif eventName == 'AFFILIATE_IPADDR': if not self.opts.get('checkaffiliates', False): return targetType = 'ip' evtType = 'MALICIOUS_AFFILIATE_IPADDR' elif eventName == 'NETBLOCK_OWNER': if not self.opts.get('checknetblocks', False): return targetType = 'netblock' evtType = 'MALICIOUS_NETBLOCK' elif eventName == 'NETBLOCK_MEMBER': if not self.opts.get('checksubnets', False): return targetType = 'netblock' evtType = 'MALICIOUS_SUBNET' elif eventName == "INTERNET_NAME": targetType = 'domain' evtType = "MALICIOUS_INTERNET_NAME" elif eventName == 'AFFILIATE_INTERNET_NAME': if not self.opts.get('checkaffiliates', False): return targetType = 'domain' evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME' elif eventName == 'CO_HOSTED_SITE': if not self.opts.get('checkcohosts', False): return targetType = 'domain' evtType = 'MALICIOUS_COHOST' else: return if targetType in ['ip', 'netblock']: self.debug(f"Checking maliciousness of {eventData} ({eventName}) with Abuse.ch Feodo Tracker") if self.queryFeodoTrackerBlacklist(eventData, targetType): url = "https://feodotracker.abuse.ch/downloads/ipblocklist.txt" text = f"Abuse.ch Feodo Tracker [{eventData}]\n{url}" evt = SpiderFootEvent(evtType, text, self.__name__, event) self.notifyListeners(evt) self.debug(f"Checking maliciousness of {eventData} ({eventName}) with Abuse.ch SSL Blacklist") if self.querySslBlacklist(eventData, targetType): url = "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv" text = f"Abuse.ch SSL Blacklist [{eventData}]\n{url}" evt = SpiderFootEvent(evtType, text, self.__name__, event) self.notifyListeners(evt) if targetType in ['ip', 'domain']: self.debug(f"Checking maliciousness of {eventData} ({eventName}) with Abuse.ch URL Haus") if self.queryUrlHausBlacklist(eventData, targetType): url = "https://urlhaus.abuse.ch/downloads/csv_recent/" text = f"Abuse.ch URL Haus Blacklist [{eventData}]\n{url}" evt = SpiderFootEvent(evtType, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_abusech class ================================================ FILE: modules/sfp_abuseipdb.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_abuseipdb # Purpose: Check if an IP address is malicious according to AbuseIPDB.com. # # Author: steve@binarypool.com # # Created: 06/09/2018 # Copyright: (c) Steve Micallef, 2018 # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_abuseipdb(SpiderFootPlugin): meta = { 'name': "AbuseIPDB", 'summary': "Check if an IP address is malicious according to AbuseIPDB.com blacklist.", 'flags': ["apikey"], 'useCases': ["Passive", "Investigate"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://www.abuseipdb.com", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://docs.abuseipdb.com/#introduction", "https://www.abuseipdb.com/fail2ban.html", "https://www.abuseipdb.com/csf", "https://www.abuseipdb.com/suricata", "https://www.abuseipdb.com/splunk", "https://www.abuseipdb.com/categories" ], 'apiKeyInstructions': [ "Visit https://www.abuseipdb.com/pricing", "Select the plan required", "Register a new account with an email", "Navigate to https://www.abuseipdb.com/account/api", "The API Key is listed under 'Keys'" ], 'favIcon': "https://www.abuseipdb.com/favicon.ico", 'logo': "https://www.abuseipdb.com/img/abuseipdb.png.pagespeed.ce.CI8T6WsXU7.png", 'description': "AbuseIPDB is a project dedicated to helping combat the spread of hackers," "spammers, and abusive activity on the internet.\n" "Our mission is to help make Web safer by providing a central blacklist for" "webmasters, system administrators, and other interested parties to" "report and find IP addresses that have been associated with malicious activity online." } } opts = { 'api_key': '', 'confidenceminimum': 90, 'checkaffiliates': True, 'limit': 10000 } optdescs = { 'api_key': "AbuseIPDB.com API key.", 'confidenceminimum': "The minimium AbuseIPDB confidence level to require.", 'checkaffiliates': "Apply checks to affiliates?", 'limit': 'Maximum number of results to retrieve.', } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "IPV6_ADDRESS", "AFFILIATE_IPADDR", "AFFILIATE_IPV6_ADDRESS", ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", ] def queryBlacklist(self): blacklist = self.sf.cacheGet('abuseipdb', 24) if blacklist is not None: return self.parseBlacklist(blacklist) headers = { 'Key': self.opts['api_key'], 'Accept': "text/plain" } params = urllib.parse.urlencode({ 'confidenceMinimum': self.opts['confidenceminimum'], 'limit': self.opts['limit'], 'plaintext': '1' }) res = self.sf.fetchUrl( f"https://api.abuseipdb.com/api/v2/blacklist?{params}", timeout=60, # retrieving 10,000 results (default) or more can sometimes take a while useragent=self.opts['_useragent'], headers=headers ) time.sleep(1) if res['code'] == '429': self.error("You are being rate-limited by AbuseIPDB") self.errorState = True return None if res['code'] != "200": self.error(f"Error retrieving search results, code {res['code']}") self.errorState = True return None if res['code'] != "200": self.error("Error retrieving search results from AbuseIPDB") self.errorState = True return None if res['content'] is None: self.error("Received no content from AbuseIPDB") self.errorState = True return None self.sf.cachePut("abuseipdb", res['content']) return self.parseBlacklist(res['content']) def parseBlacklist(self, blacklist): """Parse plaintext blacklist Args: blacklist (str): plaintext blacklist from AbuseIPDB Returns: list: list of blacklisted IP addresses """ ips = list() if not blacklist: return ips for ip in blacklist.split('\n'): ip = ip.strip() if ip.startswith('#'): continue if not self.sf.validIP(ip) and not self.sf.validIP6(ip): continue ips.append(ip) return ips def queryIpAddress(self, ip): """Query API for an IPv4 or IPv6 address. Note: Currently unused. Args: ip (str): IP address Returns: str: API response as JSON """ headers = { 'Key': self.opts['api_key'], 'Accept': 'application/json', } params = urllib.parse.urlencode({ 'ipAddress': ip, 'maxAgeInDays': 30, }) res = self.sf.fetchUrl( f"https://api.abuseipdb.com/api/v2/check?{params}", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], headers=headers ) time.sleep(1) if res['code'] == '429': self.error("You are being rate-limited by AbuseIPDB") self.errorState = True return None if res['code'] != "200": self.error("Error retrieving search results from AbuseIPDB") self.errorState = True return None if res['content'] is None: self.error("Received no content from AbuseIPDB") self.errorState = True return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None return None def queryNetblock(self, ip): """Query API for a netblock. Note: Currently unused. Args: ip (str): CIDR range Returns: str: API response as JSON """ headers = { 'Key': self.opts['api_key'], 'Accept': 'application/json', } params = urllib.parse.urlencode({ 'ipAddress': ip, 'maxAgeInDays': 30, }) res = self.sf.fetchUrl( f"https://api.abuseipdb.com/api/v2/check-block?{params}", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], headers=headers ) time.sleep(1) if res['code'] == '429': self.error("You are being rate-limited by AbuseIPDB") self.errorState = True return None if res['code'] != "200": self.error("Error retrieving search results from AbuseIPDB") self.errorState = True return None if res['content'] is None: self.error("Received no content from AbuseIPDB") self.errorState = True return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.error( f"You enabled {self.__class__.__name__} but did not set an API key!" ) self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName.startswith("AFFILIATE") and not self.opts['checkaffiliates']: return if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']: blacklist_type = "BLACKLISTED_IPADDR" malicious_type = 'MALICIOUS_IPADDR' elif eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS']: blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" malicious_type = 'MALICIOUS_AFFILIATE_IPADDR' else: self.debug(f"Unexpected event type {eventName}, skipping") return self.debug(f"Checking maliciousness of IP address {eventData} with AbuseIPDB") blacklist = self.queryBlacklist() if not blacklist: return if eventData not in blacklist: return self.info(f"Malicious IP address {eventData} found in AbuseIPDB blacklist") url = f"https://www.abuseipdb.com/check/{eventData}" evt = SpiderFootEvent( malicious_type, f"AbuseIPDB [{eventData}]\n{url}", self.__name__, event ) self.notifyListeners(evt) evt = SpiderFootEvent( blacklist_type, f"AbuseIPDB [{eventData}]\n{url}", self.__name__, event ) self.notifyListeners(evt) # End of sfp_abuseipdb class ================================================ FILE: modules/sfp_abusix.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_abusix # Purpose: SpiderFoot plug-in for looking up whether IPs/Netblocks/Domains # appear in the Abusix Mail Intelligence blacklist. # # Author: # # Created: 2021-10-17 # Copyright: (c) bcoles 2021 # Licence: MIT # ------------------------------------------------------------------------------- import ipaddress from netaddr import IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_abusix(SpiderFootPlugin): meta = { 'name': "Abusix Mail Intelligence", 'summary': "Check if a netblock or IP address is in the Abusix Mail Intelligence blacklist.", 'flags': ['apikey'], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://abusix.org/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://abusix.com/products/abusix-mail-intelligence/", "https://docs.abusix.com/105726-setup-abusix-mail-intelligence/ami%2Fsetup%2Fexample-queries", "https://docs.abusix.com/105725-detailed-list-information/ami%2Freturn-codes", ], 'apiKeyInstructions': [ "Visit https://app.abusix.com/signup", "Register a free account", "Browse to 'Account Settings' page", "The API key is listed on the 'Email protection' page." ], 'logo': "https://abusix.com/wp-content/uploads/2020/10/Footer_logo.png", 'description': "Abusix Mail Intelligence is an innovative set of blocklists (RBL/DNSBL) " "that adds real-time threat data to your existing email protection. " "Considered as the first line of defense, blocklists help to prevent email-borne threats " "such as spam and malware from entering your network." } } opts = { 'api_key': "", 'checkaffiliates': True, 'checkcohosts': True, 'netblocklookup': True, 'maxnetblock': 24, 'maxv6netblock': 120, 'subnetlookup': True, 'maxsubnet': 24, 'maxv6subnet': 120, } optdescs = { 'api_key': "Abusix Mail Intelligence API key.", 'checkaffiliates': "Apply checks to affiliates?", 'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?", 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?", 'maxnetblock': "If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'maxv6netblock': "If looking up owned netblocks, the maximum IPv6 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'subnetlookup': "Look up all IPs on subnets which your target is a part of for blacklisting?", 'maxsubnet': "If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'maxv6subnet': "If looking up subnets, the maximum IPv6 subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", } results = None errorState = False checks = { "127.0.0.2": "black", "127.0.0.3": "black (composite/heuristic)", "127.0.0.4": "exploit / authbl", "127.0.0.5": "forged", "127.0.0.6": "backscatter", "127.0.0.11": "policy (generic rDNS)", "127.0.0.12": "policy (missing rDNS)", "127.0.0.100": "noip", "127.0.1.1": "dblack", "127.0.1.2": "dblack (Newly Observed Domain)", "127.0.1.3": "dblack (Unshortened)", "127.0.2.1": "white", "127.0.3.1": "shorthash", "127.0.3.2": "diskhash", "127.0.4.1": "btc-wallets", "127.0.5.1": "attachhash", } def setup(self, sfc, userOpts=dict()): self.sf = sfc self.errorState = False self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ 'IP_ADDRESS', 'IPV6_ADDRESS', 'AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS', "NETBLOCK_MEMBER", "NETBLOCKV6_MEMBER", "NETBLOCK_OWNER", "NETBLOCKV6_OWNER", 'INTERNET_NAME', 'AFFILIATE_INTERNET_NAME', 'CO_HOSTED_SITE', ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "BLACKLISTED_SUBNET", "BLACKLISTED_NETBLOCK", "BLACKLISTED_INTERNET_NAME", "BLACKLISTED_AFFILIATE_INTERNET_NAME", "BLACKLISTED_COHOST", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK", "MALICIOUS_SUBNET", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_COHOST", ] def reverseIpAddress(self, ipaddr): if not self.sf.validIP(ipaddr): self.debug(f"Invalid IPv4 address {ipaddr}") return None return ipaddress.ip_address(ipaddr).reverse_pointer.replace('.in-addr.arpa', '') def reverseIp6Address(self, ipaddr): if not self.sf.validIP6(ipaddr): self.debug(f"Invalid IPv6 address {ipaddr}") return None return ipaddress.ip_address(ipaddr).reverse_pointer.replace('.ip6.arpa', '') def query(self, qaddr): """Query Abusix Mail Intelligence DNS. Args: qaddr (str): Host name or IPv4 address. Returns: list: Abusix DNS entries """ if self.sf.validIP(qaddr): lookup = f"{self.reverseIpAddress(qaddr)}.{self.opts['api_key']}.combined.mail.abusix.zone" elif self.sf.validIP6(qaddr): lookup = f"{self.reverseIp6Address(qaddr)}.{self.opts['api_key']}.combined.mail.abusix.zone" else: lookup = f"{qaddr}.{self.opts['api_key']}.combined.mail.abusix.zone" self.debug(f"Checking Abusix Mail Intelligence blacklist: {lookup}") try: return self.sf.resolveHost(lookup) except Exception as e: self.debug(f"Abusix Mail Intelligence did not resolve {qaddr} / {lookup}: {e}") return None def handleEvent(self, event): eventName = event.eventType eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {event.module}") if not self.opts['api_key']: self.error(f"You enabled {self.__class__.__name__} but did not set an API key!") self.errorState = True return if eventData in self.results: return self.results[eventData] = True if eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS']: if not self.opts.get('checkaffiliates', False): return malicious_type = "MALICIOUS_AFFILIATE_IPADDR" blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" elif eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']: malicious_type = "MALICIOUS_IPADDR" blacklist_type = "BLACKLISTED_IPADDR" elif eventName in ['NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']: if not self.opts['subnetlookup']: return if eventName == 'NETBLOCKV6_MEMBER': max_subnet = self.opts['maxv6subnet'] else: max_subnet = self.opts['maxsubnet'] if IPNetwork(eventData).prefixlen < max_subnet: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}") return malicious_type = "MALICIOUS_SUBNET" blacklist_type = "BLACKLISTED_SUBNET" elif eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']: if not self.opts['netblocklookup']: return if eventName == 'NETBLOCKV6_OWNER': max_netblock = self.opts['maxv6netblock'] else: max_netblock = self.opts['maxnetblock'] if IPNetwork(eventData).prefixlen < max_netblock: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}") return malicious_type = "MALICIOUS_NETBLOCK" blacklist_type = "BLACKLISTED_NETBLOCK" elif eventName == "INTERNET_NAME": malicious_type = "MALICIOUS_INTERNET_NAME" blacklist_type = "BLACKLISTED_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": if not self.opts.get('checkaffiliates', False): return malicious_type = "MALICIOUS_AFFILIATE_INTERNET_NAME" blacklist_type = "BLACKLISTED_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": if not self.opts.get('checkcohosts', False): return malicious_type = "MALICIOUS_COHOST" blacklist_type = "BLACKLISTED_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") return addrs = list() if eventName.startswith("NETBLOCK"): for addr in IPNetwork(eventData): addrs.append(str(addr)) else: addrs.append(eventData) for addr in addrs: if self.checkForStop(): return if self.errorState: return res = self.query(addr) self.results[addr] = True if not res: continue self.debug(f"{addr} found in Abusix Mail Intelligence DNS") for result in res: k = str(result) if k not in self.checks: if 'mail.abusix.zone' not in result: # This is an error. The "checks" dict may need to be updated. self.error(f"Abusix Mail Intelligence resolved address {addr} to unknown IP address {result} not found in Abusix Mail Intelligence list.") continue text = f"Abusix Mail Intelligence - {self.checks[k]} [{addr}]\nhttps://lookup.abusix.com/search?q={addr}" evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_abusix class ================================================ FILE: modules/sfp_accounts.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_accounts # Purpose: Identify the existence of a given acount on various sites thanks # to Micah Hoffman's (https://github.com/WebBreacher) list. # # Author: Steve Micallef # # Created: 18/02/2015 # Copyright: (c) Steve Micallef 2015 # Licence: MIT # ------------------------------------------------------------------------------- import json import random import threading import time from queue import Empty as QueueEmpty from queue import Queue from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_accounts(SpiderFootPlugin): meta = { 'name': "Account Finder", 'summary': "Look for possible associated accounts on over 500 social and other websites such as Instagram, Reddit, etc.", 'useCases': ["Footprint", "Passive"], 'categories': ["Social Media"] } # Default options opts = { "ignorenamedict": True, "ignoreworddict": True, "musthavename": True, "userfromemail": True, "permutate": False, "usernamesize": 4, "_maxthreads": 20 } # Option descriptions optdescs = { "ignorenamedict": "Don't bother looking up names that are just stand-alone first names (too many false positives).", "ignoreworddict": "Don't bother looking up names that appear in the dictionary.", "musthavename": "The username must be mentioned on the social media page to consider it valid (helps avoid false positives).", "userfromemail": "Extract usernames from e-mail addresses at all? If disabled this can reduce false positives for common usernames but for highly unique usernames it would result in missed accounts.", "permutate": "Look for the existence of account name permutations. Useful to identify fraudulent social media accounts or account squatting.", "usernamesize": "The minimum length of a username to query across social media sites. Helps avoid false positives for very common short usernames.", "_maxthreads": "Maximum threads" } results = None reportedUsers = list() siteResults = dict() sites = list() errorState = False distrustedChecked = False lock = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.commonNames = list() self.reportedUsers = list() self.errorState = False self.distrustedChecked = False self.__dataSource__ = "Social Media" self.lock = threading.Lock() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] self.commonNames = SpiderFootHelpers.humanNamesFromWordlists() self.words = SpiderFootHelpers.dictionaryWordsFromWordlists() content = self.sf.cacheGet("sfaccountsv2", 48) if content is None: url = "https://raw.githubusercontent.com/WebBreacher/WhatsMyName/main/wmn-data.json" data = self.sf.fetchUrl(url, useragent="SpiderFoot") if data['content'] is None: self.error(f"Unable to fetch {url}") self.errorState = True return content = data['content'] self.sf.cachePut("sfaccountsv2", content) try: self.sites = [site for site in json.loads(content)['sites'] if not site.get('valid', True) is False] except Exception as e: self.error(f"Unable to parse social media accounts list: {e}") self.errorState = True return def watchedEvents(self): return ["EMAILADDR", "DOMAIN_NAME", "HUMAN_NAME", "USERNAME"] def producedEvents(self): return ["USERNAME", "ACCOUNT_EXTERNAL_OWNED", "SIMILAR_ACCOUNT_EXTERNAL"] def checkSite(self, name, site): if 'uri_check' not in site: return url = site['uri_check'].format(account=name) if 'uri_pretty' in site: ret_url = site['uri_pretty'].format(account=name) else: ret_url = url retname = f"{site['name']} (Category: {site['cat']})\n{ret_url}" post = None if site.get('post_body'): post = site['post_body'] res = self.sf.fetchUrl( url, postData=post, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], noLog=True, verify=False ) if not res['content']: with self.lock: self.siteResults[retname] = False return if site.get('e_code') != site.get('m_code'): if res['code'] != str(site.get('e_code')): with self.lock: self.siteResults[retname] = False return if site.get('e_string') not in res['content'] or (site.get('m_string') and site.get('m_string') in res['content']): with self.lock: self.siteResults[retname] = False return if self.opts['musthavename']: if name.lower() not in res['content'].lower(): self.debug(f"Skipping {site['name']} as username not mentioned.") with self.lock: self.siteResults[retname] = False return # Some sites can't handle periods so treat bob.abc and bob as the same # TODO: fix this once WhatsMyName has support for usernames with '.' if "." in name: firstname = name.split(".")[0] if firstname + "<" in res['content'] or firstname + '"' in res['content']: with self.lock: self.siteResults[retname] = False return with self.lock: self.siteResults[retname] = True def checkSites(self, username, sites=None): def processSiteQueue(username, queue): try: while True: site = queue.get(timeout=0.1) try: self.checkSite(username, site) except Exception as e: self.debug(f'Thread {threading.current_thread().name} exception: {e}') except QueueEmpty: return startTime = time.monotonic() # results will be collected in siteResults self.siteResults = {} sites = self.sites if sites is None else sites # load the queue queue = Queue() for site in sites: queue.put(site) # start the scan threads threads = [] for i in range(min(len(sites), self.opts['_maxthreads'])): thread = threading.Thread( name=f'sfp_accounts_scan_{i}', target=processSiteQueue, args=(username, queue)) thread.start() threads.append(thread) # wait for all scan threads to finish while threads: threads.pop(0).join() duration = time.monotonic() - startTime scanRate = len(sites) / duration self.debug(f'Scan statistics: name={username}, count={len(self.siteResults)}, duration={duration:.2f}, rate={scanRate:.0f}') return [site for site, found in self.siteResults.items() if found] def generatePermutations(self, username): permutations = list() prefixsuffix = ['_', '-'] replacements = { 'a': ['4', 's'], 'b': ['v', 'n'], 'c': ['x', 'v'], 'd': ['s', 'f'], 'e': ['w', 'r'], 'f': ['d', 'g'], 'g': ['f', 'h'], 'h': ['g', 'j', 'n'], 'i': ['o', 'u', '1'], 'j': ['k', 'h', 'i'], 'k': ['l', 'j'], 'l': ['i', '1', 'k'], 'm': ['n'], 'n': ['m'], 'o': ['p', 'i', '0'], 'p': ['o', 'q'], 'r': ['t', 'e'], 's': ['a', 'd', '5'], 't': ['7', 'y', 'z', 'r'], 'u': ['v', 'i', 'y', 'z'], 'v': ['u', 'c', 'b'], 'w': ['v', 'vv', 'q', 'e'], 'x': ['z', 'y', 'c'], 'y': ['z', 'x'], 'z': ['y', 'x'], '0': ['o'], '1': ['l'], '2': ['5'], '3': ['e'], '4': ['a'], '5': ['s'], '6': ['b'], '7': ['t'], '8': ['b'], '9': [] } pairs = { 'oo': ['00'], 'll': ['l1l', 'l1l', '111', '11'], '11': ['ll', 'lll', 'l1l', '1l1'] } # Generate a set with replacements, then # add suffixes and prefixes. pos = 0 for c in username: if c not in replacements: continue if len(replacements[c]) == 0: continue npos = pos + 1 for xc in replacements[c]: newuser = username[0:pos] + xc + username[npos:len(username)] permutations.append(newuser) pos += 1 # Search for common double-letter replacements for p in pairs: if p in username: for r in pairs[p]: permutations.append(username.replace(p, r)) # Search for prefixed and suffixed usernames for c in prefixsuffix: permutations.append(username + c) permutations.append(c + username) # Search for double character usernames pos = 0 for c in username: permutations.append(username[0:pos] + c + c + username[(pos + 1):len(username)]) pos += 1 return list(set(permutations)) def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data users = list() if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") # Skip events coming from me unless they are USERNAME events if eventName != "USERNAME" and srcModuleName == "sfp_accounts": self.debug(f"Ignoring {eventName}, from self.") return if eventData in list(self.results.keys()): return self.results[eventData] = True # If being called for the first time, let's see how trusted the # sites are by attempting to fetch a garbage user. if not self.distrustedChecked: # Check if a state cache exists first, to not have to do this all the time content = self.sf.cacheGet("sfaccounts_state_v3", 72) if content: if content != "None": # "None" is written to the cached file when no sites are distrusted delsites = list() for line in content.split("\n"): if line == '': continue delsites.append(line) self.sites = [d for d in self.sites if d['name'] not in delsites] else: randpool = 'abcdefghijklmnopqrstuvwxyz1234567890' randuser = ''.join([random.SystemRandom().choice(randpool) for x in range(10)]) res = self.checkSites(randuser) if res: delsites = list() for site in res: sitename = site.split(" (Category:")[0] self.debug(f"Distrusting {sitename}") delsites.append(sitename) self.sites = [d for d in self.sites if d['name'] not in delsites] else: # The caching code needs *some* content delsites = "None" self.sf.cachePut("sfaccounts_state_v3", delsites) self.distrustedChecked = True if eventName == "HUMAN_NAME": names = [eventData.lower().replace(" ", ""), eventData.lower().replace(" ", ".")] for name in names: users.append(name) if eventName == "DOMAIN_NAME": kw = self.sf.domainKeyword(eventData, self.opts['_internettlds']) if not kw: return users.append(kw) if eventName == "EMAILADDR" and self.opts['userfromemail']: name = eventData.split("@")[0].lower() users.append(name) if eventName == "USERNAME": users.append(eventData) for user in set(users): if user in self.opts['_genericusers'].split(","): self.debug(f"{user} is a generic account name, skipping.") continue if self.opts['ignorenamedict'] and user in self.commonNames: self.debug(f"{user} is found in our name dictionary, skipping.") continue if self.opts['ignoreworddict'] and user in self.words: self.debug(f"{user} is found in our word dictionary, skipping.") continue if user not in self.reportedUsers and eventData != user: if len(user) < self.opts['usernamesize']: self.debug(f"{user} is too short, skipping.") continue evt = SpiderFootEvent("USERNAME", user, self.__name__, event) self.notifyListeners(evt) self.reportedUsers.append(user) # Only look up accounts when we've received a USERNAME event (possibly from # ourselves), since we want them to have gone through some verification by # this module, and we don't want duplicates (one based on EMAILADDR and another # based on USERNAME). if eventName == "USERNAME": res = self.checkSites(user) for site in res: evt = SpiderFootEvent( "ACCOUNT_EXTERNAL_OWNED", site, self.__name__, event ) self.notifyListeners(evt) if self.opts['permutate']: permutations = self.generatePermutations(user) for puser in permutations: res = self.checkSites(puser) for site in res: evt = SpiderFootEvent( "SIMILAR_ACCOUNT_EXTERNAL", site, self.__name__, event ) self.notifyListeners(evt) # End of sfp_accounts class ================================================ FILE: modules/sfp_adblock.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_adblock # Purpose: SpiderFoot plug-in to test if external/internally linked pages # would be blocked by AdBlock Plus. # # Author: Steve Micallef # # Created: 22/09/2014 # Copyright: (c) Steve Micallef 2014 # Licence: MIT # ------------------------------------------------------------------------------- import adblockparser from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_adblock(SpiderFootPlugin): meta = { 'name': "AdBlock Check", 'summary': "Check if linked pages would be blocked by AdBlock Plus.", 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://adblockplus.org/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://help.eyeo.com/en/adblockplus/", "https://adblockplus.org/en/download", "https://adblockplus.org/en/filters#options", "https://chrome.google.com/webstore/detail/adblock-plus-free-ad-bloc/cfhdojbkjhnklbpkdaibdccddilifddb" ], 'favIcon': "https://www.google.com/s2/favicons?domain=https://adblockplus.org/en/", 'logo': "https://adblockplus.org/img/navbar-logo.svg", 'description': "Adblock Plus is a free extension that allows you to customize your web experience." "You can block annoying ads, disable tracking and lots more." "It’s available for all major desktop browsers and for your mobile devices.\n" "Block ads that interrupt your browsing experience." "Say goodbye to video ads, pop-ups, flashing banners and more." "Blocking these annoyances means pages load faster.\n" "With Adblock Plus avoiding tracking and malware is easy." "Blocking intrusive ads reduces the risk of \"malvertising\" infections." "Blocking tracking stops companies following your online activity." } } # Default options opts = { "blocklist": "https://easylist-downloads.adblockplus.org/easylist.txt", 'cacheperiod': 24, } optdescs = { "blocklist": "AdBlockPlus block list.", 'cacheperiod': "Hours to cache list data before re-fetching.", } results = None rules = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.rules = None self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["LINKED_URL_INTERNAL", "LINKED_URL_EXTERNAL", "PROVIDER_JAVASCRIPT"] # What events this module produces def producedEvents(self): return ["URL_ADBLOCKED_INTERNAL", "URL_ADBLOCKED_EXTERNAL"] def retrieveBlocklist(self, blocklist_url): if not blocklist_url: return None blocklist = self.sf.cacheGet(f"adblock_{blocklist_url}", 24) if blocklist is not None: return self.setBlocklistRules(blocklist) res = self.sf.fetchUrl(blocklist_url, timeout=30) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} for {blocklist_url}") self.errorState = True return None if res['content'] is None: self.error(f"Unable to download AdBlock Plus blocklist: {blocklist_url}") self.errorState = True return None self.sf.cachePut(f"adblock_{blocklist_url}", res['content']) return self.setBlocklistRules(res['content']) def setBlocklistRules(self, blocklist): """Parse AdBlock Plus blocklist and set blocklist rules Args: blocklist (str): plaintext AdBlock Plus blocklist """ if not blocklist: return lines = blocklist.split('\n') self.debug(f"Retrieved {len(lines)} AdBlock blocklist rules") try: self.rules = adblockparser.AdblockRules(lines) except adblockparser.AdblockParsingError as e: self.errorState = True self.error(f"Parsing error handling AdBlock list: {e}") # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug("Already checked this URL for AdBlock matching, skipping.") return self.results[eventData] = True if self.errorState: return if not self.opts["blocklist"]: self.error( f"You enabled {self.__class__.__name__} but did not set a blocklist URL!" ) self.errorState = True return if not self.rules: self.retrieveBlocklist(self.opts['blocklist']) if not self.rules: self.error("No AdBlock Plus rules loaded") self.errorState = True return try: if eventName == 'PROVIDER_JAVASCRIPT': if self.rules and self.rules.should_block(eventData, {'third-party': True, 'script': True}): evt = SpiderFootEvent("URL_ADBLOCKED_EXTERNAL", eventData, self.__name__, event) self.notifyListeners(evt) if eventName == 'LINKED_URL_EXTERNAL': if self.rules and self.rules.should_block(eventData, {'third-party': True}): evt = SpiderFootEvent("URL_ADBLOCKED_EXTERNAL", eventData, self.__name__, event) self.notifyListeners(evt) if eventName == 'LINKED_URL_INTERNAL': if self.rules and self.rules.should_block(eventData): evt = SpiderFootEvent("URL_ADBLOCKED_INTERNAL", eventData, self.__name__, event) self.notifyListeners(evt) except ValueError as e: self.error(f"Parsing error handling AdBlock list: {e}") self.errorState = True # End of sfp_adblock class ================================================ FILE: modules/sfp_adguard_dns.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_adguard_dns # Purpose: SpiderFoot plug-in for looking up whether hosts are blocked by # AdGuard DNS servers. # # Author: # # Created: 2021-10-11 # Copyright: (c) bcoles 2021 # Licence: MIT # ------------------------------------------------------------------------------- import dns.resolver from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_adguard_dns(SpiderFootPlugin): meta = { 'name': "AdGuard DNS", 'summary': "Check if a host would be blocked by AdGuard DNS.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://adguard.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://adguard.com/en/adguard-dns/overview.html", ], 'favIcon': "https://adguard.com/img/favicons/favicon.ico", 'logo': "https://adguard.com/img/favicons/apple-touch-icon.png", 'description': "AdGuard DNS is a foolproof way to block Internet ads that does not require installing any applications. " "It is easy to use, absolutely free, easily set up on any device, and provides you with minimal necessary functions " "to block ads, counters, malicious websites, and adult content." } } opts = { } optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "CO_HOSTED_SITE" ] def producedEvents(self): return [ "BLACKLISTED_INTERNET_NAME", "BLACKLISTED_AFFILIATE_INTERNET_NAME", "BLACKLISTED_COHOST", ] def queryDefaultDNS(self, qaddr): res = dns.resolver.Resolver() res.nameservers = ["94.140.14.14", "94.140.15.15"] try: return res.resolve(qaddr) except Exception: self.debug(f"Unable to resolve {qaddr}") return None def queryFamilyDNS(self, qaddr): res = dns.resolver.Resolver() res.nameservers = ["94.140.14.15", "94.140.15.16"] try: return res.resolve(qaddr) except Exception: self.debug(f"Unable to resolve {qaddr}") return None def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: return self.results[eventData] = True if eventName == "INTERNET_NAME": blacklist_type = "BLACKLISTED_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": blacklist_type = "BLACKLISTED_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": blacklist_type = "BLACKLISTED_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") return family = self.sf.normalizeDNS(self.queryFamilyDNS(eventData)) default = self.sf.normalizeDNS(self.queryDefaultDNS(eventData)) if not family or not default: return if '94.140.14.35' in family: self.debug(f"{eventData} blocked by AdGuard Family DNS") evt = SpiderFootEvent(blacklist_type, f"AdGuard - Family Filter [{eventData}]", self.__name__, event) self.notifyListeners(evt) if '94.140.14.35' in default: self.debug(f"{eventData} blocked by AdGuard Default DNS") evt = SpiderFootEvent(blacklist_type, f"AdGuard - Default Filter [{eventData}]", self.__name__, event) self.notifyListeners(evt) # End of sfp_adguard_dns class ================================================ FILE: modules/sfp_ahmia.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_ahmia # Purpose: Searches the Tor search engine 'Ahmia' for content related to the # target. # # Author: Steve Micallef # # Created: 20/06/2017 # Copyright: (c) Steve Micallef 2017 # Licence: MIT # ------------------------------------------------------------------------------- import re import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_ahmia(SpiderFootPlugin): meta = { 'name': "Ahmia", 'flags': ["tor"], 'summary': "Search Tor 'Ahmia' search engine for mentions of the target.", 'useCases': ["Footprint", "Investigate"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://ahmia.fi/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://ahmia.fi/documentation/", "https://github.com/ahmia/", "http://msydqstlz2kzerdg.onion/", "https://ahmia.fi/stats" ], 'favIcon': "https://ahmia.fi/static/images/favicon.ico", 'logo': "https://ahmia.fi/static/images/ahmiafi_black.png", 'description': "Ahmia searches hidden services on the Tor network. To access these hidden services," "you need the Tor browser bundle. Abuse material is not allowed on Ahmia. " "See our service blacklist and report abuse material if you find it in the index. " "It will be removed as soon as possible.\n" "Contributors to Ahmia believe that the Tor network is an important and " "resilient distributed platform for anonymity and privacy worldwide. " "By providing a search engine for what many call the \"deep web\" or \"dark net\", " "Ahmia makes hidden services accessible to a wide range of people, not just Tor network early adopters." } } # Default options opts = { 'fetchlinks': True, 'fullnames': True } # Option descriptions optdescs = { 'fetchlinks': "Fetch the darknet pages (via TOR, if enabled) to verify they mention your target.", 'fullnames': "Search for human names?" } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME", "HUMAN_NAME", "EMAILADDR"] # What events this module produces def producedEvents(self): return ["DARKNET_MENTION_URL", "DARKNET_MENTION_CONTENT"] def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if not self.opts['fullnames'] and eventName == 'HUMAN_NAME': self.debug(f"Skipping HUMAN_NAME: {eventData}") return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True params = urllib.parse.urlencode({ 'q': eventData }) data = self.sf.fetchUrl( f"https://ahmia.fi/search/?{params}", useragent=self.opts['_useragent'], timeout=15 ) if not data: self.info(f"No results for {eventData} returned from Ahmia.fi.") return content = data.get('content') if not content: self.info(f"No results for {eventData} returned from Ahmia.fi.") return # We don't bother with pagination as Ahmia seems fairly limited in coverage # and displays hundreds of results per page links = re.findall("redirect_url=(.[^\"]+)\"", content, re.IGNORECASE | re.DOTALL) if not links: self.info(f"No results for {eventData} returned from Ahmia.fi.") return for link in links: if self.checkForStop(): return if link in self.results: continue self.results[link] = True self.debug(f"Found a darknet mention: {link}") if not self.sf.urlFQDN(link).endswith(".onion"): continue if not self.opts['fetchlinks']: evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event) self.notifyListeners(evt) continue res = self.sf.fetchUrl( link, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], verify=False ) if res['content'] is None: self.debug(f"Ignoring {link} as no data returned") continue if eventData not in res['content']: self.debug(f"Ignoring {link} as no mention of {eventData}") continue evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event) self.notifyListeners(evt) try: startIndex = res['content'].index(eventData) - 120 endIndex = startIndex + len(eventData) + 240 except Exception: self.debug(f"String '{eventData}' not found in content.") continue wdata = res['content'][startIndex:endIndex] evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", f"...{wdata}...", self.__name__, evt) self.notifyListeners(evt) # End of sfp_ahmia class ================================================ FILE: modules/sfp_alienvault.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_alienvault # Purpose: Query AlienVault OTX # # Author: Steve Micallef # # Created: 26/03/2017 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib.error import urllib.parse import urllib.request from datetime import datetime from netaddr import IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_alienvault(SpiderFootPlugin): meta = { 'name': "AlienVault OTX", 'summary': "Obtain information from AlienVault Open Threat Exchange (OTX)", 'flags': ["apikey"], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://otx.alienvault.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://otx.alienvault.com/faq", "https://otx.alienvault.com/api", "https://otx.alienvault.com/submissions/list", "https://otx.alienvault.com/pulse/create", "https://otx.alienvault.com/endpoint-security/welcome", "https://otx.alienvault.com/browse/" ], 'apiKeyInstructions': [ "Visit https://otx.alienvault.com/", "Sign up for a free account", "Navigate to https://otx.alienvault.com/settings", "The API key is listed under 'OTX Key'" ], 'favIcon': "https://www.google.com/s2/favicons?domain=https://otx.alienvault.com/", 'logo': "https://otx.alienvault.com/assets/images/otx-logo.svg", 'description': "The World’s First Truly Open Threat Intelligence Community\n" "Open Threat Exchange is the neighborhood watch of the global intelligence community. " "It enables private companies, independent security researchers, and government agencies to " "openly collaborate and share the latest information about emerging threats, attack methods, " "and malicious actors, promoting greater security across the entire community.\n" "OTX changed the way the intelligence community creates and consumes threat data. " "In OTX, anyone in the security community can contribute, discuss, research, validate, " "and share threat data. You can integrate community-generated OTX threat data directly " "into your AlienVault and third-party security products, so that your threat detection defenses " "are always up to date with the latest threat intelligence. " "Today, 100,000 participants in 140 countries contribute over 19 million threat indicators daily." } } # Default options opts = { "api_key": "", "verify": True, "reputation_age_limit_days": 30, "cohost_age_limit_days": 30, "threat_score_min": 2, 'netblocklookup': True, 'maxnetblock': 24, 'maxv6netblock': 120, 'subnetlookup': True, 'maxsubnet': 24, 'maxv6subnet': 120, 'max_pages': 50, 'maxcohost': 100, 'checkaffiliates': True } # Option descriptions optdescs = { "api_key": "AlienVault OTX API Key.", "verify": "Verify co-hosts are valid by checking if they still resolve to the shared IP.", "reputation_age_limit_days": "Ignore any reputation records older than this many days. 0 = unlimited.", "cohost_age_limit_days": "Ignore any co-hosts older than this many days. 0 = unlimited.", "threat_score_min": "Minimum AlienVault threat score.", 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?", 'maxnetblock': "If looking up owned netblocks, the maximum IPv4 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'maxv6netblock': "If looking up owned netblocks, the maximum IPv6 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'subnetlookup': "Look up all IPs on subnets which your target is a part of for blacklisting?", 'maxsubnet': "If looking up subnets, the maximum IPv4 subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'maxv6subnet': "If looking up subnets, the maximum IPv6 subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'max_pages': "Maximum number of pages of URL results to fetch.", 'maxcohost': "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.", 'checkaffiliates': "Apply checks to affiliates?" } results = None errorState = False cohostcount = 0 def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.cohostcount = 0 self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return [ "INTERNET_NAME", "IP_ADDRESS", "IPV6_ADDRESS", "AFFILIATE_IPADDR", "AFFILIATE_IPV6_ADDRESS", "NETBLOCK_OWNER", "NETBLOCKV6_OWNER", "NETBLOCK_MEMBER", "NETBLOCKV6_MEMBER", "NETBLOCK_OWNER", "NETBLOCK_MEMBER", ] # What events this module produces def producedEvents(self): return [ "IP_ADDRESS", "IPV6_ADDRESS", "AFFILIATE_IPADDR", "AFFILIATE_IPV6_ADDRESS", "CO_HOSTED_SITE", "INTERNET_NAME", "INTERNET_NAME_UNRESOLVED", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK", "LINKED_URL_INTERNAL" ] # Parse API response def parseApiResponse(self, res: dict): if not res: self.error("No response from AlienVault OTX.") return None # Future proofing - AlienVault OTX does not implement rate limiting if res['code'] == '429': self.error("You are being rate-limited by AienVault OTX") self.errorState = True return None if res['code'] == "403": self.error("AlienVault OTX API key seems to have been rejected or you have exceeded usage limits for the month.") self.errorState = True return None if res['content'] is None or res['code'] == "404": return None try: return json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from AlienVault OTX: {e}") return None def queryReputation(self, qry): if ":" in qry: target_type = "IPv6" elif self.sf.validIP(qry): target_type = "IPv4" else: self.info(f"Could not determine target type for {qry}") return None headers = { 'Accept': 'application/json', 'X-OTX-API-KEY': self.opts['api_key'] } res = self.sf.fetchUrl( f"https://otx.alienvault.com/api/v1/indicators/{target_type}/{qry}/reputation", timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=headers) return self.parseApiResponse(res) def queryPassiveDns(self, qry): if ":" in qry: target_type = "IPv6" elif self.sf.validIP(qry): target_type = "IPv4" else: self.info(f"Could not determine target type for {qry}") return None headers = { 'Accept': 'application/json', 'X-OTX-API-KEY': self.opts['api_key'] } res = self.sf.fetchUrl( f"https://otx.alienvault.com/api/v1/indicators/{target_type}/{qry}/passive_dns", timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=headers) return self.parseApiResponse(res) def queryDomainUrlList(self, qry, page=1, per_page=50): params = urllib.parse.urlencode({ 'page': page, 'limit': per_page }) headers = { 'Accept': 'application/json', 'X-OTX-API-KEY': self.opts['api_key'] } res = self.sf.fetchUrl( f"https://otx.alienvault.com/api/v1/indicators/domain/{qry}/url_list?{params}", timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=headers) return self.parseApiResponse(res) def queryHostnameUrlList(self, qry, page=1, per_page=50): params = urllib.parse.urlencode({ 'page': page, 'limit': per_page }) headers = { 'Accept': 'application/json', 'X-OTX-API-KEY': self.opts['api_key'] } res = self.sf.fetchUrl( f"https://otx.alienvault.com/api/v1/indicators/hostname/{qry}/url_list?{params}", timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=headers) return self.parseApiResponse(res) # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "": self.error(f"You enabled {self.__class__.__name__} but did not set an API key!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName.startswith("AFFILIATE") and not self.opts['checkaffiliates']: return if eventName == 'INTERNET_NAME': urls = list() page = 1 while page <= self.opts['max_pages']: if self.checkForStop(): break if self.errorState: break data = self.queryHostnameUrlList(eventData, page=page) page += 1 url_list = data.get('url_list') if not url_list: break for url in url_list: u = url.get('url') if not u: continue urls.append(u) if not data.get('has_next'): break if self.sf.isDomain(eventData, self.opts['_internettlds']): page = 1 while page <= self.opts['max_pages']: if self.checkForStop(): break if self.errorState: break data = self.queryDomainUrlList(eventData, page=page) page += 1 url_list = data.get('url_list') if not url_list: break for url in url_list: u = url.get('url') if not u: continue urls.append(u) if not data.get('has_next'): break for url in set(urls): if not url: continue host = self.sf.urlFQDN(url.lower()) if not self.getTarget().matches(host, includeChildren=True, includeParents=True): continue if url not in self.results: self.results[url] = True evt = SpiderFootEvent('LINKED_URL_INTERNAL', url, self.__name__, event) self.notifyListeners(evt) return if eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']: if not self.opts['netblocklookup']: return if eventName == 'NETBLOCKV6_OWNER': max_netblock = self.opts['maxv6netblock'] else: max_netblock = self.opts['maxnetblock'] if IPNetwork(eventData).prefixlen < max_netblock: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}") return if eventName in ['NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']: if not self.opts['subnetlookup']: return if eventName == 'NETBLOCKV6_MEMBER': max_subnet = self.opts['maxv6subnet'] else: max_subnet = self.opts['maxsubnet'] if IPNetwork(eventData).prefixlen < max_subnet: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}") return qrylist = list() if eventName.startswith("NETBLOCK"): for ipaddr in IPNetwork(eventData): qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) # For IP addresses, do the additional passive DNS lookup if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']: ret = self.queryPassiveDns(eventData) if ret is None: self.info(f"No Passive DNS info for {eventData}") else: passive_dns = ret.get('passive_dns') if passive_dns: self.debug(f"Found passive DNS results for {eventData} in AlienVault OTX") for rec in passive_dns: host = rec.get('hostname') if not host: continue if self.getTarget().matches(host, includeParents=True): evtType = "INTERNET_NAME" if not self.sf.resolveHost(host) and not self.sf.resolveHost6(host): evtType = "INTERNET_NAME_UNRESOVLED" evt = SpiderFootEvent(evtType, host, self.__name__, event) self.notifyListeners(evt) continue if self.opts['cohost_age_limit_days'] > 0: try: last = rec.get("last", "") last_dt = datetime.strptime(last, '%Y-%m-%dT%H:%M:%S') last_ts = int(time.mktime(last_dt.timetuple())) age_limit_ts = int(time.time()) - (86400 * self.opts['cohost_age_limit_days']) if last_ts < age_limit_ts: self.debug(f"Passive DNS record {host} found for {eventData} is too old ({last_dt}), skipping.") continue except Exception: self.info("Could not parse date from AlienVault data, so ignoring cohost_age_limit_days") if self.opts["verify"] and not self.sf.validateIP(host, eventData): self.debug(f"Co-host {host} no longer resolves to {eventData}, skipping") continue if self.cohostcount < self.opts['maxcohost']: e = SpiderFootEvent("CO_HOSTED_SITE", host, self.__name__, event) self.notifyListeners(e) self.cohostcount += 1 else: self.info(f"Maximum co-host threshold exceeded ({self.opts['maxcohost']}), ignoring co-host {host}") if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS', 'NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']: evtType = 'MALICIOUS_IPADDR' elif eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS', 'NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']: evtType = 'MALICIOUS_AFFILIATE_IPADDR' elif eventName == "INTERNET_NAME": evtType = 'MALICIOUS_INTERNET_NAME' else: self.debug(f"Unexpected event type {eventName}, skipping") return for addr in qrylist: if self.checkForStop(): return if self.errorState: return rec = self.queryReputation(addr) if not rec: continue if rec.get("reputation", None): self.debug(f"Found reputation info for {addr} in AlienVault OTX") rec_history = rec['reputation'].get("activities", list()) threat_score = rec['reputation']['threat_score'] threat_score_min = self.opts['threat_score_min'] if threat_score < threat_score_min: self.debug(f"Threat score {threat_score} smaller than {threat_score_min}, skipping.") continue descr = f"AlienVault Threat Score: {threat_score}" for result in rec_history: nm = result.get("name", None) if nm is None or nm in descr: continue descr += "\n - " + nm created = result.get("last_date", "") if self.opts['reputation_age_limit_days'] > 0: try: # 2014-11-06T10:45:00.000 created_dt = datetime.strptime(created, '%Y-%m-%dT%H:%M:%S') created_ts = int(time.mktime(created_dt.timetuple())) age_limit_ts = int(time.time()) - (86400 * self.opts['reputation_age_limit_days']) if created_ts < age_limit_ts: self.debug(f"Reputation record found for {addr} is too old ({created_dt}), skipping.") continue except Exception: self.info("Could not parse date from AlienVault data, so ignoring reputation_age_limit_days") # For netblocks, we need to create the IP address event so that # the threat intel event is more meaningful. if eventName == 'NETBLOCK_OWNER': pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) if eventName == 'NETBLOCKV6_OWNER': pevent = SpiderFootEvent("IPV6_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) elif eventName == 'NETBLOCK_MEMBER': pevent = SpiderFootEvent("AFFILIATE_IPADDR", addr, self.__name__, event) self.notifyListeners(pevent) elif eventName == 'NETBLOCKV6_MEMBER': pevent = SpiderFootEvent("AFFILIATE_IPV6_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) else: pevent = event e = SpiderFootEvent(evtType, descr, self.__name__, pevent) self.notifyListeners(e) # End of sfp_alienvault class ================================================ FILE: modules/sfp_alienvaultiprep.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_alienvaultiprep # Purpose: Check if an IP or netblock is malicious according to the AlienVault # IP Reputation database. # # Author: steve@binarypool.com # # Created: 14/12/2013 # Copyright: (c) Steve Micallef, 2013 # Licence: MIT # ------------------------------------------------------------------------------- from netaddr import IPAddress, IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_alienvaultiprep(SpiderFootPlugin): meta = { 'name': "AlienVault IP Reputation", 'summary': "Check if an IP or netblock is malicious according to the AlienVault IP Reputation database.", 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://cybersecurity.att.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://cybersecurity.att.com/documentation/", "https://cybersecurity.att.com/resource-center", "https://success.alienvault.com/s/article/Can-I-use-the-OTX-IP-Reputation-List-as-a-blocklist", ], 'favIcon': "https://cdn-cybersecurity.att.com/images/uploads/logos/att-globe.svg", 'logo': "https://cdn-cybersecurity.att.com/images/uploads/logos/att-business-web.svg", 'description': "Looking at security through new eyes.\n" "AT&T Business and AlienVault have joined forces to create AT&T Cybersecurity, " "with a vision to bring together the people, process, and technology " "that help businesses of any size stay ahead of threats.", } } # Default options opts = { 'checkaffiliates': True, 'cacheperiod': 18, 'checknetblocks': True, 'checksubnets': True } # Option descriptions optdescs = { 'checkaffiliates': "Apply checks to affiliates?", 'cacheperiod': "Hours to cache list data before re-fetching.", 'checknetblocks': "Report if any malicious IPs are found within owned netblocks?", 'checksubnets': "Check if any malicious IPs are found within the same subnet of the target?" } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "AFFILIATE_IPADDR", "NETBLOCK_MEMBER", "NETBLOCK_OWNER" ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "BLACKLISTED_SUBNET", "BLACKLISTED_NETBLOCK", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_SUBNET", "MALICIOUS_NETBLOCK", ] def queryBlacklist(self, target, targetType): blacklist = self.retrieveBlacklist() if not blacklist: return False if targetType == "ip": if target in blacklist: self.debug(f"IP address {target} found in AlienVault IP Reputation Database blacklist.") return True elif targetType == "netblock": netblock = IPNetwork(target) for ip in blacklist: if IPAddress(ip) in netblock: self.debug(f"IP address {ip} found within netblock/subnet {target} in AlienVault IP Reputation Database blacklist.") return True return False def retrieveBlacklist(self): blacklist = self.sf.cacheGet('alienvaultiprep', 24) if blacklist is not None: return self.parseBlacklist(blacklist) res = self.sf.fetchUrl( "https://reputation.alienvault.com/reputation.generic", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from AlienVault IP Reputation Database.") self.errorState = True return None if res['content'] is None: self.error("Received no content from AlienVault IP Reputation Database") self.errorState = True return None self.sf.cachePut("alienvaultiprep", res['content']) return self.parseBlacklist(res['content']) def parseBlacklist(self, blacklist): """Parse plaintext blacklist Args: blacklist (str): plaintext blacklist from AlienVault IP Reputation Database Returns: list: list of blacklisted IP addresses """ ips = list() if not blacklist: return ips for ip in blacklist.split('\n'): ip = ip.strip().split(" #")[0] if ip.startswith('#'): continue if not self.sf.validIP(ip): continue ips.append(ip) return ips # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName == 'IP_ADDRESS': targetType = 'ip' malicious_type = "MALICIOUS_IPADDR" blacklist_type = "BLACKLISTED_IPADDR" elif eventName == 'AFFILIATE_IPADDR': if not self.opts.get('checkaffiliates', False): return targetType = 'ip' malicious_type = "MALICIOUS_AFFILIATE_IPADDR" blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" elif eventName == 'NETBLOCK_OWNER': if not self.opts.get('checknetblocks', False): return targetType = 'netblock' malicious_type = "MALICIOUS_NETBLOCK" blacklist_type = "BLACKLISTED_NETBLOCK" elif eventName == 'NETBLOCK_MEMBER': if not self.opts.get('checksubnets', False): return targetType = 'netblock' malicious_type = "MALICIOUS_SUBNET" blacklist_type = "BLACKLISTED_SUBNET" else: self.debug(f"Unexpected event type {eventName}, skipping") return self.debug(f"Checking maliciousness of {eventData} ({eventName}) with AlienVault IP Reputation Database") if not self.queryBlacklist(eventData, targetType): return url = "https://reputation.alienvault.com/reputation.generic" text = f"AlienVault IP Reputation Database [{eventData}]\n{url}" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_alienvaultiprep class ================================================ FILE: modules/sfp_apple_itunes.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_apple_itunes # Purpose: Query Apple iTunes for mobile apps. # # Author: # # Created: 2020-09-19 # Copyright: (c) bcoles 2019 # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_apple_itunes(SpiderFootPlugin): meta = { 'name': "Apple iTunes", 'summary': "Search Apple iTunes for mobile apps.", 'flags': [], 'useCases': ["Investigate", "Footprint", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://itunes.apple.com/", 'model': "FREE_AUTH_UNLIMITED", 'favIcon': "https://itunes.apple.com/favicon.ico", 'logo': "https://itunes.apple.com/favicon.ico", 'description': "The Apple iTunes store is a store for downloading " "and purchasing apps for Apple devices.", } } opts = { } optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ 'DOMAIN_NAME' ] def producedEvents(self): return [ 'APPSTORE_ENTRY', 'INTERNET_NAME', 'LINKED_URL_INTERNAL', 'AFFILIATE_INTERNET_NAME', 'RAW_RIR_DATA' ] def query(self, qry, limit=100): params = urllib.parse.urlencode({ 'media': 'software', 'entity': 'software,iPadSoftware,softwareDeveloper', 'limit': limit, 'term': qry.encode('raw_unicode_escape').decode("ascii", errors='replace') }) res = self.sf.fetchUrl( f"https://itunes.apple.com/search?{params}", useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout'] ) time.sleep(1) if res['content'] is None: return None try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from Apple iTunes: {e}") return None results = data.get('results') if not results: self.debug(f"No results found for {qry}") return None return results def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if eventName not in self.watchedEvents(): return self.results[eventData] = True domain_reversed = '.'.join(list(reversed(eventData.lower().split('.')))) data = self.query(domain_reversed) if not data: self.info(f"No results found for {eventData}") return urls = list() hosts = list() found = False for result in data: bundleId = result.get('bundleId') if not bundleId: continue trackName = result.get('trackName') if not trackName: continue version = result.get('version') if not version: continue app_full_name = f"{trackName} {version} ({bundleId})" if ( domain_reversed != bundleId.lower() and not bundleId.lower().startswith(f"{domain_reversed}.") and not bundleId.lower().endswith(f".{domain_reversed}") and f".{domain_reversed}." not in bundleId.lower() ): self.debug(f"App {app_full_name} does not match {domain_reversed}, skipping") continue trackViewUrl = result.get('trackViewUrl') if not trackViewUrl: continue app_data = f"{app_full_name}\n{trackViewUrl}" evt = SpiderFootEvent('APPSTORE_ENTRY', app_data, self.__name__, event) self.notifyListeners(evt) found = True sellerUrl = result.get('sellerUrl') if not sellerUrl: continue urls.append(sellerUrl) for url in set(urls): host = self.sf.urlFQDN(url) if not host: continue if self.getTarget().matches(host, includeChildren=True, includeParents=True): evt = SpiderFootEvent('LINKED_URL_INTERNAL', url, self.__name__, event) self.notifyListeners(evt) found = True hosts.append(host) for host in set(hosts): if not host: continue if self.getTarget().matches(host, includeChildren=True, includeParents=True): evt = SpiderFootEvent('INTERNET_NAME', host, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent('AFFILIATE_INTERNET_NAME', host, self.__name__, event) self.notifyListeners(evt) found = True if found: evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(data), self.__name__, event) self.notifyListeners(evt) # End of sfp_apple_itunes class ================================================ FILE: modules/sfp_archiveorg.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_archiveorg # Purpose: Queries archive.org (Wayback machine) for historic versions of # certain pages. # # Author: Steve Micallef # # Created: 20/07/2015 # Copyright: (c) Steve Micallef 2015 # Licence: MIT # ------------------------------------------------------------------------------- import datetime import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_archiveorg(SpiderFootPlugin): meta = { 'name': "Archive.org", 'summary': "Identifies historic versions of interesting files/pages from the Wayback Machine.", 'flags': ["slow"], 'useCases': ["Footprint", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://archive.org/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://archive.org/projects/", "https://archive.org/services/docs/api/" ], 'favIcon': "https://archive.org/images/glogo.jpg", 'logo': "https://archive.org/images/glogo.jpg", 'description': "Internet Archive is a non-profit library of millions of free books, movies, software, music, websites, and more.\n" "The Internet Archive, a 501(c)(3) non-profit, is building a digital library of Internet sites " "and other cultural artifacts in digital form. Like a paper library, we provide free access to " "researchers, historians, scholars, the print disabled, and the general public. " "Our mission is to provide Universal Access to All Knowledge.\n" "We began in 1996 by archiving the Internet itself, a medium that was just beginning to grow in use. " "Like newspapers, the content published on the web was ephemeral - but unlike newspapers, no one was saving it. " "Today we have 20+ years of web history accessible through the Wayback Machine and we work with 625+ library and " "other partners through our Archive-It program to identify important web pages.", } } # Default options opts = { 'farback': "30,60,90", 'intfiles': True, 'passwordpages': True, 'formpages': False, 'flashpages': False, 'javapages': False, 'staticpages': False, 'uploadpages': False, 'webframeworkpages': False, 'javascriptpages': False } # Option descriptions optdescs = { 'farback': "Number of days back to look for older versions of files/pages in the Wayback Machine snapshots. Comma-separate the values, so for example 30,60,90 means to look for snapshots 30 days, 60 days and 90 days back.", 'intfiles': "Query the Wayback Machine for historic versions of Interesting Files.", 'passwordpages': "Query the Wayback Machine for historic versions of URLs with passwords.", 'formpages': "Query the Wayback Machine for historic versions of URLs with forms.", 'uploadpages': "Query the Wayback Machine for historic versions of URLs accepting uploads.", 'flashpages': "Query the Wayback Machine for historic versions of URLs containing Flash.", 'javapages': "Query the Wayback Machine for historic versions of URLs using Java Applets.", 'staticpages': "Query the Wayback Machine for historic versions of purely static URLs.", "webframeworkpages": "Query the Wayback Machine for historic versions of URLs using Javascript frameworks.", "javascriptpages": "Query the Wayback Machine for historic versions of URLs using Javascript." } results = None foundDates = list() errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.foundDates = list() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["INTERESTING_FILE", "URL_PASSWORD", "URL_FORM", "URL_FLASH", "URL_STATIC", "URL_JAVA_APPLET", "URL_UPLOAD", "URL_JAVASCRIPT", "URL_WEB_FRAMEWORK"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["INTERESTING_FILE_HISTORIC", "URL_PASSWORD_HISTORIC", "URL_FORM_HISTORIC", "URL_FLASH_HISTORIC", "URL_STATIC_HISTORIC", "URL_JAVA_APPLET_HISTORIC", "URL_UPLOAD_HISTORIC", "URL_WEB_FRAMEWORK_HISTORIC", "URL_JAVASCRIPT_HISTORIC"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventName == "INTERESTING_FILE" and not self.opts['intfiles']: return if eventName == "URL_PASSWORD" and not self.opts['passwordpages']: return if eventName == "URL_STATIC" and not self.opts['staticpages']: return if eventName == "URL_FORM" and not self.opts['formpages']: return if eventName == "URL_UPLOAD" and not self.opts['uploadpages']: return if eventName == "URL_JAVA_APPLET" and not self.opts['javapages']: return if eventName == "URL_FLASH" and not self.opts['flashpages']: return if eventName == "URL_JAVASCRIPT" and not self.opts['javascriptpages']: return if eventName == "URL_WEB_FRAMEWORK" and not self.opts['webframeworkpages']: return if eventData in self.results: return self.results[eventData] = True for daysback in self.opts['farback'].split(","): try: newDate = datetime.datetime.now() - datetime.timedelta(days=int(daysback)) except Exception: self.error("Unable to parse option for number of days back to search.") self.errorState = True return maxDate = newDate.strftime("%Y%m%d") url = "https://archive.org/wayback/available?url=" + eventData + \ "×tamp=" + maxDate res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if res['content'] is None: self.error(f"Unable to fetch {url}") continue try: ret = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from Archive.org: {e}") ret = None if not ret: self.debug(f"Empty response from archive.org for {eventData}") continue if len(ret['archived_snapshots']) < 1: self.debug("No archived snapshots for " + eventData) continue wbmlink = ret['archived_snapshots']['closest']['url'] if wbmlink in self.foundDates: self.debug("Snapshot already fetched.") continue self.foundDates.append(wbmlink) name = eventName + "_HISTORIC" self.info("Found a historic file: " + wbmlink) evt = SpiderFootEvent(name, wbmlink, self.__name__, event) self.notifyListeners(evt) # End of sfp_archiveorg class ================================================ FILE: modules/sfp_arin.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_arin # Purpose: Queries the ARIN internet registry to get netblocks and other # bits of info. # # Author: Steve Micallef # # Created: 23/02/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_arin(SpiderFootPlugin): meta = { 'name': "ARIN", 'summary': "Queries ARIN registry for contact information.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Public Registries"], 'dataSource': { 'website': "https://www.arin.net/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://www.arin.net/resources/", "https://www.arin.net/reference/", "https://www.arin.net/participate/", "https://www.arin.net/resources/guide/request/", "https://www.arin.net/resources/registry/transfers/", "https://www.arin.net/resources/guide/ipv6/" ], 'favIcon': "https://www.arin.net/img/favicon.ico", 'logo': "https://www.arin.net/img/logo-stnd.svg", 'description': "ARIN is a nonprofit, member-based organization that administers IP addresses & " "ASNs in support of the operation and growth of the Internet.\n" "Established in December 1997 as a Regional Internet Registry, " "the American Registry for Internet Numbers (ARIN) is responsible for the management " "and distribution of Internet number resources such as Internet Protocol (IP) addresses " "and Autonomous System Numbers (ASNs). ARIN manages these resources within its service region, " "which is comprised of Canada, the United States, and many Caribbean and North Atlantic islands.", } } # Default options opts = {} optdescs = {} results = None currentEventSrc = None keywords = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.currentEventSrc = None for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['DOMAIN_NAME', 'HUMAN_NAME'] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["RAW_RIR_DATA", "HUMAN_NAME"] # Fetch content and notify of the raw data def fetchRir(self, url): head = {"Accept": "application/json"} res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], headers=head) if res['content'] is not None and res['code'] != "404": return res return None # Owner information about an AS def query(self, qtype, value): url = "https://whois.arin.net/rest/" if qtype == "domain": url += "pocs;domain=@" + value try: if qtype == "name": fname, lname = value.split(" ", 1) if fname.endswith(","): t = fname fname = lname lname = t url += "pocs;first=" + fname + ";last=" + lname except Exception as e: self.debug("Couldn't process name: " + value + " (" + str(e) + ")") return None if qtype == "contact": url = value res = self.fetchRir(url) if not res: self.debug("No info found/available for " + value + " at ARIN.") return None try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, self.currentEventSrc) self.notifyListeners(evt) return data # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.currentEventSrc = event self.debug(f"Received event, {eventName}, from {srcModuleName}") # Don't look up stuff twice if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == "DOMAIN_NAME": ret = self.query("domain", eventData) if not ret: return if "pocs" in ret: if "pocRef" in ret['pocs']: ref = list() # Might be a list or a dictionary if type(ret['pocs']['pocRef']) == dict: ref = [ret['pocs']['pocRef']] else: ref = ret['pocs']['pocRef'] for p in ref: name = p['@name'] if "," in name: sname = name.split(", ", 1) name = sname[1] + " " + sname[0] evt = SpiderFootEvent("HUMAN_NAME", name, self.__name__, self.currentEventSrc) self.notifyListeners(evt) # We just want the raw data so we can get potential # e-mail addresses. self.query("contact", p['$']) if eventName == "HUMAN_NAME": ret = self.query("name", eventData) if not ret: return if "pocs" in ret: if "pocRef" in ret['pocs']: ref = list() # Might be a list or a dictionary if type(ret['pocs']['pocRef']) == dict: ref = [ret['pocs']['pocRef']] else: ref = ret['pocs']['pocRef'] for p in ref: # We just want the raw data so we can get potential # e-mail addresses. self.query("contact", p['$']) # End of sfp_arin class ================================================ FILE: modules/sfp_azureblobstorage.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_azureblobstorage # Purpose: SpiderFoot plug-in for identifying potential Azure blobs related # to the target. # # Author: Steve Micallef # # Created: 14/07/2019 # Copyright: (c) Steve Micallef 2019 # Licence: MIT # ------------------------------------------------------------------------------- import random import threading import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_azureblobstorage(SpiderFootPlugin): meta = { 'name': "Azure Blob Finder", 'summary': "Search for potential Azure blobs associated with the target and attempt to list their contents.", 'flags': [], 'useCases': ["Footprint", "Passive"], 'categories': ["Crawling and Scanning"], 'dataSource': { 'website': "https://azure.microsoft.com/en-in/services/storage/blobs/", 'model': "FREE_NOAUTH_UNLIMITED", 'favIcon': 'https://azurecomcdn.azureedge.net/cvt-4fd6fa9ffb60246fd6387e4b34f89dc454cdf3df85d2b5d3215846066fceb0b6/images/icon/favicon.ico', 'logo': 'https://azurecomcdn.azureedge.net/cvt-4fd6fa9ffb60246fd6387e4b34f89dc454cdf3df85d2b5d3215846066fceb0b6/images/icon/favicon.ico', 'description': "Massively scalable and secure object storage for cloud-native workloads," "archives, data lakes, high-performance computing and machine learning." } } # Default options opts = { "suffixes": "test,dev,web,beta,bucket,space,files,content,data,prod,staging,production,stage,app,media,development,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data,-prod,-staging,-production,-stage,-app,-media,-development", "_maxthreads": 20 } # Option descriptions optdescs = { "suffixes": "List of suffixes to append to domains tried as blob storage names", "_maxthreads": "Maximum threads" } results = None s3results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.s3results = self.tempStorage() self.results = self.tempStorage() self.lock = threading.Lock() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME", "LINKED_URL_EXTERNAL"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["CLOUD_STORAGE_BUCKET"] def checkSite(self, url): res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True) if res['code']: with self.lock: self.s3results[url] = True def threadSites(self, siteList): self.s3results = dict() running = True i = 0 t = [] for site in siteList: if self.checkForStop(): return None self.info("Spawning thread to check bucket: " + site) tname = str(random.SystemRandom().randint(0, 999999999)) t.append(threading.Thread(name='thread_sfp_azureblobstorages_' + tname, target=self.checkSite, args=(site,))) t[i].start() i += 1 # Block until all threads are finished while running: found = False for rt in threading.enumerate(): if rt.name.startswith("thread_sfp_azureblobstorages_"): found = True if not found: running = False time.sleep(0.25) # Return once the scanning has completed return self.s3results def batchSites(self, sites): i = 0 res = list() siteList = list() for site in sites: if i >= self.opts['_maxthreads']: data = self.threadSites(siteList) if data is None: return res for ret in list(data.keys()): if data[ret]: res.append(ret) i = 0 siteList = list() siteList.append(site) i += 1 return res # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventName == "LINKED_URL_EXTERNAL": if ".blob.core.windows.net" in eventData: b = self.sf.urlFQDN(eventData) evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event) self.notifyListeners(evt) return targets = [eventData.replace('.', '')] kw = self.sf.domainKeyword(eventData, self.opts['_internettlds']) if kw: targets.append(kw) urls = list() for t in targets: suffixes = [''] + self.opts['suffixes'].split(',') for s in suffixes: if self.checkForStop(): return b = t + s + ".blob.core.windows.net" url = "https://" + b urls.append(url) # Batch the scans ret = self.batchSites(urls) for b in ret: evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event) self.notifyListeners(evt) # End of sfp_azureblobstorage class ================================================ FILE: modules/sfp_base64.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_base64 # Purpose: Identifies (best-effort) Base64-encoded strings in URLs. # # Author: Steve Micallef # # Created: 31/01/2017 # Copyright: (c) Steve Micallef 2017 # ------------------------------------------------------------------------------- import base64 import re import urllib.parse from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_base64(SpiderFootPlugin): meta = { 'name': "Base64 Decoder", 'summary': "Identify Base64-encoded strings in URLs, often revealing interesting hidden information.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Content Analysis"] } # Default options opts = { 'minlength': 10 } # Option descriptions optdescs = { 'minlength': "The minimum length a string that looks like a base64-encoded string needs to be." } def setup(self, sfc, userOpts=dict()): self.sf = sfc self.__dataSource__ = "Target Website" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["LINKED_URL_INTERNAL"] # What events this module produces def producedEvents(self): return ["BASE64_DATA"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") decoded_data = urllib.parse.unquote(eventData) # Note: this will miss base64 encoded strings with no padding # (strings which do not end with '=' or '==') pat = re.compile(r"([A-Za-z0-9+\/]+={1,2})") m = re.findall(pat, decoded_data) for match in m: if self.checkForStop(): return minlen = int(self.opts['minlength']) if len(match) < minlen: continue # Base64-encoded strings don't look like normal strings caps = sum(1 for c in match if c.isupper()) if caps < (minlen / 4): continue if isinstance(match, str): string = match else: string = str(match) self.info(f"Found Base64 string: {match}") try: string += f" ({base64.b64decode(match).decode('utf-8')})" except Exception as e: self.debug(f"Unable to base64-decode string: {e}") continue evt = SpiderFootEvent("BASE64_DATA", string, self.__name__, event) self.notifyListeners(evt) # End of sfp_base64 class ================================================ FILE: modules/sfp_bgpview.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_bgpview # Purpose: Query BGPView API - https://bgpview.docs.apiary.io/ # # Author: # # Created: 2019-09-03 # Copyright: (c) bcoles 2019 # Licence: MIT # ------------------------------------------------------------------------------- import json import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_bgpview(SpiderFootPlugin): meta = { 'name': "BGPView", 'summary': "Obtain network information from BGPView API.", 'flags': [], 'useCases': ["Investigate", "Footprint", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://bgpview.io/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://bgpview.docs.apiary.io/#", "https://bgpview.docs.apiary.io/api-description-document" ], 'favIcon': "https://bgpview.io/favicon-32x32.png", 'logo': "https://bgpview.io/assets/logo.png", 'description': "BGPView is a simple API allowing consumers to view all sort of analytics data about the current state and structure of the internet.", } } opts = { } optdescs = { } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ 'IP_ADDRESS', 'IPV6_ADDRESS', 'BGP_AS_MEMBER', 'NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER' ] def producedEvents(self): return [ 'BGP_AS_MEMBER', 'NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER', 'PHYSICAL_ADDRESS', 'RAW_RIR_DATA' ] def queryAsn(self, qry): res = self.sf.fetchUrl("https://api.bgpview.io/asn/" + qry.replace('AS', ''), useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout']) time.sleep(1) if res['content'] is None: return None try: json_data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from BGPView: {e}") return None if json_data.get('status') != 'ok': self.debug("No results found for ASN " + qry) return None data = json_data.get('data') if not data: self.debug("No results found for ASN " + qry) return None return data def queryIp(self, qry): res = self.sf.fetchUrl("https://api.bgpview.io/ip/" + qry, useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout']) time.sleep(1) if res['content'] is None: return None try: json_data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from BGPView: {e}") return None if json_data.get('status') != 'ok': self.debug("No results found for IP address " + qry) return None data = json_data.get('data') if not data: self.debug("No results found for IP address " + qry) return None return data def queryNetblock(self, qry): res = self.sf.fetchUrl("https://api.bgpview.io/prefix/" + qry, useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout']) time.sleep(1) if res['content'] is None: return None try: json_data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from BGPView: {e}") return None if json_data.get('status') != 'ok': self.debug("No results found for netblock " + qry) return None data = json_data.get('data') if not data: self.debug("No results found for netblock " + qry) return None return data def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == 'BGP_AS_MEMBER': data = self.queryAsn(eventData) if not data: self.info("No results found for ASN " + eventData) return e = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(e) address = data.get('owner_address') if not address: return evt = SpiderFootEvent('PHYSICAL_ADDRESS', ', '.join([_f for _f in address if _f]), self.__name__, event) self.notifyListeners(evt) if eventName in ['NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']: data = self.queryNetblock(eventData) if not data: self.info("No results found for netblock " + eventData) return e = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(e) address = data.get('owner_address') if not address: return evt = SpiderFootEvent('PHYSICAL_ADDRESS', ', '.join([_f for _f in address if _f]), self.__name__, event) self.notifyListeners(evt) if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']: data = self.queryIp(eventData) if not data: self.info("No results found for IP address " + eventData) return e = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(e) prefixes = data.get('prefixes') if not prefixes: self.info("No prefixes found for IP address " + eventData) return for prefix in prefixes: p = prefix.get('prefix') if not p: continue if not prefix.get('asn'): continue asn = prefix.get('asn').get('asn') if not asn: continue self.info(f"Netblock found: {p} ({asn})") evt = SpiderFootEvent("BGP_AS_MEMBER", str(asn), self.__name__, event) self.notifyListeners(evt) if self.sf.validIpNetwork(p): if ":" in p: evt = SpiderFootEvent("NETBLOCKV6_MEMBER", p, self.__name__, event) else: evt = SpiderFootEvent("NETBLOCK_MEMBER", p, self.__name__, event) self.notifyListeners(evt) # End of sfp_bgpview class ================================================ FILE: modules/sfp_binaryedge.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_binaryedge # Purpose: Query binaryedge.io using their API # # Author: Steve Micallef # # Created: 02/04/2019 # Copyright: (c) Steve Micallef 2019 # Licence: MIT # ------------------------------------------------------------------------------- import json import time from netaddr import IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_binaryedge(SpiderFootPlugin): meta = { 'name': "BinaryEdge", 'summary': "Obtain information from BinaryEdge.io Internet scanning systems, including breaches, vulnerabilities, torrents and passive DNS.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://www.binaryedge.io/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://docs.binaryedge.io/", "https://www.binaryedge.io/data.html" ], 'apiKeyInstructions': [ "Visit https://www.binaryedge.io/pricing.html", "Select a plan", "Sign up with new account", "Go to Account", "The API key is listed under 'API Access'" ], 'favIcon': "https://www.binaryedge.io/img/favicon/favicon-32x32.png", 'logo': "https://www.binaryedge.io/img/logo.png", 'description': "We scan the entire public internet, create real-time threat intelligence streams, " "and reports that show the exposure of what is connected to the Internet.\n" "We have built a distributed platform of scanners and honeypots, to acquire, classify and correlate different types of data.\n" "We use all of these datapoints to match those digital assets to an organization, " "allowing us to provide a global, up-to-date, view of organizations known and unknown assets.", } } opts = { 'binaryedge_api_key': "", 'torrent_age_limit_days': 30, 'cve_age_limit_days': 30, 'port_age_limit_days': 90, 'maxpages': 10, 'verify': True, 'netblocklookup': False, 'maxnetblock': 24, 'subnetlookup': False, 'maxsubnet': 24, 'maxcohost': 100 } optdescs = { 'binaryedge_api_key': "BinaryEdge.io API Key.", 'torrent_age_limit_days': "Ignore any torrent records older than this many days. 0 = unlimited.", 'cve_age_limit_days': "Ignore any vulnerability records older than this many days. 0 = unlimited.", 'port_age_limit_days': "Ignore any discovered open ports/banners older than this many days. 0 = unlimited.", 'verify': 'Verify that any hostnames found on the target domain still resolve?', 'maxpages': "Maximum number of pages to iterate through, to avoid exceeding BinaryEdge API usage limits. APIv2 has a maximum of 500 pages (10,000 results).", 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?", 'maxnetblock': "If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'subnetlookup': "Look up all IPs on subnets which your target is a part of?", 'maxsubnet': "If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'maxcohost': "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting." } results = None errorState = False cohostcount = 0 reportedhosts = None checkedips = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.reportedhosts = self.tempStorage() self.checkedips = self.tempStorage() self.cohostcount = 0 self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "DOMAIN_NAME", "EMAILADDR", "NETBLOCK_OWNER", "NETBLOCK_MEMBER" ] def producedEvents(self): return [ "INTERNET_NAME", "DOMAIN_NAME", "VULNERABILITY_CVE_CRITICAL", "VULNERABILITY_CVE_HIGH", "VULNERABILITY_CVE_MEDIUM", "VULNERABILITY_CVE_LOW", "VULNERABILITY_GENERAL", "TCP_PORT_OPEN", "TCP_PORT_OPEN_BANNER", "EMAILADDR_COMPROMISED", "UDP_PORT_OPEN", "UDP_PORT_OPEN_INFO", "CO_HOSTED_SITE", "MALICIOUS_IPADDR" ] def query(self, qry, querytype, page=1): retarr = list() if self.errorState: return None if querytype == "email": queryurl = "dataleaks/email" elif querytype == "ip": queryurl = "ip" elif querytype == "torrent": queryurl = "torrent/historical" elif querytype == "vuln": queryurl = "cve/ip" elif querytype == "subs": queryurl = "domains/subdomain" elif querytype == "passive": queryurl = "domains/ip" else: self.error(f"Invalid query type: {querytype}") return None headers = { 'X-Key': self.opts['binaryedge_api_key'] } res = self.sf.fetchUrl( f"https://api.binaryedge.io/v2/query/{queryurl}/{qry}?page={page}", timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=headers ) if res['code'] in ["429", "500"]: self.error("BinaryEdge.io API key seems to have been rejected or you have exceeded usage limits for the month.") self.errorState = True return None if not res['content']: self.info(f"No BinaryEdge.io info found for {qry}") return None try: info = json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from BinaryEdge.io: {e}") return None if info.get('page') and info['total'] > info.get('pagesize', 100) * info.get('page', 0): page = info['page'] + 1 if page > self.opts['maxpages']: self.error("Maximum number of pages reached.") return [info] retarr.append(info) e = self.query(qry, querytype, page) if e: retarr.extend(e) else: retarr.append(info) return retarr def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["binaryedge_api_key"] == "": self.error( f"You enabled {self.__class__.__name__} but did not set an API key!" ) self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == "EMAILADDR": ret = self.query(eventData, "email") if ret is None: self.info(f"No leak info for {eventData}") return for rec in ret: events = rec.get('events') if not events: continue self.debug("Found compromised account results in BinaryEdge.io") for leak in events: e = SpiderFootEvent('EMAILADDR_COMPROMISED', f"{eventData} [{leak}]", self.__name__, event) self.notifyListeners(e) # No further API endpoints available for email addresses. we can bail out here return if eventName == 'NETBLOCK_OWNER': if not self.opts['netblocklookup']: return net_size = IPNetwork(eventData).prefixlen max_netblock = self.opts['maxnetblock'] if net_size < max_netblock: self.debug(f"Network size bigger than permitted: {net_size} > {max_netblock}") return if eventName == 'NETBLOCK_MEMBER': if not self.opts['subnetlookup']: return net_size = IPNetwork(eventData).prefixlen max_subnet = self.opts['maxsubnet'] if net_size < max_subnet: self.debug(f"Network size bigger than permitted: {net_size} > {max_subnet}") return # For IP Addresses, do the additional passive DNS lookup if eventName == "IP_ADDRESS": evtType = "CO_HOSTED_SITE" ret = self.query(eventData, "passive") if ret is None: self.info(f"No Passive DNS info for {eventData}") return for rec in ret: events = rec.get('events') if not events: continue self.debug("Found passive DNS results in BinaryEdge.io") for rec in events: host = rec['domain'] if host == eventData: continue if self.getTarget().matches(host, includeParents=True): if self.opts['verify'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host): continue evt = SpiderFootEvent("INTERNET_NAME", host, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(host, self.opts['_internettlds']): evt = SpiderFootEvent("DOMAIN_NAME", host, self.__name__, event) self.notifyListeners(evt) self.reportedhosts[host] = True continue if self.cohostcount < self.opts['maxcohost']: e = SpiderFootEvent(evtType, host, self.__name__, event) self.notifyListeners(e) self.cohostcount += 1 if eventName == "DOMAIN_NAME": ret = self.query(eventData, "subs") if ret is None: self.info(f"No hosts found for {eventData}") return for rec in ret: events = rec.get('events') if not events: continue self.debug("Found host results in BinaryEdge.io") for rec in events: if rec in self.reportedhosts: continue self.reportedhosts[rec] = True if self.opts['verify'] and not self.sf.resolveHost(rec) and not self.sf.resolveHost6(rec): self.debug(f"Couldn't resolve {rec}, so skipping.") continue e = SpiderFootEvent('INTERNET_NAME', rec, self.__name__, event) self.notifyListeners(e) # Loop through all IP addresses / host names qrylist = list() if eventName.startswith("NETBLOCK_"): for ipaddr in IPNetwork(eventData): qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) for addr in qrylist: if self.checkForStop(): return if self.errorState: return if addr in self.checkedips: continue ret = self.query(addr, "torrent") if ret is None: self.info(f"No torrent info for {addr}") continue for rec in ret: events = rec.get('events') if not events: continue self.debug(f"Found torrent results for {addr} in BinaryEdge.io") for rec in events: created_ts = rec['origin'].get('ts') / 1000 age_limit_ts = int(time.time()) - (86400 * self.opts['torrent_age_limit_days']) if self.opts['torrent_age_limit_days'] > 0 and created_ts < age_limit_ts: self.debug("Record found but too old, skipping.") continue dat = "Torrent: " + rec.get("torrent", "???").get("name") + " @ " + rec.get('torrent').get("source", "???") e = SpiderFootEvent('MALICIOUS_IPADDR', dat, self.__name__, event) self.notifyListeners(e) for addr in qrylist: if self.checkForStop(): return if self.errorState: return if addr in self.checkedips: continue ret = self.query(addr, "vuln") if ret is None: self.info(f"No vulnerability info for {addr}") continue for rec in ret: events = rec.get('events') if not events: continue results = events.get('results') if not results: continue self.debug("Found vulnerability results in BinaryEdge.io") for rec in results: created_ts = rec.get('ts') / 1000 age_limit_ts = int(time.time()) - (86400 * self.opts['cve_age_limit_days']) if self.opts['cve_age_limit_days'] > 0 and created_ts < age_limit_ts: self.debug("Record found but too old, skipping.") continue cves = rec.get('cves') if cves: for c in cves: etype, cvetext = self.sf.cveInfo(c['cve']) e = SpiderFootEvent(etype, cvetext, self.__name__, event) self.notifyListeners(e) for addr in qrylist: if self.checkForStop(): return if self.errorState: return if addr in self.checkedips: continue ret = self.query(addr, "ip") if ret is None: self.info(f"No port/banner info for {addr}") return for rec in ret: events = rec.get('events') if not events: continue self.debug("Found port/banner results in BinaryEdge.io") ports = list() for res in events: for prec in res['results']: created_ts = prec['origin'].get('ts') / 1000 age_limit_ts = int(time.time()) - (86400 * self.opts['port_age_limit_days']) if self.opts['port_age_limit_days'] > 0 and created_ts < age_limit_ts: self.debug("Record found but too old, skipping.") continue port = str(prec['target']['port']) entity = prec['target']['ip'] + ":" + port evttype = "TCP_PORT_OPEN" evtbtype = "TCP_PORT_OPEN_BANNER" if prec['target']['protocol'] == "udp": evttype = "UDP_PORT_OPEN" evtbtype = "UDP_PORT_OPEN_INFO" if f"{evttype}:{port}" not in ports: ev = SpiderFootEvent(evttype, entity, self.__name__, event) self.notifyListeners(ev) ports.append(f"{evttype}:{port}") try: banner = prec['result']['data']['service']['banner'] if '\\r\\n\\r\\n' in banner and "HTTP/" in banner: # We don't want the content after HTTP banners banner = banner.split('\\r\\n\\r\\n')[0] banner = banner.replace("\\r\\n", "\n") except Exception: self.debug("No banner information found.") continue e = SpiderFootEvent(evtbtype, banner, self.__name__, ev) self.notifyListeners(e) for addr in qrylist: self.checkedips[addr] = True # End of sfp_binaryedge class ================================================ FILE: modules/sfp_bingsearch.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_bingsearch # Purpose: Searches Bing for content related to the domain in question. # # Author: Steve Micallef # # Created: 06/10/2013 # Copyright: (c) Steve Micallef 2013 # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_bingsearch(SpiderFootPlugin): meta = { 'name': "Bing", 'summary': "Obtain information from bing to identify sub-domains and links.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://www.bing.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://docs.microsoft.com/en-us/azure/cognitive-services/bing-web-search/" ], 'apiKeyInstructions': [ "Visit https://azure.microsoft.com/en-in/services/cognitive-services/bing-web-search-api/", "Register a free account", "Select on Bing Custom Search", "The API keys are listed under 'Key1' and 'Key2' (both should work)" ], 'favIcon': "https://www.bing.com/sa/simg/bing_p_rr_teal_min.ico", 'logo': "https://www.bing.com/sa/simg/bing_p_rr_teal_min.ico", 'description': "The Bing Search APIs let you build web-connected apps and services that " "find webpages, images, news, locations, and more without advertisements.", } } # Default options opts = { "pages": 20, "api_key": "" } # Option descriptions optdescs = { "pages": "Number of max bing results to request from the API.", "api_key": "Bing API Key for Bing search." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["INTERNET_NAME"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["LINKED_URL_INTERNAL", "RAW_RIR_DATA"] def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "": self.error("You enabled sfp_bingsearch but did not set a Bing API key!") self.errorState = True return if eventData in self.results: self.debug("Already did a search for " + eventData + ", skipping.") return self.results[eventData] = True # Sites hosted on the domain res = self.sf.bingIterate( searchString="site:" + eventData, opts={ "timeout": self.opts["_fetchtimeout"], "useragent": self.opts["_useragent"], "count": self.opts["pages"], "api_key": self.opts["api_key"], }, ) if res is None: # Failed to talk to the bing API or no results returned return urls = res["urls"] new_links = list(set(urls) - set(self.results.keys())) # Add new links to results for link in new_links: self.results[link] = True internal_links = [ link for link in new_links if self.sf.urlFQDN(link).endswith(eventData) ] for link in internal_links: self.debug("Found a link: " + link) evt = SpiderFootEvent("LINKED_URL_INTERNAL", link, self.__name__, event) self.notifyListeners(evt) if internal_links: evt = SpiderFootEvent( "RAW_RIR_DATA", str(res), self.__name__, event ) self.notifyListeners(evt) # End of sfp_bingsearch class ================================================ FILE: modules/sfp_bingsharedip.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_bingsharedip # Purpose: Searches Bing for hosts sharing the same IP. # # Author: Steve Micallef # # Created: 12/04/2014 # Copyright: (c) Steve Micallef 2014 # Licence: MIT # ------------------------------------------------------------------------------- from netaddr import IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_bingsharedip(SpiderFootPlugin): meta = { 'name': "Bing (Shared IPs)", 'summary': "Search Bing for hosts sharing the same IP.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://www.bing.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://docs.microsoft.com/en-us/azure/cognitive-services/bing-web-search/" ], 'apiKeyInstructions': [ "Visit https://azure.microsoft.com/en-in/services/cognitive-services/bing-web-search-api/", "Register a free account", "Select on Bing Custom Search", "The API keys are listed under 'Key1' and 'Key2' (both should work)" ], 'favIcon': "https://www.bing.com/sa/simg/bing_p_rr_teal_min.ico", 'logo': "https://www.bing.com/sa/simg/bing_p_rr_teal_min.ico", 'description': "The Bing Search APIs let you build web-connected apps and services that " "find webpages, images, news, locations, and more without advertisements.", } } # Default options opts = { "cohostsamedomain": False, "pages": 20, "verify": True, "maxcohost": 100, "api_key": "" } # Option descriptions optdescs = { "cohostsamedomain": "Treat co-hosted sites on the same target domain as co-hosting?", "pages": "Number of max bing results to request from API.", "verify": "Verify co-hosts are valid by checking if they still resolve to the shared IP.", "maxcohost": "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.", "api_key": "Bing API Key for shared IP search." } results = None cohostcount = 0 errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.cohostcount = 0 self.__dataSource__ = "Bing" self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["IP_ADDRESS", "NETBLOCK_OWNER"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["CO_HOSTED_SITE", "IP_ADDRESS", "RAW_RIR_DATA"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.currentEventSrc = event if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "" and self.opts['api_key'] == "": self.error("You enabled sfp_bingsharedip but did not set a Bing API key!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return # Ignore IP addresses from myself as they are just for creating # a link from the netblock to the co-host. if eventName == "IP_ADDRESS" and srcModuleName == "sfp_bingsharedip": self.debug("Ignoring " + eventName + ", from self.") return if self.cohostcount > self.opts["maxcohost"]: return qrylist = list() if eventName.startswith("NETBLOCK_"): for ipaddr in IPNetwork(eventData): if str(ipaddr) not in self.results: qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) self.results[eventData] = True myres = list() for ip in qrylist: if self.checkForStop(): return res = self.sf.bingIterate( searchString="ip:" + ip, opts={ "timeout": self.opts["_fetchtimeout"], "useragent": self.opts["_useragent"], "count": self.opts["pages"], "api_key": self.opts["api_key"], }, ) if res is None: # Failed to talk to bing api or no results returned return urls = res["urls"] for url in urls: self.info("Found something on same IP: " + url) site = self.sf.urlFQDN(url.lower()) if site not in myres and site != ip: if not self.opts["cohostsamedomain"]: if self.getTarget().matches(site, includeParents=True): self.debug( f"Skipping {site} because it is on the same domain." ) continue if self.opts["verify"] and not self.sf.validateIP(site, ip): self.debug(f"Host {site} no longer resolves to {ip}") continue # Create an IP Address event stemming from the netblock as the # link to the co-host. if eventName == "NETBLOCK_OWNER": ipe = SpiderFootEvent("IP_ADDRESS", ip, self.__name__, event) self.notifyListeners(ipe) evt = SpiderFootEvent( "CO_HOSTED_SITE", site, self.__name__, ipe ) self.notifyListeners(evt) else: evt = SpiderFootEvent( "CO_HOSTED_SITE", site, self.__name__, event ) self.notifyListeners(evt) self.cohostcount += 1 myres.append(site) if urls: evt = SpiderFootEvent( "RAW_RIR_DATA", str(res), self.__name__, event ) self.notifyListeners(evt) # End of sfp_bingsharedip class ================================================ FILE: modules/sfp_binstring.py ================================================ # coding: utf-8 # ------------------------------------------------------------------------------- # Name: sfp_binstring # Purpose: Identify strings in binary content. # # Author: Steve Micallef # # Created: 03/12/2016 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import string from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_binstring(SpiderFootPlugin): meta = { 'name': "Binary String Extractor", 'summary': "Attempt to identify strings in binary content.", 'flags': ["errorprone"], 'useCases': ["Footprint"], 'categories': ["Content Analysis"] } # Default options opts = { 'minwordsize': 5, 'maxwords': 100, 'maxfilesize': 1000000, 'usedict': True, 'fileexts': ['png', 'gif', 'jpg', 'jpeg', 'tiff', 'tif', 'ico', 'flv', 'mp4', 'mp3', 'avi', 'mpg', 'mpeg', 'dat', 'mov', 'swf', 'exe', 'bin'], 'filterchars': '#}{|%^&*()=+,;[]~' } # Option descriptions optdescs = { 'minwordsize': "Upon finding a string in a binary, ensure it is at least this length. Helps weed out false positives.", 'usedict': "Use the dictionary to further reduce false positives - any string found must contain a word from the dictionary (can be very slow, especially for larger files).", 'fileexts': "File types to fetch and analyse.", 'maxfilesize': "Maximum file size in bytes to download for analysis.", 'maxwords': "Stop reporting strings from a single binary after this many are found.", 'filterchars': "Ignore strings with these characters, as they may just be garbage ASCII." } results = list() d = None n = None fq = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = list() self.__dataSource__ = "Target Website" self.d = SpiderFootHelpers.dictionaryWordsFromWordlists() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def getStrings(self, content): words = list() result = "" if not content: return None for c in content: c = str(c) if len(words) >= self.opts['maxwords']: break if c in string.printable and c not in string.whitespace: result += c continue if len(result) >= self.opts['minwordsize']: if self.opts['usedict']: accept = False for w in self.d: if result.startswith(w) or result.endswith(w): accept = True break if self.opts['filterchars']: accept = True for x in self.opts['filterchars']: if x in result: accept = False break if not self.opts['filterchars'] and not self.opts['usedict']: accept = True if accept: words.append(result) result = "" if len(words) == 0: return None return words # What events is this module interested in for input def watchedEvents(self): return ["LINKED_URL_INTERNAL"] # What events this module produces def producedEvents(self): return ["RAW_FILE_META_DATA"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: return self.results.append(eventData) for fileExt in self.opts['fileexts']: if eventData.lower().endswith(f".{fileExt.lower()}") or f".{fileExt.lower()}?" in eventData.lower(): res = self.sf.fetchUrl( eventData, useragent=self.opts['_useragent'], disableContentEncoding=True, sizeLimit=self.opts['maxfilesize'], verify=False ) if not res: continue self.debug(f"Searching {eventData} for strings") words = self.getStrings(res['content']) if words: wordstr = '\n'.join(words[0:self.opts['maxwords']]) evt = SpiderFootEvent("RAW_FILE_META_DATA", wordstr, self.__name__, event) self.notifyListeners(evt) # End of sfp_binstring class ================================================ FILE: modules/sfp_bitcoin.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_bitcoin # Purpose: SpiderFoot plug-in for scanning retrieved content by other # modules (such as sfp_spider) and identifying bitcoin numbers. # # Author: Steve Micallef # # Created: 27/05/2017 # Copyright: (c) Steve Micallef 2017 # Licence: MIT # ------------------------------------------------------------------------------- import codecs import re from hashlib import sha256 from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_bitcoin(SpiderFootPlugin): meta = { 'name': "Bitcoin Finder", 'summary': "Identify bitcoin addresses in scraped webpages.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Content Analysis"] } opts = {} optdescs = {} results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["TARGET_WEB_CONTENT"] def producedEvents(self): return ["BITCOIN_ADDRESS"] def to_bytes(self, n, length): h = '%x' % n return codecs.decode(('0' * (len(h) % 2) + h).zfill(length * 2), "hex") def decode_base58(self, bc, length): digits58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' n = 0 for char in bc: n = n * 58 + digits58.index(char) return self.to_bytes(n, length) def check_bc(self, bc): bcbytes = self.decode_base58(bc, 25) return bcbytes[-4:] == sha256(sha256(bcbytes[:-4]).digest()).digest()[:4] def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data sourceData = self.sf.hashstring(eventData) if sourceData in self.results: return self.results[sourceData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") addrs = list() # thanks to https://stackoverflow.com/questions/21683680/regex-to-match-bitcoin-addresses # Does not support keys or testnet addresses matches = re.findall(r"[\s:=\>](bc(0([ac-hj-np-z02-9]{39}|[ac-hj-np-z02-9]{59})|1[ac-hj-np-z02-9]{8,87})|[13][a-km-zA-HJ-NP-Z1-9]{25,35})", eventData) for m in matches: address = m[0] self.debug(f"Potential Bitcoin address match: {address}") if address.startswith('1') or address.startswith('3'): if self.check_bc(address): addrs.append(address) else: addrs.append(address) for address in set(addrs): evt = SpiderFootEvent("BITCOIN_ADDRESS", address, self.__name__, event) self.notifyListeners(evt) # End of sfp_bitcoin class ================================================ FILE: modules/sfp_bitcoinabuse.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_bitcoinabuse # Purpose: Check bitcoin address agains bitcoinabuse.com database # # Author: Leo Trubach # # Created: 2020-09-01 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib.parse from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_bitcoinabuse(SpiderFootPlugin): meta = { "name": "BitcoinAbuse", "summary": "Check Bitcoin addresses against the bitcoinabuse.com database of suspect/malicious addresses.", 'flags': ["apikey"], "useCases": ["Passive", "Investigate"], "categories": ["Reputation Systems"], "dataSource": { "website": "https://www.bitcoinabuse.com/", "model": "FREE_AUTH_UNLIMITED", "references": ["https://www.bitcoinabuse.com/api-docs"], "apiKeyInstructions": [ "Visit https://www.bitcoinabuse.com/register", "Register a free account", "Click on the account icon and click on 'Your Settings'", "Click on 'API'", "Enter a token name and press 'Create'", ], "favIcon": "https://www.bitcoinabuse.com/favicon-32x32.png", "logo": "https://www.bitcoinabuse.com/img/logo-sm.png", "description": "BitcoinAbuse.com is a public database of bitcoin " "addresses used by scammers, hackers, and criminals." "Bitcoin is anonymous if used perfectly. Luckily, no " "one is perfect. Even hackers make mistakes. It only " "takes one slip to link stolen bitcoin to a hacker's " "their real identity. It is our hope that by making a " "public database of bitcoin addresses used by criminals it " "will be harder for criminals to convert the digital currency" " back into fiat money.", }, } opts = { "api_key": "", } optdescs = { "api_key": "BitcoinAbuse API Key.", } results = None errorState = False def setup(self, sfc, userOpts=None): self.sf = sfc self.errorState = False self.results = self.tempStorage() if userOpts: self.opts.update(userOpts) def watchedEvents(self): return ["BITCOIN_ADDRESS"] def producedEvents(self): return [ "MALICIOUS_BITCOIN_ADDRESS", "RAW_RIR_DATA", ] def queryAddress(self, address: str): params = { "address": address, "api_token": self.opts["api_key"] } res = self.sf.fetchUrl( f"https://www.bitcoinabuse.com/api/reports/check?{urllib.parse.urlencode(params)}", timeout=self.opts["_fetchtimeout"], useragent="SpiderFoot", ) # All endpoints other than Report Address have a rate limit of # 30 requests per minute or one request every two seconds on average. time.sleep(2) return self.parseApiResponse(res) def parseApiResponse(self, res: dict): if not res: self.error("No response from BitcoinAbuse.") return None if res['code'] == '404': self.debug("No results for query") return None if res['code'] == "401": self.error("Invalid BitcoinAbuse API key.") self.errorState = True return None if res['code'] == '429': self.error("You are being rate-limited by BitcoinAbuse") self.errorState = True return None if res['code'] in ['500', '502', '503']: self.error("BitcoinAbuse service unavailable") self.errorState = True return None # Catch all other non-200 status codes, and presume something went wrong if res['code'] != '200': self.error("Failed to retrieve content from BitcoinAbuse") self.errorState = True return None if res['content'] is None: return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from BitcoinAbuse: {e}") return None def handleEvent(self, event): if self.errorState: return eventName = event.eventType self.debug(f"Received event, {eventName}, from {event.module}") if self.opts["api_key"] == "": self.error(f"You enabled {self.__class__.__name__} but did not set an API key!") self.errorState = True return eventData = event.data if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == "BITCOIN_ADDRESS": rec = self.queryAddress(eventData) if not rec: return if not isinstance(rec, dict): return count = rec.get("count") if not count: return if not isinstance(count, int): return address = rec.get('address') if not address: return url = f"https://www.bitcoinabuse.com/reports/{address}" evt = SpiderFootEvent( "MALICIOUS_BITCOIN_ADDRESS", f"BitcoinAbuse [{address}]\n{url}", self.__name__, event ) self.notifyListeners(evt) rirevt = SpiderFootEvent( "RAW_RIR_DATA", json.dumps(rec), self.__name__, event ) self.notifyListeners(rirevt) ================================================ FILE: modules/sfp_bitcoinwhoswho.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_bitcoinwhoswho # Purpose: Bitcoin Who's Who database lookup module # # Author: Leo Trubach # # Created: 2020-09-09 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import urllib.parse from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_bitcoinwhoswho(SpiderFootPlugin): meta = { 'name': "Bitcoin Who's Who", 'summary': "Check for Bitcoin addresses against the Bitcoin Who's Who database of suspect/malicious addresses.", 'flags': ["apikey"], 'useCases': ["Passive", "Investigate"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://bitcoinwhoswho.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://bitcoinwhoswho.com/api" ], 'apiKeyInstructions': [ "Visit https://bitcoinwhoswho.com/signup", "Register a free account", "Verify your email and sign into your account", "Visit https://bitcoinwhoswho.com/api/register and request an API Key", "Wait for a few days, you'll receive it to your email" ], 'favIcon': "https://bitcoinwhoswho.com/public/images/ico/favicon.ico", 'logo': "https://bitcoinwhoswho.com/public/images/logo2.png", 'description': ( "Bitcoin Who's Who is dedicated to profiling the extraordinary members of the bitcoin " "ecosystem.Our goal is to help you verify a bitcoin address owner and avoid a bitcoin " "scam or fraud." ), } } opts = { 'api_key': '', } optdescs = { "api_key": "Bitcoin Who's Who API Key." } results = None errorState = False def setup(self, sfc, userOpts=None): self.sf = sfc self.results = self.tempStorage() if userOpts: self.opts.update(userOpts) def watchedEvents(self): return ["BITCOIN_ADDRESS"] def producedEvents(self): return ["MALICIOUS_BITCOIN_ADDRESS", "RAW_RIR_DATA"] def query(self, qry): qs = urllib.parse.urlencode({"address": qry}) res = self.sf.fetchUrl( f"https://bitcoinwhoswho.com/api/scam/{self.opts['api_key']}?{qs}", timeout=self.opts["_fetchtimeout"], useragent="SpiderFoot", ) if res["content"] is None: self.info(f"No {self.meta['name']} info found for {qry}") return None try: return json.loads(res["content"]) except Exception as e: self.error(f"Error processing JSON response from {self.meta['name']}: {e}") return None def emit(self, etype, data, pevent, notify=True): evt = SpiderFootEvent(etype, data, self.__name__, pevent) if notify: self.notifyListeners(evt) return evt def generate_events(self, data, pevent): if not isinstance(data, dict): return False scams = data.get("scams", []) if scams: self.emit("MALICIOUS_BITCOIN_ADDRESS", f"Bitcoin Who's Who [{pevent.data}][https://bitcoinwhoswho.com/address/{pevent.data}]", pevent) return True return False def handleEvent(self, event): if self.errorState: return self.debug(f"Received event, {event.eventType}, from {event.module}") if self.opts["api_key"] == "": self.error(f"You enabled {self.__class__.__name__} but did not set an API key!") self.errorState = True return if event.data in self.results: self.debug(f"Skipping {event.data}, already checked.") return self.results[event.data] = True if event.eventType == "BITCOIN_ADDRESS": data = self.query(event.data) r = self.generate_events(data, event) if r: self.emit("RAW_RIR_DATA", json.dumps(data), event) # End of sfp_bitcoinwhoswho class ================================================ FILE: modules/sfp_blockchain.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_blockchain # Purpose: SpiderFoot plug-in to look up a bitcoin wallet's balance by # querying blockchain.info. # # Author: Steve Micallef # # Created: 18/06/2017 # Copyright: (c) Steve Micallef 2017 # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_blockchain(SpiderFootPlugin): meta = { 'name': "Blockchain", 'summary': "Queries blockchain.info to find the balance of identified bitcoin wallet addresses.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Public Registries"], 'dataSource': { 'website': "https://www.blockchain.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://exchange.blockchain.com/api/#introduction", "https://exchange.blockchain.com/markets", "https://exchange.blockchain.com/fees", "https://exchange.blockchain.com/trade" ], 'favIcon': "https://www.blockchain.com/static/favicon.ico", 'logo': "https://exchange.blockchain.com/api/assets/images/logo.png", 'description': "Blockchain Exchange is the most secure place to buy, sell, and trade crypto.\n" "Use the most popular block explorer to search and " "verify transactions on the Bitcoin, Ethereum, and Bitcoin Cash blockchains.\n" "Stay on top of Bitcoin and other top cryptocurrency prices, news, and market information.", } } # Default options opts = {} optdescs = {} results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['BITCOIN_ADDRESS'] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["BITCOIN_BALANCE"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True # Wallet balance res = self.sf.fetchUrl("https://blockchain.info/balance?active=" + eventData, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if res['content'] is None: self.info("No Blockchain info found for " + eventData) return try: data = json.loads(res['content']) balance = float(data[eventData]['final_balance']) / 100000000 except Exception as e: self.debug(f"Error processing JSON response: {e}") return evt = SpiderFootEvent("BITCOIN_BALANCE", str(balance) + " BTC", self.__name__, event) self.notifyListeners(evt) # End of sfp_blockchain class ================================================ FILE: modules/sfp_blocklistde.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_blocklistde # Purpose: Check if a netblock or IP is malicious according to blocklist.de. # # Author: steve@binarypool.com # # Created: 14/12/2013 # Copyright: (c) Steve Micallef, 2013 # Licence: MIT # ------------------------------------------------------------------------------- from netaddr import IPAddress, IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_blocklistde(SpiderFootPlugin): meta = { 'name': "blocklist.de", 'summary': "Check if a netblock or IP is malicious according to blocklist.de.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "http://www.blocklist.de/en/index.html", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "http://www.blocklist.de/en/api.html", "http://www.blocklist.de/en/rbldns.html", "http://www.blocklist.de/en/httpreports.html", "http://www.blocklist.de/en/export.html", "http://www.blocklist.de/en/delist.html?ip=" ], 'favIcon': "http://www.blocklist.de/templates/css/logo_web-size.jpg", 'logo': "http://www.blocklist.de/templates/css/logo_web-size.jpg", 'description': "www.blocklist.de is a free and voluntary service provided by a Fraud/Abuse-specialist, " "whose servers are often attacked via SSH-, Mail-Login-, FTP-, Webserver- and other services.\n" "The mission is to report any and all attacks to the respective abuse departments of the infected PCs/servers, " "to ensure that the responsible provider can inform their customer about the infection and disable the attacker." } } opts = { 'checkaffiliates': True, 'cacheperiod': 18, 'checknetblocks': True, 'checksubnets': True } optdescs = { 'checkaffiliates': "Apply checks to affiliates?", 'cacheperiod': "Hours to cache list data before re-fetching.", 'checknetblocks': "Report if any malicious IPs are found within owned netblocks?", 'checksubnets': "Check if any malicious IPs are found within the same subnet of the target?" } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "IPV6_ADDRESS", "AFFILIATE_IPADDR", "AFFILIATE_IPV6_ADDRESS", "NETBLOCK_MEMBER", "NETBLOCKV6_MEMBER", "NETBLOCK_OWNER", "NETBLOCKV6_OWNER", ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "BLACKLISTED_SUBNET", "BLACKLISTED_NETBLOCK", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK", "MALICIOUS_SUBNET", ] def queryBlacklist(self, target, targetType): blacklist = self.retrieveBlacklist() if not blacklist: return False if targetType == "ip": if target in blacklist: self.debug(f"IP address {target} found in blocklist.de blacklist.") return True elif targetType == "netblock": netblock = IPNetwork(target) for ip in blacklist: if IPAddress(ip) in netblock: self.debug(f"IP address {ip} found within netblock/subnet {target} in blocklist.de blacklist.") return True return False def retrieveBlacklist(self): blacklist = self.sf.cacheGet('blocklistde', 24) if blacklist is not None: return self.parseBlacklist(blacklist) res = self.sf.fetchUrl( "https://lists.blocklist.de/lists/all.txt", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from blocklist.de.") self.errorState = True return None if res['content'] is None: self.error("Received no content from blocklist.de") self.errorState = True return None self.sf.cachePut("blocklistde", res['content']) return self.parseBlacklist(res['content']) def parseBlacklist(self, blacklist): """Parse plaintext blacklist Args: blacklist (str): plaintext blacklist from blocklist.de Returns: list: list of blacklisted IP addresses """ ips = list() if not blacklist: return ips for ip in blacklist.split('\n'): ip = ip.strip() if ip.startswith('#'): continue if not self.sf.validIP(ip) and not self.sf.validIP6(ip): continue ips.append(ip) return ips def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']: targetType = 'ip' malicious_type = 'MALICIOUS_IPADDR' blacklist_type = 'BLACKLISTED_IPADDR' elif eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS']: if not self.opts.get('checkaffiliates', False): return targetType = 'ip' malicious_type = 'MALICIOUS_AFFILIATE_IPADDR' blacklist_type = 'BLACKLISTED_AFFILIATE_IPADDR' elif eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']: if not self.opts.get('checknetblocks', False): return targetType = 'netblock' malicious_type = 'MALICIOUS_NETBLOCK' blacklist_type = 'BLACKLISTED_NETBLOCK' elif eventName in ['NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']: if not self.opts.get('checksubnets', False): return targetType = 'netblock' malicious_type = 'MALICIOUS_SUBNET' blacklist_type = 'BLACKLISTED_SUBNET' else: self.debug(f"Unexpected event type {eventName}, skipping") return self.debug(f"Checking maliciousness of {eventData} ({eventName}) with blocklist.de") if self.queryBlacklist(eventData, targetType): # https://www.blocklist.de/en/search.html?ip= url = "https://lists.blocklist.de/lists/all.txt" text = f"blocklist.de [{eventData}]\n{url}" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_blocklistde class ================================================ FILE: modules/sfp_botscout.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_botscout # Purpose: SpiderFoot plug-in to search botsout.com using their API, for # potential malicious IPs and e-mail addresses. # # Author: Steve Micallef # # Created: 25/07/2016 # Copyright: (c) Steve Micallef 2016 # Licence: MIT # ------------------------------------------------------------------------------- import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_botscout(SpiderFootPlugin): meta = { 'name': "BotScout", 'summary': "Searches BotScout.com's database of spam-bot IP addresses and e-mail addresses.", 'flags': ["apikey"], 'useCases': ["Passive", "Investigate"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://botscout.com/", 'model': "FREE_NOAUTH_LIMITED", 'references': [ "http://botscout.com/api.htm", "http://botscout.com/api_queries.htm", "http://botscout.com/getkey.htm", "http://botscout.com/corp_users.htm" ], 'apiKeyInstructions': [ "Visit http://botscout.com/getkey.htm", "Register a free account", "The API key will be emailed to your account" ], 'favIcon': "https://botscout.com/favicon.ico", 'logo': "http://botscout.com/image/bslogo.gif", 'description': "BotScout helps prevent automated web scripts, known as \"bots\", " "from registering on forums, polluting databases, spreading spam, " "and abusing forms on web sites. We do this by tracking the names, IPs, " "and email addresses that bots use and logging them as unique signatures for future reference. " "We also provide a simple yet powerful API that you can use to test forms " "when they're submitted on your site.", } } opts = { "api_key": "" } optdescs = { "api_key": "Botscout.com API key. Without this you will be limited to 100 look-ups per day." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ['IP_ADDRESS', 'EMAILADDR'] def producedEvents(self): return ["MALICIOUS_IPADDR", "BLACKLISTED_IPADDR", "MALICIOUS_EMAILADDR"] def queryIp(self, ip): if not self.sf.validIP(ip): return None params = urllib.parse.urlencode({ 'ip': ip, 'key': self.opts['api_key'], }) res = self.sf.fetchUrl( f"https://botscout.com/test/?{params}", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) return self.parseApiResponse(res) def queryEmail(self, email): if not SpiderFootHelpers.validEmail(email): return None params = urllib.parse.urlencode({ 'mail': email, 'key': self.opts['api_key'], }) res = self.sf.fetchUrl( f"https://botscout.com/test/?{params}", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) return self.parseApiResponse(res) def parseApiResponse(self, res: dict): if not res: self.error("No response from BotScout.") return None if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from BotScout.") self.errorState = True return None if not res['content']: self.error("No response from BotScout.") return None if res['content'].startswith("! "): self.error(f"Received error from BotScout: {res['content']}") self.errorState = True return None if not res['content'].startswith("Y|") and not res['content'].startswith("N|"): self.error("Error encountered processing response from BotScout.") return None return res['content'] def handleEvent(self, event): eventName = event.eventType eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {event.module}") if not self.opts['api_key']: self.info("You enabled sfp_botscout but did not set an API key! Queries will be limited to 100 per day.") if eventData in self.results: self.debug(f"Skipping {eventData} as already searched.") return self.results[eventData] = True if eventName == "IP_ADDRESS": res = self.queryIp(eventData) if not res: return if not res.startswith("Y|"): return self.info(f"Found BotScout entry for {eventData}: {res}") url = f"https://botscout.com/ipcheck.htm?ip={eventData}" text = f"BotScout [{eventData}]\n{url}" evt = SpiderFootEvent("MALICIOUS_IPADDR", text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent("BLACKLISTED_IPADDR", text, self.__name__, event) self.notifyListeners(evt) elif eventName == "EMAILADDR": res = self.queryEmail(eventData) if not res: return if not res.startswith("Y|"): return url = f"https://botscout.com/search.htm?sterm={eventData}&stype=q" text = f"BotScout [{eventData}]\n{url}" evt = SpiderFootEvent("MALICIOUS_EMAILADDR", text, self.__name__, event) self.notifyListeners(evt) else: self.debug(f"Unexpected event type {eventName}, skipping") # End of sfp_botscout class ================================================ FILE: modules/sfp_botvrij.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_botvrij # Purpose: Check if a domain is malicious according to botvrij.eu. # # Author: steve@binarypool.com # # Created: 16/05/2020 # Copyright: (c) Steve Micallef, 2020 # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_botvrij(SpiderFootPlugin): meta = { 'name': "botvrij.eu", 'summary': "Check if a domain is malicious according to botvrij.eu.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://botvrij.eu/", 'model': "FREE_NOAUTH_UNLIMITED", 'description': "Botvrij.eu provides different sets " " of open source IOCs that you can use in your " " security devices to detect possible malicious activity.\n" "The information contains network info (IPs), file hashes," " file paths, domain names, URLs.", } } opts = { 'checkaffiliates': True, 'checkcohosts': True, 'cacheperiod': 18 } optdescs = { 'checkaffiliates': "Apply checks to affiliates?", 'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?", 'cacheperiod': "Hours to cache list data before re-fetching." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "CO_HOSTED_SITE", ] def producedEvents(self): return [ "BLACKLISTED_INTERNET_NAME", "BLACKLISTED_AFFILIATE_INTERNET_NAME", "BLACKLISTED_COHOST", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_COHOST", ] def queryBlacklist(self, target): blacklist = self.retrieveBlacklist() if not blacklist: return False if target.lower() in blacklist: self.debug(f"Host name {target} found in botvrij.eu blacklist.") return True return False def retrieveBlacklist(self): blacklist = self.sf.cacheGet('botvrij', 24) if blacklist is not None: return self.parseBlacklist(blacklist) res = self.sf.fetchUrl( "https://www.botvrij.eu/data/blocklist/blocklist_full.csv", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from botvrij.eu.") self.errorState = True return None if res['content'] is None: self.error("Received no content from botvrij.eu") self.errorState = True return None self.sf.cachePut("botvrij", res['content']) return self.parseBlacklist(res['content']) def parseBlacklist(self, blacklist): """Parse plaintext blacklist Args: blacklist (str): plaintext blacklist from botvrij.eu Returns: list: list of blacklisted host names """ hosts = list() if not blacklist: return hosts for line in blacklist.split('\n'): if not line: continue if line.startswith('#'): continue host = line.strip().split(",")[0].lower() # Note: Validation with sf.validHost() is too slow to use here # if not self.sf.validHost(host, self.opts['_internettlds']): # continue hosts.append(host) return hosts def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName == "INTERNET_NAME": malicious_type = "MALICIOUS_INTERNET_NAME" blacklist_type = "BLACKLISTED_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": if not self.opts.get('checkaffiliates', False): return malicious_type = "MALICIOUS_AFFILIATE_INTERNET_NAME" blacklist_type = "BLACKLISTED_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": if not self.opts.get('checkcohosts', False): return malicious_type = "MALICIOUS_COHOST" blacklist_type = "BLACKLISTED_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") return self.debug(f"Checking maliciousness of {eventData} ({eventName}) with botvrij.eu") if not self.queryBlacklist(eventData): return url = "https://www.botvrij.eu/data/blocklist/blocklist_full.csv" text = f"botvrij.eu Domain Blocklist [{eventData}]\n{url}" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_botvrij class ================================================ FILE: modules/sfp_builtwith.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_builtwith # Purpose: Query builtwith.com using their API. # # Author: Steve Micallef # # Created: 10/08/2017 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import time from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_builtwith(SpiderFootPlugin): meta = { 'name': "BuiltWith", 'summary': "Query BuiltWith.com's Domain API for information about your target's web technology stack, e-mail addresses and more.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://builtwith.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://api.builtwith.com/", "https://kb.builtwith.com/", "https://builtwith.com/screencast", "https://builtwith.com/faq" ], 'apiKeyInstructions': [ "Visit https://api.builtwith.com/free-api", "Register a free account", "Navigate to https://api.builtwith.com/free-api", "The API key is listed under 'Your API Key'" ], 'favIcon': "https://d28rh9vvmrd65v.cloudfront.net/favicon.ico", 'logo': "https://d28rh9vvmrd65v.cloudfront.net/favicon.ico", 'description': "Build lists of websites from our database of 38,701+ web technologies and over a quarter of a billion websites showing " "which sites use shopping carts, analytics, hosting and many more. " "Filter by location, traffic, vertical and more.\n" "Know your prospects platform before you talk to them. " "Improve your conversions with validated market adoption.\n" "Get advanced technology market share information and country based analytics for all web technologies.", } } # Default options opts = { "api_key": "", "maxage": 30 } # Option descriptions optdescs = { "api_key": "Builtwith.com Domain API key.", "maxage": "The maximum age of the data returned, in days, in order to be considered valid." } # Be sure to completely clear any class variables in setup() # or you run the risk of data persisting between scan runs. results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False # Clear / reset any other class member variables here # or you risk them persisting between threads. for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME"] # What events this module produces def producedEvents(self): return ["INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA", "WEBSERVER_TECHNOLOGY", "PHONE_NUMBER", "DOMAIN_NAME", "CO_HOSTED_SITE", "IP_ADDRESS", "WEB_ANALYTICS_ID"] def queryRelationships(self, t): url = f"https://api.builtwith.com/rv1/api.json?LOOKUP={t}&KEY={self.opts['api_key']}" res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot") if res['code'] == "404": return None if not res['content']: return None try: return json.loads(res['content'])['Relationships'] except Exception as e: self.error(f"Error processing JSON response from builtwith.com: {e}") return None def queryDomainInfo(self, t): url = f"https://api.builtwith.com/rv1/api.json?LOOKUP={t}&KEY={self.opts['api_key']}" res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot") if res['code'] == "404": return None if not res['content']: return None try: return json.loads(res['content'])['Results'][0] except Exception as e: self.error(f"Error processing JSON response from builtwith.com: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "": self.error("You enabled sfp_builtwith but did not set an API key!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True data = self.queryDomainInfo(eventData) if data is not None: if "Meta" in data: if data['Meta'].get("Names", []): for nb in data['Meta']['Names']: e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + nb['Name'], self.__name__, event) self.notifyListeners(e) if nb.get('Email', None): if SpiderFootHelpers.validEmail(nb['Email']): if nb['Email'].split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" e = SpiderFootEvent(evttype, nb['Email'], self.__name__, event) self.notifyListeners(e) if data['Meta'].get("Emails", []): for email in data['Meta']['Emails']: if SpiderFootHelpers.validEmail(email): if email.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" e = SpiderFootEvent(evttype, email, self.__name__, event) self.notifyListeners(e) if data['Meta'].get("Telephones", []): for phone in data['Meta']['Telephones']: phone = phone.replace("-", "").replace("(", "").replace(")", "").replace(" ", "") e = SpiderFootEvent("PHONE_NUMBER", phone, self.__name__, event) self.notifyListeners(e) if "Paths" in data.get("Result", []): for p in data["Result"]['Paths']: if p.get("SubDomain", ""): h = p["SubDomain"] + "." + eventData ev = SpiderFootEvent("INTERNET_NAME", h, self.__name__, event) self.notifyListeners(ev) if self.sf.isDomain(h, self.opts['_internettlds']): ev = SpiderFootEvent("DOMAIN_NAME", h, self.__name__, event) self.notifyListeners(ev) else: ev = None # If we have a subdomain, let's get its tech info # and associate it with the subdomain event. for t in p.get("Technologies", []): if ev: src = ev else: src = event agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage']) if t.get("LastDetected", 0) < agelimit: self.debug("Data found too old, skipping.") continue e = SpiderFootEvent("WEBSERVER_TECHNOLOGY", t["Name"], self.__name__, src) self.notifyListeners(e) data = self.queryRelationships(eventData) if data is None: return agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage']) for r in data: if "Domain" not in r or "Identifiers" not in r: self.debug("Data returned not in the format requested.") continue if r['Domain'] != eventData: self.debug("Data returned doesn't match data requested, skipping.") continue for i in r['Identifiers']: if "Last" not in i or "Type" not in i or "Value" not in i: self.debug("Data returned not in the format requested.") continue if i['Last'] < agelimit: self.debug("Data found too old, skipping.") continue evttype = None # Related through shared IP if i['Type'] == "ip": if self.sf.validIP(i['Value']): val = i['Value'] evttype = "IP_ADDRESS" else: val = i['Value'].strip(".") if self.getTarget.matches(val): evttype = "INTERNET_NAME" else: evttype = "CO_HOSTED_SITE" # Create the name/co-host e = SpiderFootEvent(evttype, val, self.__name__, event) self.notifyListeners(e) continue # Related through shared analytics ID txt = i['Type'] + ": " + str(i['Value']) e = SpiderFootEvent("WEB_ANALYTICS_ID", txt, self.__name__, event) self.notifyListeners(e) if i['Matches']: for m in i['Matches']: if "Domain" not in m: continue evt = SpiderFootEvent("AFFILIATE_INTERNET_NAME", m['Domain'], self.__name__, e) self.notifyListeners(evt) if self.sf.isDomain(m['Domain'], self.opts['_internettlds']): evt = SpiderFootEvent("AFFILIATE_DOMAIN_NAME", m['Domain'], self.__name__, e) self.notifyListeners(evt) # End of sfp_builtwith class ================================================ FILE: modules/sfp_c99.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_c99 # Purpose: SpiderFoot plug-in that queries c99 API # # Author: Filip Aleksić # # Created: 2020-08-27 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_c99(SpiderFootPlugin): meta = { "name": "C99", "summary": "Queries the C99 API which offers various data (geo location, proxy detection, phone lookup, etc).", 'flags': ["apikey"], "useCases": ["Footprint", "Passive", "Investigate"], "categories": ["Search Engines"], "dataSource": { "website": "https://api.c99.nl/", "model": "COMMERCIAL_ONLY", "references": ["https://api.c99.nl/api_overview", "https://api.c99.nl/faq"], "apiKeyInstructions": [ "Visit https://api.c99.nl", "Click shop in the top navigation or go to https://api.c99.nl/dashboard/shop", "Click purchase key on option 'C99.NL API KEY' (you can also purchase a 1 year key)", "You will receive your API key by email.", ], "favIcon": "https://api.c99.nl/favicon.ico", "logo": "https://api.c99.nl/assets/images/logo.png", "description": "C99 API service is versatile source of information. " "They offer over 57 different APIs of which 10 are integrated in this module. " "APIs that are integrated are subdomain finder, phone lookup, Skype resolver, " "IP to Skype, firewall technology WAF detector, domain history, " "IP to domains, IP geo location, proxy detector.", }, } opts = { "api_key": "", "verify": True, "cohostsamedomain": False, "maxcohost": 100, } optdescs = { "api_key": "C99 API Key.", "verify": "Verify identified domains still resolve to the associated specified IP address.", "maxcohost": "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.", "cohostsamedomain": "Treat co-hosted sites on the same target domain as co-hosting?", } results = None errorState = False cohostcount = 0 def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.cohostcount = 0 for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "DOMAIN_NAME", "PHONE_NUMBER", "IP_ADDRESS", "USERNAME", "EMAILADDR", ] def producedEvents(self): return [ "RAW_RIR_DATA", "GEOINFO", "INTERNET_NAME", "INTERNET_NAME_UNRESOLVED", "PROVIDER_TELCO", "PHYSICAL_ADDRESS", "PHYSICAL_COORDINATES", "PROVIDER_DNS", "IP_ADDRESS", "USERNAME", "ACCOUNT_EXTERNAL_OWNED", "WEBSERVER_TECHNOLOGY", "PROVIDER_HOSTING", "CO_HOSTED_SITE" ] def query(self, path, queryParam, queryData): res = self.sf.fetchUrl( f"https://api.c99.nl/{path}?key={self.opts['api_key']}&{queryParam}={queryData}&json", timeout=self.opts["_fetchtimeout"], useragent="SpiderFoot", ) if res["code"] == "429": self.error("Reaching rate limit on C99 API") self.errorState = True return None if res["code"] == 400: self.error("Invalid request or API key on C99 API") self.errorState = True return None if res["content"] is None: self.info(f"No C99 info found for {queryData}") return None try: info = json.loads(res["content"]) except Exception as e: self.errorState = True self.error(f"Error processing response from C99: {e}") return None if not info.get('success', False): return None return info def emitRawRirData(self, data, event): evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) def emitPhoneData(self, phoneData, event): provider = phoneData.get("provider") carrier = phoneData.get("carrier") city = phoneData.get("city") countryName = phoneData.get("country_name") region = phoneData.get("region") found = False if provider or carrier: evt = SpiderFootEvent( "PROVIDER_TELCO", f"Provider: {provider}, Carrier: {carrier}", self.__name__, event, ) self.notifyListeners(evt) found = True if city or countryName or region: evt = SpiderFootEvent( "PHYSICAL_ADDRESS", f"Country: {countryName}, Region: {region}, City: {city}", self.__name__, event, ) self.notifyListeners(evt) found = True if found: self.emitRawRirData(phoneData, event) def emitSubDomainData(self, subDomainData, event): found = False for subDomainElem in subDomainData: if self.checkForStop(): return subDomain = subDomainElem.get("subdomain", "").strip() if subDomain: self.emitHostname(subDomain, event) found = True if found: self.emitRawRirData(subDomainData, event) def emitDomainHistoryData(self, domainHistoryData, event): found = False for domainHistoryElem in domainHistoryData: if self.checkForStop(): return ip = domainHistoryElem.get("ip_address") if self.sf.validIP(ip): evt = SpiderFootEvent( "IP_ADDRESS", ip, self.__name__, event, ) self.notifyListeners(evt) found = True if found: self.emitRawRirData(domainHistoryData, event) def emitIpToSkypeData(self, data, event): skype = data.get("skype") if skype: evt = SpiderFootEvent( "ACCOUNT_EXTERNAL_OWNED", f"Skype [{skype}]", self.__name__, event, ) self.notifyListeners(evt) evt = SpiderFootEvent( "USERNAME", skype, self.__name__, event, ) self.notifyListeners(evt) self.emitRawRirData(data, event) def emitIpToDomainsData(self, data, event): domains = data.get("domains") found = False if isinstance(domains, list): for domain in domains: if self.checkForStop(): return domain = domain.strip() if domain: self.emitHostname(domain, event) found = True if found: self.emitRawRirData(data, event) def emitProxyDetectionData(self, data, event): isProxy = data.get("proxy") if isProxy: evt = SpiderFootEvent( "WEBSERVER_TECHNOLOGY", f"Server is proxy: {isProxy}", self.__name__, event, ) self.notifyListeners(evt) self.emitRawRirData(data, event) def emitGeoIPData(self, data, event): found = False hostName = data.get("hostname", "").strip() if hostName: self.emitHostname(hostName, event) found = True record = data.get("records") if record: country = record.get("country_name") region = record["region"].get("name") if record.get("region") else None city = record.get("city") postalCode = record.get("postal_code") latitude = record.get("latitude") longitude = record.get("longitude") provider = record.get("isp") if provider: evt = SpiderFootEvent( "PROVIDER_HOSTING", provider, self.__name__, event, ) self.notifyListeners(evt) found = True if latitude and longitude: evt = SpiderFootEvent( "PHYSICAL_COORDINATES", f"{latitude}, {longitude}", self.__name__, event, ) self.notifyListeners(evt) found = True if region or country or city or postalCode: evt = SpiderFootEvent( "GEOINFO", f"Country: {country}, Region: {region}, City: {city}, Postal code: {postalCode}", self.__name__, event, ) self.notifyListeners(evt) found = True if found: self.emitRawRirData(data, event) def emitSkypeResolverData(self, data, event): ip = data.get("ip") ips = data.get("ips") found = False if ip and ip not in ips: evt = SpiderFootEvent( "IP_ADDRESS", ip, self.__name__, event, ) self.notifyListeners(evt) found = True if isinstance(ips, list): found = True for ipElem in ips: if self.checkForStop(): return evt = SpiderFootEvent( "IP_ADDRESS", ipElem.strip(), self.__name__, event, ) self.notifyListeners(evt) if found: self.emitRawRirData(data, event) def emitWafDetectorData(self, data, event): firewall = data.get("result") if firewall: evt = SpiderFootEvent( "WEBSERVER_TECHNOLOGY", f"Firewall detected: {firewall}", self.__name__, event, ) self.notifyListeners(evt) self.emitRawRirData(data, event) def emitHostname(self, data, event): if not self.sf.validHost(data, self.opts['_internettlds']): return if self.opts["verify"] and not self.sf.resolveHost(data) and not self.sf.resolveHost6(data): self.debug(f"Host {data} could not be resolved.") if self.getTarget().matches(data): evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", data, self.__name__, event) self.notifyListeners(evt) return if self.getTarget().matches(data): evt = SpiderFootEvent('INTERNET_NAME', data, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(data, self.opts['_internettlds']): evt = SpiderFootEvent('DOMAIN_NAME', data, self.__name__, event) self.notifyListeners(evt) return if self.cohostcount < self.opts['maxcohost']: if self.opts["verify"] and not self.sf.validateIP(data, event.data): self.debug("Host no longer resolves to our IP.") return if not self.opts["cohostsamedomain"]: if self.getTarget().matches(data, includeParents=True): self.debug( f"Skipping {data} because it is on the same domain." ) return if self.cohostcount < self.opts["maxcohost"]: evt = SpiderFootEvent("CO_HOSTED_SITE", data, self.__name__, event) self.notifyListeners(evt) self.cohostcount += 1 def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data # Once we are in this state, return immediately. if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.error("You enabled sfp_c99, but did not set an API key!") self.errorState = True return # Don't look up stuff twice if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == "PHONE_NUMBER": phoneData = self.query("phonelookup", "number", eventData) phoneData = phoneData.get("details") if phoneData else None if phoneData: self.emitPhoneData(phoneData, event) if eventName == "DOMAIN_NAME": subDomainData = self.query("subdomainfinder", "domain", eventData) subDomainData = ( subDomainData.get("subdomains") if subDomainData is not None else None ) if isinstance(subDomainData, list): self.emitSubDomainData(subDomainData, event) domainHistoryData = self.query("domainhistory", "domain", eventData) domainHistoryData = ( domainHistoryData.get("result") if domainHistoryData else None ) if isinstance(domainHistoryData, list): self.emitDomainHistoryData(domainHistoryData, event) wafDetectorData = self.query("firewalldetector", "url", eventData) if wafDetectorData: self.emitWafDetectorData(wafDetectorData, event) if eventName == "IP_ADDRESS": ipToSkypeData = self.query("ip2skype", "ip", eventData) if ipToSkypeData: self.emitIpToSkypeData(ipToSkypeData, event) ipToDomainsData = self.query("ip2domains", "ip", eventData) if ipToDomainsData: self.emitIpToDomainsData(ipToDomainsData, event) proxyDetectionData = self.query("proxydetector", "ip", eventData) if proxyDetectionData: self.emitProxyDetectionData(proxyDetectionData, event) geoIPData = self.query("geoip", "host", eventData) if geoIPData: self.emitGeoIPData(geoIPData, event) if eventName == "USERNAME": skypeResolverData = self.query("skyperesolver", "username", eventData) if skypeResolverData: self.emitSkypeResolverData(skypeResolverData, event) # End of sfp_c99 class ================================================ FILE: modules/sfp_callername.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_callername # Purpose: SpiderFoot plug-in to search CallerName.com for a phone number # (US only) and retrieve location and reputation information. # # Author: # # Created: 2019-05-28 # Copyright: (c) bcoles 2019 # Licence: MIT # ------------------------------------------------------------------------------- import re import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_callername(SpiderFootPlugin): meta = { 'name': "CallerName", 'summary': "Lookup US phone number location and reputation information.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Real World"], 'dataSource': { 'website': "http://callername.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://callername.com/faq", "https://callername.com/stats" ], 'favIcon': "http://static.callername.com/favicon.ico", 'logo': "http://static.callername.com/img/logo.min.png", 'description': "CallerName is a free, reverse phone lookup service for both cell and landline numbers. " "It relies on a database of white pages and business pages taken from public sources. " "The easy-to-use and streamlined interface allow users to look up the caller ID information of any number quickly. " "Just type the unknown number into the search bar to start. " "You need not pay nor register to use this 100% free service.", } } # Default options opts = { } # Option descriptions optdescs = { } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['PHONE_NUMBER'] # What events this module produces def producedEvents(self): return ['GEOINFO', 'MALICIOUS_PHONE_NUMBER'] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") # Only US numbers are supported (+1) if not eventData.startswith('+1'): self.debug('Unsupported phone number: ' + eventData) return # Strip country code (+1) and formatting number = eventData.lstrip('+1').strip('(').strip(')').strip('-').strip(' ') if not number.isdigit(): self.debug('Invalid phone number: ' + number) return # Query CallerName.com for the specified phone number url = f"https://callername.com/{number}" res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) time.sleep(1) if res['content'] is None: self.debug('No response from CallerName.com') return if res['code'] != '200': self.debug('No phone information found for ' + eventData) return location_match = re.findall(r'

.*?

(.+?)

', str(res['content']), re.MULTILINE | re.DOTALL) if location_match: location = location_match[0] if len(location) < 5 or len(location) > 100: self.debug("Skipping likely invalid location.") else: evt = SpiderFootEvent('GEOINFO', location, self.__name__, event) self.notifyListeners(evt) rep_good_match = re.findall(r'>SAFE.*?>(\d+) votes?<', str(res['content'])) rep_bad_match = re.findall(r'>UNSAFE.*?>(\d+) votes?<', str(res['content'])) if rep_good_match and rep_bad_match: good_votes = int(rep_good_match[0]) bad_votes = int(rep_bad_match[0]) if bad_votes > good_votes: text = f"CallerName [{eventData}]\n{url}" evt = SpiderFootEvent('MALICIOUS_PHONE_NUMBER', text, self.__name__, event) self.notifyListeners(evt) # End of sfp_callername class ================================================ FILE: modules/sfp_censys.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_censys # Purpose: Query Censys.io API # # Author: Steve Micallef # # Created: 01/02/2017 # Copyright: (c) Steve Micallef 2017 # Licence: MIT # ------------------------------------------------------------------------------- import base64 import json import time import urllib.error import urllib.parse import urllib.request from datetime import datetime from netaddr import IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_censys(SpiderFootPlugin): meta = { 'name': "Censys", 'summary': "Obtain host information from Censys.io.", 'flags': ["apikey"], 'useCases': ["Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://censys.io/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://search.censys.io/api", "https://search.censys.io/search/language", "https://github.com/censys/censys-postman/blob/main/Censys_Search.postman_collection.json", ], 'apiKeyInstructions': [ "Visit https://censys.io/", "Register a free account", "Navigate to https://censys.io/account", "Click on 'API'", "The API key combination is listed under 'API ID' and 'Secret'" ], 'favIcon': "https://censys.io/assets/favicon.png", 'logo': "https://censys.io/assets/logo.png", 'description': "Discover exposures and other common entry points for attackers.\n" "Censys scans the entire internet constantly, including obscure ports. " "We use a combination of banner grabs and deep protocol handshakes " "to provide industry-leading visibility and an accurate depiction of what is live on the internet.", } } opts = { "censys_api_key_uid": "", "censys_api_key_secret": "", 'delay': 3, 'netblocklookup': True, 'maxnetblock': 24, 'maxv6netblock': 120, "age_limit_days": 90, } optdescs = { "censys_api_key_uid": "Censys.io API UID.", "censys_api_key_secret": "Censys.io API Secret.", 'delay': 'Delay between requests, in seconds.', 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?", 'maxnetblock': "If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'maxv6netblock': "If looking up owned netblocks, the maximum IPv6 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", "age_limit_days": "Ignore any records older than this many days. 0 = unlimited.", } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "IPV6_ADDRESS", "NETBLOCK_OWNER", "NETBLOCKV6_OWNER", ] def producedEvents(self): return [ "BGP_AS_MEMBER", "UDP_PORT_OPEN", "TCP_PORT_OPEN", "TCP_PORT_OPEN_BANNER", "OPERATING_SYSTEM", "SOFTWARE_USED", "WEBSERVER_HTTPHEADERS", "NETBLOCK_MEMBER", "NETBLOCKV6_MEMBER", "GEOINFO", "RAW_RIR_DATA" ] def queryHosts(self, qry): secret = self.opts['censys_api_key_uid'] + ':' + self.opts['censys_api_key_secret'] auth = base64.b64encode(secret.encode('utf-8')).decode('utf-8') headers = { 'Authorization': f"Basic {auth}" } res = self.sf.fetchUrl( f"https://search.censys.io/api/v2/hosts/{qry}", timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=headers ) # API rate limit: 0.4 actions/second (120.0 per 5 minute interval) time.sleep(self.opts['delay']) return self.parseApiResponse(res) def queryHostsSearch(self, qry): secret = self.opts['censys_api_key_uid'] + ':' + self.opts['censys_api_key_secret'] auth = base64.b64encode(secret.encode('utf-8')).decode('utf-8') headers = { 'Authorization': f"Basic {auth}" } params = urllib.parse.urlencode({ 'q': qry, }) res = self.sf.fetchUrl( f"https://search.censys.io/api/v2/hosts/search/?{params}", timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=headers ) # API rate limit: 0.4 actions/second (120.0 per 5 minute interval) time.sleep(self.opts['delay']) return self.parseApiResponse(res) def parseApiResponse(self, res: dict): if not res: self.error("No response from Censys.io.") return None if res['code'] == "400": self.error("Invalid request.") return None if res['code'] == "404": self.info('Censys.io returned no results') return None if res['code'] == "403": self.error("Invalid API key.") self.errorState = True return None if res['code'] == "429": self.error("Request rate limit exceeded.") self.errorState = True return None # Catch all non-200 status codes, and presume something went wrong if res['code'] != '200': self.error(f"Unexpected HTTP response code {res['code']} from Censys API.") self.errorState = True return None if res['content'] is None: self.info('Censys.io returned no results') return None try: data = json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from Censys.io: {e}") return None error_type = data.get('error_type') if error_type: self.error(f"Censys returned an unexpected error: {error_type}") return None return data def handleEvent(self, event): if self.errorState: return eventName = event.eventType self.debug(f"Received event, {eventName}, from {event.module}") if self.opts['censys_api_key_uid'] == "" or self.opts['censys_api_key_secret'] == "": self.error(f"You enabled {self.__class__.__name__} but did not set an API uid/secret!") self.errorState = True return eventData = event.data if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']: if not self.opts['netblocklookup']: return if eventName == 'NETBLOCKV6_OWNER': max_netblock = self.opts['maxv6netblock'] else: max_netblock = self.opts['maxnetblock'] if IPNetwork(eventData).prefixlen < max_netblock: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}") return qrylist = list() if eventName.startswith("NETBLOCK"): for ipaddr in IPNetwork(eventData): qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) for addr in qrylist: if self.checkForStop(): return data = self.queryHosts(addr) if not data: continue rec = data.get("result") if not rec: self.info(f"Censys.io returned no results for {addr}") continue self.debug(f"Found results for {addr} in Censys.io") # For netblocks, we need to create the associated IP address event first. if eventName == 'NETBLOCK_OWNER': pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) elif eventName == 'NETBLOCKV6_OWNER': pevent = SpiderFootEvent("IPV6_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) else: pevent = event e = SpiderFootEvent("RAW_RIR_DATA", json.dumps(rec), self.__name__, pevent) self.notifyListeners(e) try: # Date format: 2021-09-22T16:46:47.623Z created_dt = datetime.strptime(rec.get('last_updated_at', "1970-01-01T00:00:00.000Z"), '%Y-%m-%dT%H:%M:%S.%fZ') created_ts = int(time.mktime(created_dt.timetuple())) age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days']) if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts: self.debug(f"Record found but too old ({created_dt}), skipping.") continue except Exception as e: self.error(f"Error encountered processing last_updated_at record for {addr}: {e}") try: location = rec.get('location') if location: geoinfo = ', '.join( [ _f for _f in [ location.get('city'), location.get('province'), location.get('postal_code'), location.get('country'), location.get('continent'), ] if _f ] ) if geoinfo: e = SpiderFootEvent("GEOINFO", geoinfo, self.__name__, pevent) self.notifyListeners(e) except Exception as e: self.error(f"Error encountered processing location record for {addr}: {e}") try: services = rec.get('services') if services: softwares = list() tcp_banners = list() for service in services: port = service.get('port') if port: transport_protocol = service.get('transport_protocol') banner = service.get('banner') if transport_protocol == "UDP": evt = SpiderFootEvent("UDP_PORT_OPEN", f"{addr}:{port}", self.__name__, pevent) self.notifyListeners(evt) elif transport_protocol == "TCP": evt = SpiderFootEvent("TCP_PORT_OPEN", f"{addr}:{port}", self.__name__, pevent) self.notifyListeners(evt) if banner: tcp_banners.append(banner) software = service.get('software', list()) if software: for sw in software: s = ' '.join( filter( None, [ sw.get('vendor'), sw.get('product'), sw.get('version') ] ) ) if s: softwares.append(s) http = service.get('http') if http: response = http.get('response') if response: headers = response.get('headers') if headers: e = SpiderFootEvent( "WEBSERVER_HTTPHEADERS", json.dumps(headers, ensure_ascii=False), self.__name__, pevent ) e.actualSource = addr self.notifyListeners(e) for software in set(softwares): evt = SpiderFootEvent("SOFTWARE_USED", software, self.__name__, pevent) self.notifyListeners(evt) for banner in set(tcp_banners): evt = SpiderFootEvent("TCP_PORT_OPEN_BANNER", str(banner), self.__name__, pevent) self.notifyListeners(evt) except Exception as e: self.error(f"Error encountered processing services record for {addr}: {e}") try: autonomous_system = rec.get('autonomous_system') if autonomous_system: asn = autonomous_system.get('asn') if asn: e = SpiderFootEvent("BGP_AS_MEMBER", str(asn), self.__name__, pevent) self.notifyListeners(e) bgp_prefix = autonomous_system.get('bgp_prefix') if bgp_prefix and self.sf.validIpNetwork(bgp_prefix): if ':' in bgp_prefix: e = SpiderFootEvent("NETBLOCKV6_MEMBER", str(bgp_prefix), self.__name__, pevent) else: e = SpiderFootEvent("NETBLOCK_MEMBER", str(bgp_prefix), self.__name__, pevent) self.notifyListeners(e) except Exception as e: self.error(f"Error encountered processing autonomous_system record for {addr}: {e}") try: operating_system = rec.get('operating_system') if operating_system: os = ' '.join( filter( None, [ operating_system.get('vendor'), operating_system.get('product'), operating_system.get('version'), operating_system.get('edition') ] ) ) if os: e = SpiderFootEvent("OPERATING_SYSTEM", os, self.__name__, pevent) self.notifyListeners(e) except Exception as e: self.error(f"Error encountered processing operating_system record for {addr}: {e}") # End of sfp_censys class ================================================ FILE: modules/sfp_certspotter.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_certspotter # Purpose: Gather information about SSL certificates from SSLMate CertSpotter API. # # Author: # # Created: 2021-08-15 # Copyright: (c) bcoles # Licence: MIT # ------------------------------------------------------------------------------- import base64 import json import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_certspotter(SpiderFootPlugin): meta = { 'name': "CertSpotter", 'summary': "Gather information about SSL certificates from SSLMate CertSpotter API.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Crawling and Scanning"], 'dataSource': { 'website': "https://sslmate.com/certspotter/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://sslmate.com/help/reference/ct_search_api_v1" ], 'apiKeyInstructions': [ "Visit https://sslmate.com/signup?for=ct_search_api", "Register a new account with an email", "Navigate to https://sslmate.com/account/", "The API key is listed under 'API Credentials'", ], "favIcon": "https://sslmate.com/assets/@faafe50b54dfb91476c01374043f217c.png", "logo": "https://sslmate.com/assets/@995de4b3fc64525a0c960b570432bcaf.png", "description": "Cert Spotter monitors your domains for expiring, unauthorized, " "and invalid SSL certificates, so you can act before an incident, not after." } } # Default options opts = { 'api_key': '', 'verify': True, 'max_pages': 20, 'certexpiringdays': 30 } # Option descriptions optdescs = { 'api_key': 'CertSpotter API key.', 'verify': "Verify certificate subject alternative names resolve.", 'max_pages': "Maximum number of pages of results to fetch.", 'certexpiringdays': 'Number of days in the future a certificate expires to consider it as expiring.' } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in userOpts.keys(): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['DOMAIN_NAME'] # What events this module produces def producedEvents(self): return [ 'INTERNET_NAME', 'INTERNET_NAME_UNRESOLVED', 'DOMAIN_NAME', 'CO_HOSTED_SITE', 'CO_HOSTED_SITE_DOMAIN', 'SSL_CERTIFICATE_ISSUED', 'SSL_CERTIFICATE_ISSUER', 'SSL_CERTIFICATE_MISMATCH', 'SSL_CERTIFICATE_EXPIRED', 'SSL_CERTIFICATE_EXPIRING', 'SSL_CERTIFICATE_RAW', 'RAW_RIR_DATA' ] # Query CertSpotter issuances API endpoint def queryIssuances(self, domain, after=None): params = { 'domain': domain.encode('raw_unicode_escape').decode("ascii", errors='replace'), 'include_subdomains': 'true', 'match_wildcards': 'true', 'after': (after or '') } expand = '&expand='.join(['dns_names', 'issuer', 'cert']) headers = { 'Accept': 'application/json', 'Authorization': "Basic " + base64.b64encode(f"{self.opts['api_key']}:".encode('utf-8')).decode('utf-8') } res = self.sf.fetchUrl( f"https://api.certspotter.com/v1/issuances?{urllib.parse.urlencode(params)}&expand={expand}", headers=headers, timeout=15, useragent=self.opts['_useragent'], ) # Free plan - 1,000 single-hostname queries / hour; 100 full-domain queries / hour time.sleep(1) if res['content'] is None: self.debug('No response from CertSpotter API') return None if res['code'] == '429': self.error("You are being rate-limited by CertSpotter") self.errorState = True return None if res['code'] != '200': self.error(f"Unexpected HTTP response code {res['code']} from CertSpotter") self.errorState = True return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return if self.errorState: return if self.opts['api_key'] == "": self.error(f"You enabled {self.__class__.__name__} but did not set an API key!") self.errorState = True return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") max_pages = int(self.opts['max_pages']) page = 1 last_id = None hosts = list() while page <= max_pages: if self.checkForStop(): break if self.errorState: break data = self.queryIssuances(eventData, last_id) if data is None or len(data) == 0: break page += 1 evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) for result in data: cert_hosts = result.get('dns_names') if cert_hosts: for d in cert_hosts: if d != eventData: hosts.append(d.replace("*.", "")) if result.get('cert') is None: self.debug('Response data contains no certificate data') continue try: rawcert = "-----BEGIN CERTIFICATE-----\n" rawcert += result.get('cert').get('data') rawcert += "\n-----END CERTIFICATE-----\n" cert = self.sf.parseCert(rawcert, eventData, self.opts['certexpiringdays']) except Exception as e: self.info(f"Error parsing certificate: {e}") continue if not cert.get('text'): self.info("Failed to parse the SSL certificate") continue evt = SpiderFootEvent('SSL_CERTIFICATE_RAW', cert['text'], self.__name__, event) self.notifyListeners(evt) if cert.get('issuer'): evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUER', cert['issuer'], self.__name__, event) self.notifyListeners(evt) if cert.get('issued'): evt = SpiderFootEvent('SSL_CERTIFICATE_ISSUED', cert['issued'], self.__name__, event) self.notifyListeners(evt) for san in set(cert.get('altnames', list())): hosts.append(san.replace("*.", "")) if cert.get('expired'): evt = SpiderFootEvent("SSL_CERTIFICATE_EXPIRED", cert.get('expirystr', 'Unknown'), self.__name__, event) self.notifyListeners(evt) continue if cert.get('expiring'): evt = SpiderFootEvent("SSL_CERTIFICATE_EXPIRING", cert.get('expirystr', 'Unknown'), self.__name__, event) self.notifyListeners(evt) continue # "To retrieve additional issuances, take the id field of the last issuance and pass it to the issuances endpoint in the after parameter" last_id = data[-1].get('id') if last_id is None: break if not hosts: return if self.opts['verify']: self.info(f"Resolving {len(set(hosts))} hostnames ...") for domain in set(hosts): if self.checkForStop(): return if domain in self.results: continue if self.getTarget().matches(domain, includeChildren=True, includeParents=True): evt_type = 'INTERNET_NAME' if self.opts['verify'] and not self.sf.resolveHost(domain) and not self.sf.resolveHost6(domain): self.debug(f"Host {domain} could not be resolved") evt_type += '_UNRESOLVED' else: evt_type = 'CO_HOSTED_SITE' evt = SpiderFootEvent(evt_type, domain, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(domain, self.opts['_internettlds']): if evt_type == 'CO_HOSTED_SITE': evt = SpiderFootEvent('CO_HOSTED_SITE_DOMAIN', domain, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent('DOMAIN_NAME', domain, self.__name__, event) self.notifyListeners(evt) # End of sfp_certspotter class ================================================ FILE: modules/sfp_cinsscore.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_cinsscore # Purpose: Checks if an IP address is malicious according to the CINS Army list. # # Author: steve@binarypool.com # # Created: 13/05/2018 # Copyright: (c) Steve Micallef, 2018 # Licence: MIT # ------------------------------------------------------------------------------- from netaddr import IPAddress, IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_cinsscore(SpiderFootPlugin): meta = { 'name': "CINS Army List", 'summary': "Check if a netblock or IP address is malicious according to Collective Intelligence Network Security (CINS) Army list.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://cinsscore.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'favIcon': 'https://cinsscore.com/media/images/fav-icon.png', 'logo': 'https://cinsscore.com/media/images/logo-small-grey-inset.png', 'description': "Leveraging data from our network of Sentinel " "devices and other trusted InfoSec sources, CINS is a " "Threat Intelligence database that provides an accurate " "and timely score for any IP address in the world." "The CINS Army list is a subset of the CINS Active Threat Intelligence ruleset.", } } opts = { 'checkaffiliates': True, 'cacheperiod': 18, 'checknetblocks': True, 'checksubnets': True } optdescs = { 'checkaffiliates': "Apply checks to affiliate IP addresses?", 'cacheperiod': "Hours to cache list data before re-fetching.", 'checknetblocks': "Report if any malicious IPs are found within owned netblocks?", 'checksubnets': "Check if any malicious IPs are found within the same subnet of the target?" } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "AFFILIATE_IPADDR", "NETBLOCK_MEMBER", "NETBLOCK_OWNER", ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "BLACKLISTED_SUBNET", "BLACKLISTED_NETBLOCK", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_SUBNET", "MALICIOUS_NETBLOCK", ] def query(self, qry, targetType): cid = "_cinsscore" url = "https://cinsscore.com/list/ci-badguys.txt" data = dict() data["content"] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0)) if data["content"] is None: data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if data["code"] != "200": self.error(f"Unable to fetch {url}") self.errorState = True return None if data["content"] is None: self.error(f"Unable to fetch {url}") self.errorState = True return None self.sf.cachePut("sfmal_" + cid, data['content']) for line in data["content"].split('\n'): ip = line.strip().lower() if targetType == "netblock": try: if IPAddress(ip) in IPNetwork(qry): self.debug(f"{ip} found within netblock/subnet {qry} in cinsscore.com list.") return url except Exception as e: self.debug(f"Error encountered parsing: {e}") continue if targetType == "ip": if qry.lower() == ip: self.debug(f"{qry} found in cinsscore.com list.") return url return None def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName == 'IP_ADDRESS': targetType = 'ip' malicious_type = "MALICIOUS_IPADDR" blacklist_type = "BLACKLISTED_IPADDR" elif eventName == 'AFFILIATE_IPADDR': if not self.opts.get('checkaffiliates', False): return targetType = 'ip' malicious_type = "MALICIOUS_AFFILIATE_IPADDR" blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" elif eventName == 'NETBLOCK_OWNER': if not self.opts.get('checknetblocks', False): return targetType = 'netblock' malicious_type = "MALICIOUS_NETBLOCK" blacklist_type = "BLACKLISTED_NETBLOCK" elif eventName == 'NETBLOCK_MEMBER': if not self.opts.get('checksubnets', False): return targetType = 'netblock' malicious_type = "MALICIOUS_SUBNET" blacklist_type = "BLACKLISTED_SUBNET" else: self.debug(f"Unexpected event type {eventName}, skipping") return self.debug(f"Checking maliciousness of {eventData} with cinsscore.com") url = self.query(eventData, targetType) if not url: return text = f"cinsscore.com [{eventData}]\n{url}" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_cinsscore class ================================================ FILE: modules/sfp_circllu.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_circllu # Purpose: Query circl.lu using their API # # Author: Steve Micallef # # Created: 16/02/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import base64 import json import re import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_circllu(SpiderFootPlugin): meta = { 'name': "CIRCL.LU", 'summary': "Obtain information from CIRCL.LU's Passive DNS and Passive SSL databases.", 'flags': ["apikey"], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://www.circl.lu/", 'model': "FREE_AUTH_UNLIMITED", 'references': [ "https://www.circl.lu/services/passive-dns/", "https://www.circl.lu/services/passive-ssl/", "https://www.circl.lu/services/", "https://www.circl.lu/pub/", "https://www.circl.lu/projects" ], 'apiKeyInstructions': [ "Visit https://www.circl.lu/contact/", "Contact with email or phone to request access for Passive DNS and Passive SSL API services", "The API access will be provided once approved" ], 'favIcon': "https://www.google.com/s2/favicons?domain=https://www.circl.lu/", 'logo': "https://www.circl.lu/assets/images/circl-logo.png", 'description': "The Computer Incident Response Center Luxembourg (CIRCL) is a government-driven initiative " "designed to gather, review, report and respond to computer security threats and incidents.\n" "CIRCL provides a reliable and trusted point of contact for any users, companies and organizations " "based in Luxembourg, for the handling of attacks and incidents. " "Its team of experts acts like a fire brigade, with the ability to react promptly and " "efficiently whenever threats are suspected, detected or incidents occur.", } } # Default options opts = { "api_key_login": "", "api_key_password": "", "age_limit_days": 0, "verify": True, "cohostsamedomain": False, "maxcohost": 100 } # Option descriptions optdescs = { "api_key_login": "CIRCL.LU login.", "api_key_password": "CIRCL.LU password.", "age_limit_days": "Ignore any Passive DNS records older than this many days. 0 = unlimited.", "verify": "Verify co-hosts are valid by checking if they still resolve to the shared IP.", "cohostsamedomain": "Treat co-hosted sites on the same target domain as co-hosting?", 'maxcohost': "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting." } # Be sure to completely clear any class variables in setup() # or you run the risk of data persisting between scan runs. results = None errorState = False cohostcount = 0 def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.cohostcount = 0 # Clear / reset any other class member variables here # or you risk them persisting between threads. for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["INTERNET_NAME", "NETBLOCK_OWNER", "IP_ADDRESS", "DOMAIN_NAME"] # What events this module produces def producedEvents(self): return ["IP_ADDRESS", "SSL_CERTIFICATE_ISSUED", "CO_HOSTED_SITE"] def query(self, qry, qtype): if self.errorState: return None if qtype == "PDNS": url = "https://www.circl.lu/pdns/query/" + qry else: url = "https://www.circl.lu/v2pssl/query/" + qry secret = self.opts['api_key_login'] + ':' + self.opts['api_key_password'] b64_val = base64.b64encode(secret.encode('utf-8')) headers = { 'Authorization': f"Basic {b64_val.decode('utf-8')}" } # Be more forgiving with the timeout as some queries for subnets can be slow res = self.sf.fetchUrl(url, timeout=30, useragent="SpiderFoot", headers=headers) if res['code'] not in ["200", "201"]: self.error("CIRCL.LU access seems to have been rejected or you have exceeded usage limits.") self.errorState = True return None if res['content'] is None: self.info("No CIRCL.LU info found for " + qry) return None return res['content'] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data ret = None if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") # Ignore messages from myself if srcModuleName == "sfp_circllu": self.debug("Ignoring " + eventName + ", from self.") return if self.opts['api_key_login'] == "" or self.opts['api_key_password'] == "": self.error("You enabled sfp_circllu but did not set an credentials!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName in ['IP_ADDRESS', 'NETBLOCK_OWNER']: # CIRCL.LU limit the maximum subnet size to 23 # http://circl.lu/services/passive-ssl/ if "/" in eventData: addr, mask = eventData.split("/") if int(mask) < 23: self.debug("Network size bigger than permitted by CIRCL.LU.") else: ret = self.query(eventData, "PSSL") if not ret: self.info("No CIRCL.LU passive SSL data found for " + eventData) else: ret = self.query(eventData, "PSSL") if not ret: self.info("No CIRCL.LU passive SSL data found for " + eventData) if ret: try: # Generate an event for the IP first, and then link the cert # to that event. j = json.loads(ret) for ip in j: ipe = event if ip != eventData: ipe = SpiderFootEvent("IP_ADDRESS", ip, self.__name__, event) self.notifyListeners(ipe) for crt in j[ip]['subjects']: r = re.findall(r".*[\"\'](.+CN=([a-zA-Z0-9\-\*\.])+)[\"\'].*", str(j[ip]['subjects'][crt]), re.IGNORECASE) if r: e = SpiderFootEvent("SSL_CERTIFICATE_ISSUED", r[0][0], self.__name__, ipe) self.notifyListeners(e) except Exception as e: self.error("Invalid response returned from CIRCL.LU: " + str(e)) if eventName in ['IP_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME']: ret = self.query(eventData, "PDNS") if not ret: self.info("No CIRCL.LU passive DNS data found for " + eventData) return # CIRCL.LU doesn't return valid JSON - it's one JSON record per line for line in ret.split("\n"): if len(line) < 2: continue try: rec = json.loads(line) except Exception as e: self.error("Invalid response returned from CIRCL.LU: " + str(e)) continue age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days']) if self.opts['age_limit_days'] > 0 and rec['time_last'] < age_limit_ts: self.debug("Record found but too old, skipping.") continue cohosts = list() if eventName == "IP_ADDRESS": # Record could be pointing to our IP, or from our IP if rec['rrtype'] == "A" and rec['rdata'] == eventData: if not self.getTarget().matches(rec['rrname']): # We found a co-host cohosts.append(rec['rrname']) if eventName in ["INTERNET_NAME", "DOMAIN_NAME"]: # Record could be an A/CNAME of this entity, or something pointing to it if rec['rdata'] == eventData: if not self.getTarget().matches(rec['rrname']): # We found a co-host cohosts.append(rec['rrname']) for co in cohosts: if eventName == "IP_ADDRESS" and (self.opts['verify'] and not self.sf.validateIP(co, eventData)): self.debug("Host no longer resolves to our IP.") continue if not self.opts['cohostsamedomain']: if self.getTarget().matches(co, includeParents=True): self.debug("Skipping " + co + " because it is on the same domain.") continue if self.cohostcount < self.opts['maxcohost']: e = SpiderFootEvent("CO_HOSTED_SITE", co, self.__name__, event) self.notifyListeners(e) self.cohostcount += 1 # End of sfp_circllu class ================================================ FILE: modules/sfp_citadel.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_citadel # Purpose: SpiderFoot plug-in to search Leak-Lookup using their API, # for potential data breaches. # # Author: sn # # Created: 15/08/2017 # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_citadel(SpiderFootPlugin): meta = { 'name': "Leak-Lookup", 'summary': "Searches Leak-Lookup.com's database of breaches.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Leaks, Dumps and Breaches"], 'dataSource': { 'website': "https://leak-lookup.com/", 'model': "FREE_AUTH_UNLIMITED", 'references': [ "https://leak-lookup.com/api", "https://leak-lookup.com/databases" ], 'apiKeyInstructions': [ "Visit https://leak-lookup.com", "Register an account", "Login to your account", "Click on 'Account'", "Click on 'API'", "The API key is listed under 'API Key'" ], 'favIcon': "https://leak-lookup.com/favicon.png", 'logo': "https://leak-lookup.com/favicon.png", 'description': "Leak-Lookup allows you to search across thousands of data breaches " "to stay on top of credentials that may have been compromised in the wild.\n" "The creators came together when they realized they had a vast trove of data " "that could be of great value to pen-testers seeking weaknesses in client passwords " "and those concerned about which of their credentials have been leaked into the wild.\n" "Always looking forward, Leak-Lookup invests all of its profits back into securing the " "latest data breaches and leaks / dumps as they become available, ensuring that " "as well as historical data, Leak-Lookup becomes a field leader in credential monitoring.", } } # Default options opts = { "api_key": "", "timeout": 60 } optdescs = { "api_key": "Leak-Lookup API key. Without this you're limited to the public API.", "timeout": "Custom timeout due to heavy traffic at times." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False self.__dataSource__ = "Leak-Lookup.com" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['EMAILADDR'] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["EMAILADDR_COMPROMISED"] # Query email address # https://leak-lookup.com/api def queryEmail(self, email): apikey = self.opts['api_key'] if not apikey: # Public API key apikey = "3edfb5603418f101926c64ca5dd0e409" params = { 'query': email.encode('raw_unicode_escape').decode("ascii", errors='replace'), 'type': 'email_address', 'key': apikey } res = self.sf.fetchUrl("https://leak-lookup.com/api/search", postData=urllib.parse.urlencode(params), timeout=self.opts['timeout'], useragent=self.opts['_useragent']) if res['code'] == "429": time.sleep(10) return self.queryEmail(email) if res['content'] is None: self.debug('No response from Leak-Lookup.com') return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.errorState: return # Don't look up stuff twice if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True data = self.queryEmail(eventData) if data is None: return error = data.get('error') message = data.get('message') if error == 'true': self.error(f"Error encountered processing {eventData}: {message}") if "MISSING API" in message: self.errorState = True return return if not message: return for site in message: self.info(f"Found Leak-Lookup entry for {eventData}: {site}") evt = SpiderFootEvent("EMAILADDR_COMPROMISED", f"{eventData} [{site}]", self.__name__, event) self.notifyListeners(evt) # End of sfp_citadel class ================================================ FILE: modules/sfp_cleanbrowsing.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_cleanbrowsing # Purpose: SpiderFoot plug-in for looking up whether hosts are blocked by # CleanBrowsing.org DNS content family filters # (185.228.168.168 and 185.228.168.169). # # Author: Steve Micallef # # Created: 30/05/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import dns.resolver from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_cleanbrowsing(SpiderFootPlugin): meta = { 'name': "CleanBrowsing.org", 'summary': "Check if a host would be blocked by CleanBrowsing.org DNS content filters.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://cleanbrowsing.org/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://cleanbrowsing.org/guides/", "https://cleanbrowsing.org/filters/", "https://cleanbrowsing.org/how-it-works", "https://cleanbrowsing.org/web-filtering-for-shools-and-cipa-compliance", "https://cleanbrowsing.org/getting-started" ], 'favIcon': "https://cleanbrowsing.org/favicon-new.ico", 'logo': "https://cleanbrowsing.org/images/logos/CleanBrowsing-logo-large-2019-Orange-II.png", 'description': "You get to decide what type of content is allowed in your home or network via our " "DNS-based content filtering service. Parents can protect their kids from adult content, " "schools can be CIPA compliant and businesses can block malicious domains and " "gain visibility into their network.\n" "CleanBrowsing is a DNS-based content filtering service that offers a safe way to browse the web without surprises. " "It intercepts domain requests and filter sites that should be blocked, based on your requirements. " "Our free family filter, for example, blocks adult content, while still allowing Google, " "Youtube, Bing, DuckDuckGo and the rest of the web to load safely.", } } opts = { } optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "CO_HOSTED_SITE" ] def producedEvents(self): return [ "BLACKLISTED_INTERNET_NAME", "BLACKLISTED_AFFILIATE_INTERNET_NAME", "BLACKLISTED_COHOST", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_COHOST", ] def queryFamilyDNS(self, qaddr): res = dns.resolver.Resolver() res.nameservers = ["185.228.168.168", "185.228.168.169"] try: addrs = res.resolve(qaddr) self.debug(f"Addresses returned: {addrs}") except Exception: self.debug(f"Unable to resolve {qaddr}") return False if addrs: return True return False def queryAdultDNS(self, qaddr): res = dns.resolver.Resolver() res.nameservers = ["185.228.168.10", "185.228.169.11"] try: addrs = res.resolve(qaddr) self.debug(f"Addresses returned: {addrs}") except Exception: self.debug(f"Unable to resolve {qaddr}") return False if addrs: return True return False def querySecurityDNS(self, qaddr): res = dns.resolver.Resolver() res.nameservers = ["185.228.168.9", "185.228.169.9"] try: addrs = res.resolve(qaddr) self.debug(f"Addresses returned: {addrs}") except Exception: self.debug(f"Unable to resolve {qaddr}") return False if addrs: return True return False def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: return self.results[eventData] = True if eventName == "INTERNET_NAME": malicious_type = "MALICIOUS_INTERNET_NAME" blacklist_type = "BLACKLISTED_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": malicious_type = "MALICIOUS_AFFILIATE_INTERNET_NAME" blacklist_type = "BLACKLISTED_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": malicious_type = "MALICIOUS_COHOST" blacklist_type = "BLACKLISTED_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") return # Check that it resolves first, as it becomes a valid # malicious host only if NOT resolved by CleanBrowsing DNS. if not self.sf.resolveHost(eventData) and not self.sf.resolveHost6(eventData): return family = self.queryFamilyDNS(eventData) adult = self.queryAdultDNS(eventData) security = self.querySecurityDNS(eventData) # Host was found, not blocked if family and adult and security: return self.debug(f"{eventData} was blocked by CleanBrowsing DNS") if not security: evt = SpiderFootEvent( blacklist_type, f"CleanBrowsing DNS - Security [{eventData}]", self.__name__, event ) self.notifyListeners(evt) evt = SpiderFootEvent( malicious_type, f"CleanBrowsing DNS - Security [{eventData}]", self.__name__, event ) self.notifyListeners(evt) elif not adult: evt = SpiderFootEvent( blacklist_type, f"CleanBrowsing DNS - Adult [{eventData}]", self.__name__, event ) self.notifyListeners(evt) elif not family: evt = SpiderFootEvent( blacklist_type, f"CleanBrowsing DNS - Family [{eventData}]", self.__name__, event ) self.notifyListeners(evt) # End of sfp_cleanbrowsing class ================================================ FILE: modules/sfp_cleantalk.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_cleantalk # Purpose: Checks if a netblock or IP address is on CleanTalk.org's spam IP list. # # Author: steve@binarypool.com # # Created: 05/08/2018 # Copyright: (c) Steve Micallef, 2018 # Licence: MIT # ------------------------------------------------------------------------------- from netaddr import IPAddress, IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_cleantalk(SpiderFootPlugin): meta = { 'name': "CleanTalk Spam List", 'summary': "Check if a netblock or IP address is on CleanTalk.org's spam IP list.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://cleantalk.org", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://cleantalk.org/help", "https://cleantalk.org/help/introduction", "https://cleantalk.org/help/api-spam-check", "https://cleantalk.org/wordpress-security-malware-firewall", "https://cleantalk.org/price-anti-spam", "https://cleantalk.org/ssl-certificates/cheap-positivessl-certificate", "https://cleantalk.org/email-checker", "https://cleantalk.org/blacklists" ], 'favIcon': "https://cleantalk.org/favicons/favicon-16x16.png", 'logo': "https://cleantalk.org/favicons/favicon-16x16.png", 'description': "CleanTalk is a Cloud-Based spam filtering service that allows you to protect your website from spam. " "CleanTalk provides spam protection that invisible to visitors " "without using captcha or other methods when visitors have to prove that they are real people.\n" "CleanTalk provides cloud anti-spam solutions for CMS and we developed plugins for the most of popular " "CMS: WordPress anti-spam plugin, Joomla anti-spam plugin, Drupal and etc. " "With our simple cloud spam checker, you can be sure your website is protected from spam bots, spam comments, and users.", } } opts = { 'checkaffiliates': True, 'cacheperiod': 18, 'checknetblocks': True, 'checksubnets': True } optdescs = { 'checkaffiliates': "Apply checks to affiliate IP addresses?", 'cacheperiod': "Hours to cache list data before re-fetching.", 'checknetblocks': "Report if any malicious IPs are found within owned netblocks?", 'checksubnets': "Check if any malicious IPs are found within the same subnet of the target?" } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ 'IP_ADDRESS', 'AFFILIATE_IPADDR', 'NETBLOCK_OWNER', 'NETBLOCK_MEMBER' ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "BLACKLISTED_SUBNET", "BLACKLISTED_NETBLOCK", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK", "MALICIOUS_SUBNET", ] def query(self, qry, targetType): cid = "_cleantalk" url = "https://iplists.firehol.org/files/cleantalk_7d.ipset" data = dict() data["content"] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0)) if data["content"] is None: data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if data["code"] != "200": self.error(f"Unable to fetch {url}") self.errorState = True return None if data["content"] is None: self.error(f"Unable to fetch {url}") self.errorState = True return None self.sf.cachePut("sfmal_" + cid, data['content']) for line in data["content"].split('\n'): ip = line.strip().lower() if ip.startswith('#'): continue if targetType == "netblock": try: if IPAddress(ip) in IPNetwork(qry): self.debug(f"{ip} found within netblock/subnet {qry} in CleanTalk Spam List.") return url except Exception as e: self.debug(f"Error encountered parsing: {e}") continue if targetType == "ip": if qry.lower() == ip: self.debug(f"{qry} found in CleanTalk Spam List.") return url return None def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName == 'IP_ADDRESS': targetType = 'ip' malicious_type = "MALICIOUS_IPADDR" blacklist_type = "BLACKLISTED_IPADDR" elif eventName == 'AFFILIATE_IPADDR': if not self.opts.get('checkaffiliates', False): return targetType = 'ip' malicious_type = "MALICIOUS_AFFILIATE_IPADDR" blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" elif eventName == 'NETBLOCK_OWNER': if not self.opts.get('checknetblocks', False): return targetType = 'netblock' malicious_type = "MALICIOUS_NETBLOCK" blacklist_type = "BLACKLISTED_NETBLOCK" elif eventName == 'NETBLOCK_MEMBER': if not self.opts.get('checksubnets', False): return targetType = 'netblock' malicious_type = "MALICIOUS_SUBNET" blacklist_type = "BLACKLISTED_SUBNET" else: self.debug(f"Unexpected event type {eventName}, skipping") return self.debug(f"Checking maliciousness of {eventData} with CleanTalk Spam List") url = self.query(eventData, targetType) if not url: return self.debug(f"{eventData} found in Cleantalk Spam List") text = f"CleanTalk Spam List [{eventData}]\n{url}" evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_cleantalk class ================================================ FILE: modules/sfp_clearbit.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_clearbit # Purpose: Query clearbit.com using their API. # # Author: Steve Micallef # # Created: 20/03/2017 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import base64 import urllib.error import urllib.parse import urllib.request import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_clearbit(SpiderFootPlugin): meta = { 'name': "Clearbit", 'summary': "Check for names, addresses, domains and more based on lookups of e-mail addresses on clearbit.com.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://clearbit.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://clearbit.com/docs" ], 'apiKeyInstructions': [ "Visit https://clearbit.com", "Register account for a Free Trial", "Navigate to https://dashboard.clearbit.com/api", "The API key is listed under 'Your API Key'" ], 'favIcon': "https://clearbit.com/assets/site/logo.png", 'logo': "https://clearbit.com/assets/site/logo.png", 'description': "Clearbit is the marketing data engine for all of your customer interactions. " "Deeply understand your customers, identify future prospects, " "and personalize every single marketing and sales interaction.\n" "Rely on fresh, accurate data with our proprietary real-time lookups. " "Then act on new information immediately, with sales alerting and job change notifications.\n" "Get company attributes like employee count, technologies used, and industry classification—and " "get employee details like role, seniority, and even job change notifications, right at your fingertips.\n" "With our dataset and machine learning algorithms, you’ll have all of " "the information you need to convert leads and grow your business.", } } opts = { "api_key": "" } optdescs = { "api_key": "Clearbit.com API key." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["EMAILADDR"] def producedEvents(self): return [ "RAW_RIR_DATA", "PHONE_NUMBER", "PHYSICAL_ADDRESS", "AFFILIATE_INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC", "INTERNET_NAME" ] def query(self, email: str): api_key = self.opts['api_key'] if isinstance(api_key, str): api_key = api_key.encode('utf-8') token = base64.b64encode(api_key + ':'.encode('utf-8')) headers = { 'Accept': 'application/json', 'Authorization': "Basic " + token.decode('utf-8') } params = { 'email': email } res = self.sf.fetchUrl( f"https://person.clearbit.com/v2/combined/find?{urllib.parse.urlencode(params)}", timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=headers ) return self.parseApiResponse(res) def parseApiResponse(self, res: dict): if not res: self.error("No response from Clearbit.") return None if res['code'] == '404': self.debug("No results from Clearbit.") return None if res['code'] == "401": self.error("Invalid Clearbit API key.") self.errorState = True return None if res['code'] == "402": self.error("You have exceeded your Clearbit API request quota.") self.errorState = True return None # Rate limit is 600 requests per minute # https://dashboard.clearbit.com/docs#rate-limiting if res['code'] == '429': self.error("You are being rate-limited by Clearbit.") return None if res['code'] == '500' or res['code'] == '502' or res['code'] == '503': self.error("Clearbit service is unavailable.") self.errorState = True return None # Catch all non-200 status codes, and presume something went wrong if res['code'] != '200': self.error(f"Unexpected reply from Clearbit: {res['code']}") self.errorState = True return None if res['content'] is None: return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from Clearbit: {e}") return None def handleEvent(self, event): eventName = event.eventType eventData = event.data if self.errorState: return if self.opts['api_key'] == "": self.error(f"You enabled {self.__class__.__name__} but did not set an API key!") self.errorState = True return self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True data = self.query(eventData) if not data: return try: # Get the name associated with the e-mail person = data.get('person') if person: name = person.get('name') if name: fullName = name.get('fullName') if fullName: evt = SpiderFootEvent( "RAW_RIR_DATA", f"Possible full name: {fullName}", self.__name__, event ) self.notifyListeners(evt) except Exception: self.debug("Unable to extract person name from JSON.") pass # Get the location of the person, also indicating # the location of the employer. try: geo = data.get('geo') if geo: location = ', '.join( filter( None, [ geo.get('streetNumber'), geo.get('streetName'), geo.get('city'), geo.get('postalCode'), geo.get('state'), geo.get('country') ] ) ) if location: evt = SpiderFootEvent("PHYSICAL_ADDRESS", location, self.__name__, event) self.notifyListeners(evt) except Exception: self.debug("Unable to extract location from JSON.") pass try: company = data.get('company') if company: domainAliases = company.get('domainAliases') if domainAliases: for d in domainAliases: if self.getTarget().matches(d): t = "INTERNET_NAME" else: t = "AFFILIATE_INTERNET_NAME" evt = SpiderFootEvent( t, d, self.__name__, event ) self.notifyListeners(evt) site = company.get('site') if site: if 'phoneNumbers' in site: for p in site['phoneNumbers']: evt = SpiderFootEvent("PHONE_NUMBER", p, self.__name__, event) self.notifyListeners(evt) if 'emailAddresses' in company['site']: for e in site['emailAddresses']: if e.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" evt = SpiderFootEvent(evttype, e, self.__name__, event) self.notifyListeners(evt) # Get the location of the person, also indicating # the location of the employer. company_geo = company.get('geo') if company_geo: location = ', '.join( filter( None, [ company_geo.get('streetNumber'), company_geo.get('streetName'), company_geo.get('city'), company_geo.get('postalCode'), company_geo.get('state'), company_geo.get('country') ] ) ) if location: evt = SpiderFootEvent("PHYSICAL_ADDRESS", location, self.__name__, event) self.notifyListeners(evt) except Exception: self.debug("Unable to extract company info from JSON.") pass # End of sfp_clearbit class ================================================ FILE: modules/sfp_cloudflaredns.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_cloudflaredns # Purpose: SpiderFoot plug-in for looking up whether hosts are blocked by # CloudFlare family and malware filtering DNS servers. # # Author: Steve Micallef # # Created: 11/05/2020 # Copyright: (c) Steve Micallef 2020 # Licence: MIT # ------------------------------------------------------------------------------- import dns.resolver from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_cloudflaredns(SpiderFootPlugin): meta = { 'name': "CloudFlare DNS", 'summary': "Check if a host would be blocked by CloudFlare DNS.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://www.cloudflare.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://developers.cloudflare.com/1.1.1.1/1.1.1.1-for-families/" ], 'favIcon': "https://www.cloudflare.com/img/favicon/favicon-32x32.png", 'logo': "https://www.cloudflare.com/img/logo-web-badges/cf-logo-on-white-bg.svg", 'description': "1.1.1.1 for Families is the easiest way to add a layer of protection to " "your home network and protect it from malware and adult content. " "1.1.1.1 for Families leverages Cloudflare’s global network to ensure " "that it is fast and secure around the world.", } } opts = { } optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "CO_HOSTED_SITE" ] def producedEvents(self): return [ "BLACKLISTED_INTERNET_NAME", "BLACKLISTED_AFFILIATE_INTERNET_NAME", "BLACKLISTED_COHOST", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_COHOST", ] def queryFamilyDNS(self, qaddr): res = dns.resolver.Resolver() res.nameservers = ["1.1.1.3", "1.0.0.3"] try: return res.resolve(qaddr) except Exception: self.debug(f"Unable to resolve {qaddr}") return None def queryMalwareDNS(self, qaddr): res = dns.resolver.Resolver() res.nameservers = ["1.1.1.2", "1.0.0.2"] try: return res.resolve(qaddr) except Exception: self.debug(f"Unable to resolve {qaddr}") return None def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: return self.results[eventData] = True if eventName == "INTERNET_NAME": e = "BLACKLISTED_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": e = "BLACKLISTED_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": e = "BLACKLISTED_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") return family = self.sf.normalizeDNS(self.queryFamilyDNS(eventData)) malware = self.sf.normalizeDNS(self.queryMalwareDNS(eventData)) if not family or not malware: return if '0.0.0.0' not in family and '0.0.0.0' not in malware: return # Host is blocked only by family filters if '0.0.0.0' not in malware: self.debug(f"{eventData} blocked by CloudFlare Family DNS") evt = SpiderFootEvent(e, f"CloudFlare - Family [{eventData}]", self.__name__, event) self.notifyListeners(evt) return # Host is blocked only by malware filters self.debug(f"{eventData} blocked by CloudFlare Malware DNS") evt = SpiderFootEvent(e, f"CloudFlare - Malware [{eventData}]", self.__name__, event) self.notifyListeners(evt) if eventName == "INTERNET_NAME": e = "MALICIOUS_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": e = "MALICIOUS_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": e = "MALICIOUS_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") evt = SpiderFootEvent(e, f"CloudFlare - Malware [{eventData}]", self.__name__, event) self.notifyListeners(evt) # End of sfp_cloudflaredns class ================================================ FILE: modules/sfp_coinblocker.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_coinblocker # Purpose: Checks if a hostname is listed on CoinBlockerLists. # # Author: steve@binarypool.com # # Created: 07/09/2018 # Copyright: (c) Steve Micallef, 2018 # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_coinblocker(SpiderFootPlugin): meta = { 'name': "CoinBlocker Lists", 'summary': "Check if a domain appears on CoinBlocker lists.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://zerodot1.gitlab.io/CoinBlockerListsWeb/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://zerodot1.gitlab.io/CoinBlockerListsWeb/downloads.html", "https://zerodot1.gitlab.io/CoinBlockerListsWeb/references.html", "https://zerodot1.gitlab.io/CoinBlockerListsWeb/aboutthisproject.html" ], 'favIcon': "https://zerodot1.gitlab.io/CoinBlockerListsWeb/assets/img/favicon.png", 'logo': "https://zerodot1.gitlab.io/CoinBlockerListsWeb/assets/img/favicon.png", 'description': "The CoinBlockerLists are a project to prevent illegal mining in " "browsers or other applications using IPlists and URLLists.\n" "It's not just to block everything without any reason, but to protect " "Internet users from illegal mining.", } } opts = { 'checkaffiliates': True, 'checkcohosts': True, 'cacheperiod': 18, } optdescs = { 'checkaffiliates': "Apply checks to affiliates?", 'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?", 'cacheperiod': "Hours to cache list data before re-fetching.", } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.errorState = False self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "CO_HOSTED_SITE", ] def producedEvents(self): return [ "BLACKLISTED_INTERNET_NAME", "BLACKLISTED_AFFILIATE_INTERNET_NAME", "BLACKLISTED_COHOST", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_COHOST", ] def queryBlocklist(self, target): blocklist = self.retrieveBlocklist() if not blocklist: return False if target.lower() in blocklist: self.debug(f"Host name {target} found in CoinBlocker list.") return True return False def retrieveBlocklist(self): blocklist = self.sf.cacheGet('coinblocker', self.opts.get('cacheperiod', 24)) if blocklist is not None: return self.parseBlocklist(blocklist) url = "https://zerodot1.gitlab.io/CoinBlockerLists/list.txt" res = self.sf.fetchUrl( url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from {url}") self.errorState = True return None if res['content'] is None: self.error(f"Received no content from {url}") self.errorState = True return None self.sf.cachePut("coinblocker", res['content']) return self.parseBlocklist(res['content']) def parseBlocklist(self, blocklist): """Parse plaintext CoinBlocker list Args: blocklist (str): plaintext CoinBlocker list Returns: list: list of blocked host names """ hosts = list() if not blocklist: return hosts for line in blocklist.split('\n'): if not line: continue if line.startswith('#'): continue host = line.strip() # Note: Validation with sf.validHost() is too slow to use here # if not self.sf.validHost(host, self.opts['_internettlds']): # continue if not host: continue hosts.append(host.lower()) return hosts def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName == "INTERNET_NAME": malicious_type = "MALICIOUS_INTERNET_NAME" blacklist_type = "BLACKLISTED_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": if not self.opts.get('checkaffiliates', False): return malicious_type = "MALICIOUS_AFFILIATE_INTERNET_NAME" blacklist_type = "BLACKLISTED_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": if not self.opts.get('checkcohosts', False): return malicious_type = "MALICIOUS_COHOST" blacklist_type = "BACKLISTED_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") return self.debug(f"Checking maliciousness of {eventData} ({eventName}) with CoinBlocker list") if not self.queryBlocklist(eventData): return url = "https://zerodot1.gitlab.io/CoinBlockerLists/list.txt" text = f"CoinBlocker [{eventData}]\n{url}" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_coinblocker class ================================================ FILE: modules/sfp_commoncrawl.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_commoncrawl # Purpose: Searches the commoncrawl.org project's indexes for URLs related # to the target. # # Author: Steve Micallef # # Created: 05/09/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import json import re from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_commoncrawl(SpiderFootPlugin): meta = { 'name': "CommonCrawl", 'summary': "Searches for URLs found through CommonCrawl.org.", 'flags': [], 'useCases': ["Footprint", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "http://commoncrawl.org/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://commoncrawl.org/the-data/get-started/", "https://commoncrawl.org/the-data/examples/", "https://commoncrawl.org/the-data/tutorials/" ], 'favIcon': "https://commoncrawl.org/wp-content/themes/commoncrawl/img/favicon.png", 'logo': "https://commoncrawl.org/wp-content/themes/commoncrawl/img/favicon.png", 'description': "We build and maintain an open repository of web crawl data that can be accessed and analyzed by anyone.\n" "Everyone should have the opportunity to indulge their curiosities, analyze the world and pursue brilliant ideas. " "Small startups or even individuals can now access high quality crawl data that was previously " "only available to large search engine corporations.", } } # Default options opts = { "indexes": 6 } # Option descriptions optdescs = { "indexes": "Number of most recent indexes to attempt, because results tend to be occasionally patchy." } results = None indexBase = list() errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.indexBase = list() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def search(self, target): ret = list() for index in self.indexBase: url = f"https://index.commoncrawl.org/{index}-index?url={target}/*&output=json" res = self.sf.fetchUrl(url, timeout=60, useragent="SpiderFoot") if res['code'] in ["400", "401", "402", "403", "404"]: self.error("CommonCrawl search doesn't seem to be available.") self.errorState = True return None if not res['content']: self.error("CommonCrawl search doesn't seem to be available.") self.errorState = True return None ret.append(res['content']) return ret def getLatestIndexes(self): url = "https://index.commoncrawl.org/" res = self.sf.fetchUrl(url, timeout=60, useragent="SpiderFoot") if res['code'] in ["400", "401", "402", "403", "404"]: self.error("CommonCrawl index collection doesn't seem to be available.") self.errorState = True return list() if not res['content']: self.error("CommonCrawl index collection doesn't seem to be available.") self.errorState = True return list() indexes = re.findall(r".*(CC-MAIN-\d+-\d+).*", str(res['content'])) indexlist = dict() for m in indexes: ms = m.replace("CC-MAIN-", "").replace("-", "") indexlist[ms] = True topindexes = sorted(list(indexlist.keys()), reverse=True)[0:self.opts['indexes']] if len(topindexes) < self.opts['indexes']: self.error("Not able to find latest CommonCrawl indexes.") self.errorState = True return list() retindex = list() for i in topindexes: retindex.append("CC-MAIN-" + str(i)[0:4] + "-" + str(i)[4:6]) self.debug("CommonCrawl indexes: " + str(retindex)) return retindex # What events is this module interested in for input def watchedEvents(self): return ["INTERNET_NAME"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["LINKED_URL_INTERNAL"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.errorState: return if eventData in self.results: return self.results[eventData] = True if len(self.indexBase) == 0: self.indexBase = self.getLatestIndexes() if not self.indexBase: self.error("Unable to fetch CommonCrawl index.") return if len(self.indexBase) == 0: self.error("Unable to fetch CommonCrawl index.") return data = self.search(eventData) if not data: self.error("Unable to obtain content from CommonCrawl.") return sent = list() for content in data: try: for line in content.split("\n"): if self.checkForStop(): return if len(line) < 2: continue link = json.loads(line) if 'url' not in link: continue # CommonCrawl sometimes returns hosts with a trailing . after the domain link['url'] = link['url'].replace(eventData + ".", eventData) if link['url'] in sent: continue sent.append(link['url']) evt = SpiderFootEvent("LINKED_URL_INTERNAL", link['url'], self.__name__, event) self.notifyListeners(evt) except Exception as e: self.error("Malformed JSON from CommonCrawl.org: " + str(e)) return # End of sfp_commoncrawl class ================================================ FILE: modules/sfp_comodo.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_comodo # Purpose: SpiderFoot plug-in for looking up whether hosts are blocked by # Comodo Secure DNS. # # Author: Steve Micallef # # Created: 30/05/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import dns.resolver from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_comodo(SpiderFootPlugin): meta = { 'name': "Comodo Secure DNS", 'summary': "Check if a host would be blocked by Comodo Secure DNS.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://www.comodo.com/secure-dns/", 'model': "FREE_NOAUTH_LIMITED", 'references': [ "https://cdome.comodo.com/pdf/Datasheet-Dome-Shield.pdf", "http://securedns.dnsbycomodo.com/", "https://www.comodo.com/secure-dns/secure-dns-assets/dowloads/ccs-dome-shield-whitepaper-threat-intelligence.pdf", "https://www.comodo.com/secure-dns/secure-dns-assets/dowloads/domeshield-all-use-cases.pdf", ], 'favIcon': "https://www.comodo.com/favicon.ico", 'logo': "https://www.comodo.com/new-assets/images/logo.png", 'description': "Comodo Secure DNS is a domain name resolution service " "that resolves your DNS requests through our worldwide network of redundant DNS servers, " "bringing you the most reliable fully redundant DNS service anywhere, for a safer, " "smarter and faster Internet experience." } } opts = { } optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "CO_HOSTED_SITE" ] def producedEvents(self): return [ "BLACKLISTED_INTERNET_NAME", "BLACKLISTED_AFFILIATE_INTERNET_NAME", "BLACKLISTED_COHOST", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_COHOST", ] def query(self, qaddr): res = dns.resolver.Resolver() res.nameservers = ["8.26.56.26", "8.20.247.20"] try: addrs = res.resolve(qaddr) self.debug(f"Addresses returned: {addrs}") except Exception: self.debug(f"Unable to resolve {qaddr}") return False if addrs: return True return False def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: return self.results[eventData] = True if eventName == "INTERNET_NAME": malicious_type = "MALICIOUS_INTERNET_NAME" blacklist_type = "BLACKLISTED_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": malicious_type = "MALICIOUS_AFFILIATE_INTERNET_NAME" blacklist_type = "BLACKLISTED_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": malicious_type = "MALICIOUS_COHOST" blacklist_type = "BLACKLISTED_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") return # Check that it resolves first, as it becomes a valid # malicious host only if NOT resolved by Comodo Secure DNS. if not self.sf.resolveHost(eventData) and not self.sf.resolveHost6(eventData): return found = self.query(eventData) # Host was found, not blocked if found: return evt = SpiderFootEvent(blacklist_type, f"Comodo Secure DNS [{eventData}]", self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(malicious_type, f"Comodo Secure DNS [{eventData}]", self.__name__, event) self.notifyListeners(evt) # End of sfp_comodo class ================================================ FILE: modules/sfp_company.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_company # Purpose: SpiderFoot plug-in for scanning retrieved content by other # modules (such as sfp_spider) and identifying company names. # # Author: Steve Micallef # # Created: 09/09/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import re from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_company(SpiderFootPlugin): meta = { 'name': "Company Name Extractor", 'summary': "Identify company names in any obtained data.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Content Analysis"] } opts = { 'filterjscss': True } optdescs = { 'filterjscss': "Filter out company names that originated from CSS/JS content. Enabling this avoids detection of popular Javascript and web framework author company names." } def setup(self, sfc, userOpts=dict()): self.sf = sfc for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["TARGET_WEB_CONTENT", "SSL_CERTIFICATE_ISSUED", "DOMAIN_WHOIS", "NETBLOCK_WHOIS", "AFFILIATE_DOMAIN_WHOIS", "AFFILIATE_WEB_CONTENT"] # What events this module produces def producedEvents(self): return ["COMPANY_NAME", "AFFILIATE_COMPANY_NAME"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data # Various ways to identify companies in text # Support up to three word company names with each starting with # a capital letter, allowing for hyphens brackets and numbers within. pattern_prefix = r"(?=[,;:\'\">\(= ]|^)\s?([A-Z0-9\(\)][A-Za-z0-9\-&,\.][^ \"\';:><]*)?\s?([A-Z0-9\(\)][A-Za-z0-9\-&,\.]?[^ \"\';:><]*|[Aa]nd)?\s?([A-Z0-9\(\)][A-Za-z0-9\-&,\.]?[^ \"\';:><]*)?\s+" pattern_match_re = [ 'LLC', r'L\.L\.C\.?', 'AG', r'A\.G\.?', 'GmbH', r'Pty\.?\s+Ltd\.?', r'Ltd\.?', r'Pte\.?', r'Inc\.?', r'INC\.?', 'Incorporated', 'Foundation', r'Corp\.?', 'Corporation', 'SA', r'S\.A\.?', 'SIA', 'BV', r'B\.V\.?', 'NV', r'N\.V\.?', 'PLC', 'Limited', r'Pvt\.?\s+Ltd\.?', 'SARL'] pattern_match = [ 'LLC', 'L.L.C', 'AG', 'A.G', 'GmbH', 'Pty', 'Ltd', 'Pte', 'Inc', 'INC', 'Foundation', 'Corp', 'SA', 'S.A', 'SIA', 'BV', 'B.V', 'NV', 'N.V', 'PLC', 'Limited', 'Pvt.', 'SARL'] pattern_suffix = r"(?=[ \.,:<\)\'\"]|[$\n\r])" # Filter out anything from the company name which matches the below filterpatterns = [ "Copyright", r"\d{4}" # To catch years ] # Don't re-parse company names if eventName in ["COMPANY_NAME", "AFFILIATE_COMPANY_NAME"]: return if eventName == "TARGET_WEB_CONTENT": url = event.actualSource if self.opts['filterjscss'] and (".js" in url or ".css" in url): self.debug("Ignoring web content from CSS/JS.") return self.debug(f"Received event, {eventName}, from {srcModuleName} ({len(eventData)} bytes)") # Strip out everything before the O= try: if eventName == "SSL_CERTIFICATE_ISSUED": eventData = eventData.split("O=")[1] except Exception: self.debug("Couldn't strip out 'O=' from certificate issuer, proceeding anyway...") # Find chunks of text containing what might be a company name first. # This is to avoid running very expensive regexps on large chunks of # data. chunks = list() for pat in pattern_match: start = 0 m = eventData.find(pat, start) while m > 0: start = m - 50 if start < 0: start = 0 end = m + 10 if end >= len(eventData): end = len(eventData) - 1 chunks.append(eventData[start:end]) offset = m + len(pat) m = eventData.find(pat, offset) myres = list() for chunk in chunks: for pat in pattern_match_re: matches = re.findall(pattern_prefix + "(" + pat + ")" + pattern_suffix, chunk, re.MULTILINE | re.DOTALL) for match in matches: matched = 0 for m in match: if len(m) > 0: matched += 1 if matched <= 1: continue fullcompany = "" for m in match: flt = False for f in filterpatterns: if re.match(f, m): flt = True if not flt: fullcompany += m + " " fullcompany = re.sub(r"\s+", " ", fullcompany.strip()) self.info("Found company name: " + fullcompany) if fullcompany in myres: self.debug("Already found from this source.") continue myres.append(fullcompany) if "AFFILIATE_" in eventName: etype = "AFFILIATE_COMPANY_NAME" else: etype = "COMPANY_NAME" evt = SpiderFootEvent(etype, fullcompany, self.__name__, event) if event.moduleDataSource: evt.moduleDataSource = event.moduleDataSource else: evt.moduleDataSource = "Unknown" self.notifyListeners(evt) # End of sfp_company class ================================================ FILE: modules/sfp_cookie.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_cookie # Purpose: SpiderFoot plug-in for extracting cookies from HTTP headers. # # Author: Steve Micallef # # Created: 06/04/2014 # Copyright: (c) Steve Micallef 2014 # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_cookie(SpiderFootPlugin): meta = { 'name': "Cookie Extractor", 'summary': "Extract Cookies from HTTP headers.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Content Analysis"] } opts = {} optdescs = {} results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.__dataSource__ = "Target Website" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["WEBSERVER_HTTPHEADERS"] # What events this module produces def producedEvents(self): return ["TARGET_WEB_COOKIE"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data eventSource = event.actualSource self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventSource in self.results: return self.results[eventSource] = True fqdn = self.sf.urlFQDN(eventSource) if not self.getTarget().matches(fqdn): self.debug(f"Not collecting cookies from external sites. Ignoring HTTP headers from {fqdn}") return try: data = json.loads(eventData) except Exception: self.error("Received HTTP headers from another module in an unexpected format.") return cookie = data.get('cookie') if cookie: evt = SpiderFootEvent("TARGET_WEB_COOKIE", cookie, self.__name__, event) self.notifyListeners(evt) # End of sfp_cookie class ================================================ FILE: modules/sfp_countryname.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_countryname # Purpose: SpiderFoot plug-in for scanning retrieved content by other # modules (such as sfp_iban, sfp_phone, sfp_whois) and identifying country names # # Author: Krishnasis Mandal # # Created: 28/04/2020 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import re import phonenumbers from phonenumbers.phonenumberutil import region_code_for_country_code from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_countryname(SpiderFootPlugin): meta = { 'name': "Country Name Extractor", 'summary': "Identify country names in any obtained data.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Content Analysis"] } opts = { 'cohosted': True, 'affiliate': True, 'noncountrytld': True, 'similardomain': False, } optdescs = { 'cohosted': "Obtain country name from co-hosted sites", 'affiliate': "Obtain country name from affiliate sites", 'noncountrytld': "Parse TLDs not associated with any country as default country domains", 'similardomain': "Obtain country name from similar domains" } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() # Clear / reset any other class member variables here # or you risk them persisting between threads. for opt in userOpts.keys(): self.opts[opt] = userOpts[opt] def detectCountryFromPhone(self, srcPhoneNumber: str) -> str: """Lookup name of country from phone number region code. Args: srcPhoneNumber (str): phone number Returns: str: country name """ if not isinstance(srcPhoneNumber, str): return None try: phoneNumber = phonenumbers.parse(srcPhoneNumber) except Exception: self.debug(f"Skipped invalid phone number: {srcPhoneNumber}") return None try: countryCode = region_code_for_country_code(phoneNumber.country_code) except Exception: self.debug(f"Lookup of region code failed for phone number: {srcPhoneNumber}") return None if not countryCode: return None return SpiderFootHelpers.countryNameFromCountryCode(countryCode.upper()) def detectCountryFromDomainName(self, srcDomain: str) -> str: """Lookup name of country from TLD of domain name. Args: srcDomain (str): domain Returns: str: country name """ if not isinstance(srcDomain, str): return None # Split domain into parts by '.' # Country TLDs are reserved domainParts = srcDomain.split(".") # Search for country TLD in the domain parts - reversed for part in domainParts[::-1]: country_name = SpiderFootHelpers.countryNameFromTld(part) if country_name: return country_name return None def detectCountryFromIBAN(self, srcIBAN: str) -> str: """Detect name of country from IBAN. Args: srcIBAN (str): IBAN Returns: str: country name """ if not isinstance(srcIBAN, str): return None return SpiderFootHelpers.countryNameFromCountryCode(srcIBAN[0:2]) def detectCountryFromData(self, srcData: str) -> list: """Detect name of country from event data (WHOIS lookup, Geo Info, Physical Address, etc) Args: srcData (str): event data Returns: list: list of countries """ countries = list() if not srcData: return countries # Get dictionary of country codes and country names abbvCountryCodes = SpiderFootHelpers.countryCodes() # Look for countrycodes and country in source data for countryName in abbvCountryCodes.values(): if countryName.lower() not in srcData.lower(): continue # Look for country name in source data # Spaces are not included since "New Jersey" and others # will get interpreted as "Jersey", etc. matchCountries = re.findall(r"[,'\"\:\=\[\(\[\n\t\r\.] ?" + countryName + r"[,'\"\:\=\[\(\[\n\t\r\.]", srcData, re.IGNORECASE) if matchCountries: countries.append(countryName) # Look for "Country: ", usually found in Whois records matchCountries = re.findall("country: (.+?)", srcData, re.IGNORECASE) if matchCountries: for m in matchCountries: m = m.strip() if m in abbvCountryCodes: countries.append(abbvCountryCodes[m]) if m in abbvCountryCodes.values(): countries.append(m) return list(set(countries)) # What events is this module interested in for input def watchedEvents(self): return ["IBAN_NUMBER", "PHONE_NUMBER", "AFFILIATE_DOMAIN_NAME", "CO_HOSTED_SITE_DOMAIN", "DOMAIN_NAME", "SIMILARDOMAIN", "AFFILIATE_DOMAIN_WHOIS", "CO_HOSTED_SITE_DOMAIN_WHOIS", "DOMAIN_WHOIS", "GEOINFO", "PHYSICAL_ADDRESS"] # What events this module produces def producedEvents(self): return ["COUNTRY_NAME"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if event.moduleDataSource: moduleDataSource = event.moduleDataSource else: moduleDataSource = "Unknown" self.debug(f"Received event, {eventName}, from {srcModuleName}") eventDataHash = self.sf.hashstring(eventData) if eventDataHash in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventDataHash] = True countryNames = list() # Process the event data based on incoming event type if eventName == "PHONE_NUMBER": countryNames.append(self.detectCountryFromPhone(eventData)) elif eventName == "DOMAIN_NAME": countryNames.append(self.detectCountryFromDomainName(eventData)) elif eventName == "AFFILIATE_DOMAIN_NAME" and self.opts["affiliate"]: countryNames.append(self.detectCountryFromDomainName(eventData)) elif eventName == "CO_HOSTED_SITE_DOMAIN" and self.opts["cohosted"]: countryNames.append(self.detectCountryFromDomainName(eventData)) elif eventName == "SIMILARDOMAIN" and self.opts["similardomain"]: countryNames.append(self.detectCountryFromDomainName(eventData)) elif eventName == "IBAN_NUMBER": countryNames.append(self.detectCountryFromIBAN(eventData)) elif eventName in ["DOMAIN_WHOIS", "GEOINFO", "PHYSICAL_ADDRESS"]: countryNames.extend(self.detectCountryFromData(eventData)) elif eventName == "AFFILIATE_DOMAIN_WHOIS" and self.opts["affiliate"]: countryNames.extend(self.detectCountryFromData(eventData)) elif eventName == "CO_HOSTED_SITE_DOMAIN_WHOIS" and self.opts["cohosted"]: countryNames.extend(self.detectCountryFromData(eventData)) if not countryNames: self.debug(f"Found no country names associated with {eventName}: {eventData}") return for countryName in set(countryNames): if not countryName: continue self.debug(f"Found country name: {countryName}") evt = SpiderFootEvent("COUNTRY_NAME", countryName, self.__name__, event) evt.moduleDataSource = moduleDataSource self.notifyListeners(evt) # End of sfp_countryname class ================================================ FILE: modules/sfp_creditcard.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_creditcard # Purpose: SpiderFoot plug-in for scanning retrieved content by other # modules (such as sfp_spider) and identifying credit card numbers. # # Author: Krishnasis Mandal # # Created: 21/04/2020 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_creditcard(SpiderFootPlugin): meta = { 'name': "Credit Card Number Extractor", 'summary': "Identify Credit Card Numbers in any data", 'flags': ["errorprone"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Content Analysis"] } opts = { } optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() # Override datasource for sfp_creditcard module self.__dataSource__ = "Target Website" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DARKNET_MENTION_CONTENT", "LEAKSITE_CONTENT"] # What events this module produces def producedEvents(self): return ["CREDIT_CARD_NUMBER"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") creditCards = SpiderFootHelpers.extractCreditCardsFromText(eventData) for creditCard in set(creditCards): self.info(f"Found credit card number: {creditCard}") evt = SpiderFootEvent("CREDIT_CARD_NUMBER", creditCard, self.__name__, event) if event.moduleDataSource: evt.moduleDataSource = event.moduleDataSource else: evt.moduleDataSource = "Unknown" self.notifyListeners(evt) # End of sfp_creditcard class ================================================ FILE: modules/sfp_crobat_api.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_crobat_api # Purpose: Search Crobat API for subdomains. # https://www.onsecurity.co.uk/blog/how-i-made-rapid7s-project-sonar-searchable # https://github.com/cgboal/sonarsearch # # Authors: # # Created: 2020-08-29 # Copyright: (c) bcoles 2020 # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_crobat_api(SpiderFootPlugin): meta = { 'name': "Crobat API", 'summary': "Search Crobat API for subdomains.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Passive DNS"], 'dataSource': { 'website': "https://sonar.omnisint.io/", 'model': "FREE_NOAUTH_UNLIMITED", 'logo': "https://sonar.omnisint.io/img/crobat.png", 'description': "The entire Rapid7 Sonar DNS dataset indexed," " available at your fingertips.", } } opts = { "verify": True, "max_pages": 10, "delay": 1 } optdescs = { "verify": "DNS resolve each identified subdomain.", "max_pages": "Maximum number of pages of results to fetch.", "delay": "Delay between requests, in seconds." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in userOpts.keys(): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["DOMAIN_NAME"] def producedEvents(self): return ["RAW_RIR_DATA", "INTERNET_NAME", "INTERNET_NAME_UNRESOLVED"] def queryDomain(self, qry, page=0): headers = { "Accept": "application/json" } params = urllib.parse.urlencode({ 'page': page }) domain = qry.encode('raw_unicode_escape').decode("ascii", errors='replace') res = self.sf.fetchUrl( f"https://sonar.omnisint.io/subdomains/{domain}?{params}", headers=headers, timeout=30, useragent=self.opts['_useragent'] ) time.sleep(self.opts['delay']) return self.parseApiResponse(res) def parseApiResponse(self, res: dict): if not res: self.error("No response from Crobat API.") return None # Future proofing - Crobat API does not implement rate limiting if res['code'] == '429': self.error("You are being rate-limited by Crobat API") self.errorState = True return None # Catch all non-200 status codes, and presume something went wrong if res['code'] != '200': self.error("Failed to retrieve content from Crobat API") self.errorState = True return None if res['content'] is None: return None # returns "null" when page has no data if res['content'] == "null": return None try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None if not isinstance(data, list): self.error("Failed to retrieve content from Crobat API") return None return data def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventName != "DOMAIN_NAME": return page = 0 while page < self.opts['max_pages']: if self.checkForStop(): return if self.errorState: return data = self.queryDomain(eventData, page) if not data: self.debug(f"No information found for domain {eventData} (page: {page})") return evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) page += 1 for domain in set(data): if domain in self.results: continue if not self.getTarget().matches(domain, includeChildren=True, includeParents=True): continue if self.opts['verify'] and not self.sf.resolveHost(domain) and not self.sf.resolveHost6(domain): self.debug(f"Host {domain} could not be resolved") evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", domain, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent("INTERNET_NAME", domain, self.__name__, event) self.notifyListeners(evt) # End of sfp_crobat_api class ================================================ FILE: modules/sfp_crossref.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_crossref # Purpose: SpiderFoot plug-in for scanning links identified from the # spidering process, and for external links, fetching them to # see if those sites link back to the original site, indicating a # potential relationship between the external sites. # # Author: Steve Micallef # # Created: 06/04/2012 # Copyright: (c) Steve Micallef 2012 # Licence: MIT # ------------------------------------------------------------------------------- import re from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_crossref(SpiderFootPlugin): meta = { 'name': "Cross-Referencer", 'summary': "Identify whether other domains are associated ('Affiliates') of the target by looking for links back to the target site(s).", 'flags': [], 'useCases': ["Footprint"], 'categories': ["Crawling and Scanning"] } opts = { 'checkbase': True } optdescs = { "checkbase": "Check the base URL of the potential affiliate if no direct affiliation found?" } fetched = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.fetched = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ 'LINKED_URL_EXTERNAL', 'SIMILARDOMAIN', 'CO_HOSTED_SITE', 'DARKNET_MENTION_URL' ] def producedEvents(self): return [ 'AFFILIATE_INTERNET_NAME', 'AFFILIATE_WEB_CONTENT' ] def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") # SIMILARDOMAIN and CO_HOSTED_SITE events are domains, not URLs. # Assume HTTP. if eventName in ['SIMILARDOMAIN', 'CO_HOSTED_SITE']: url = 'http://' + eventData.lower() elif 'URL' in eventName: url = eventData else: return fqdn = self.sf.urlFQDN(url) # We are only interested in external sites for the crossref if self.getTarget().matches(fqdn): self.debug(f"Ignoring {url} as not external") return if eventData in self.fetched: self.debug(f"Ignoring {url} as already tested") return if not self.sf.resolveHost(fqdn) and not self.sf.resolveHost6(fqdn): self.debug(f"Ignoring {url} as {fqdn} does not resolve") return self.fetched[url] = True self.debug(f"Testing URL for affiliation: {url}") res = self.sf.fetchUrl( url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], sizeLimit=10000000, verify=False ) if res['content'] is None: self.debug(f"Ignoring {url} as no data returned") return matched = False for name in self.getTarget().getNames(): # Search for mentions of our host/domain in the external site's data pat = re.compile( r"([\.\'\/\"\ ]" + re.escape(name) + r"[\.\'\/\"\ ])", re.IGNORECASE ) matches = re.findall(pat, str(res['content'])) if len(matches) > 0: matched = True break if not matched: # If the name wasn't found in the affiliate, and checkbase is set, # fetch the base URL of the affiliate to check for a crossref. if eventName == "LINKED_URL_EXTERNAL" and self.opts['checkbase']: # Check the base url to see if there is an affiliation url = SpiderFootHelpers.urlBaseUrl(eventData) if url in self.fetched: return self.fetched[url] = True res = self.sf.fetchUrl( url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], sizeLimit=10000000, verify=False ) if res['content'] is not None: for name in self.getTarget().getNames(): pat = re.compile( r"([\.\'\/\"\ ]" + re.escape(name) + r"[\'\/\"\ ])", re.IGNORECASE ) matches = re.findall(pat, str(res['content'])) if len(matches) > 0: matched = True break if not matched: return if not event.moduleDataSource: event.moduleDataSource = "Unknown" self.info(f"Found link to target from affiliate: {url}") evt1 = SpiderFootEvent( "AFFILIATE_INTERNET_NAME", self.sf.urlFQDN(url), self.__name__, event ) evt1.moduleDataSource = event.moduleDataSource self.notifyListeners(evt1) evt2 = SpiderFootEvent( "AFFILIATE_WEB_CONTENT", res['content'], self.__name__, evt1 ) evt2.moduleDataSource = event.moduleDataSource self.notifyListeners(evt2) # End of sfp_crossref class ================================================ FILE: modules/sfp_crt.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_crt # Purpose: SpiderFoot plug-in to identify historical certificates for a domain # from crt.sh, and from this identify hostnames. # # Author: Steve Micallef # # Created: 17/03/2017 # Copyright: (c) Steve Micallef 2017 # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib.parse from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_crt(SpiderFootPlugin): meta = { 'name': "Certificate Transparency", 'summary': "Gather hostnames from historical certificates in crt.sh.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://crt.sh/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://sectigo.com/", "https://github.com/crtsh" ], 'favIcon': "https://crt.sh/sectigo_s.png", 'logo': "https://crt.sh/sectigo_s.png", 'description': "Free CT Log Certificate Search Tool from Sectigo (formerly Comodo CA)." } } opts = { 'verify': True, 'fetchcerts': True, } optdescs = { 'verify': 'Verify certificate subject alternative names resolve.', 'fetchcerts': 'Fetch each certificate found, for processing by other modules.', } results = None cert_ids = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.errorState = False self.results = self.tempStorage() self.cert_ids = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ['DOMAIN_NAME', 'INTERNET_NAME'] def producedEvents(self): return [ "SSL_CERTIFICATE_RAW", "RAW_RIR_DATA", "INTERNET_NAME", "INTERNET_NAME_UNRESOLVED", "DOMAIN_NAME", "CO_HOSTED_SITE", "CO_HOSTED_SITE_DOMAIN" ] def queryDomain(self, qry: str): params = { 'q': '%.' + qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), 'output': 'json' } res = self.sf.fetchUrl( "https://crt.sh/?" + urllib.parse.urlencode(params), timeout=30, useragent=self.opts['_useragent'] ) time.sleep(0.5) return self.parseApiResponse(res) def parseApiResponse(self, res: dict): if not res: self.error("No response from crt.sh") return None if res['code'] == '404': self.debug("No results for query") return None # Future proofing - crt.sh does not implement rate limiting if res['code'] == '429': self.error("You are being rate-limited by crt.sh") self.errorState = True return None if res['code'] == '500' or res['code'] == '502' or res['code'] == '503': self.error("crt.sh service is unavailable") self.errorState = True return None # Catch all other non-200 status codes, and presume something went wrong if res['code'] != '200': self.error("Failed to retrieve content from crt.sh") self.errorState = True return None if not res['content']: return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None def handleEvent(self, event): if self.errorState: return if event.data in self.results: return self.results[event.data] = True self.debug(f"Received event, {event.eventType}, from {event.module}") data = self.queryDomain(event.data) if not data: self.debug(f"No certificate transparency results for domain {event.data}") return evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) domains = list() certs = list() # Extract all domains and certificate IDs from each certificate for cert_info in data: cert_id = cert_info.get('id') if not cert_id: continue if cert_id in self.cert_ids: continue self.cert_ids[cert_id] = True if self.opts['fetchcerts']: certs.append(cert_id) domain = cert_info.get('name_value') if not domain: continue for d in domain.split("\n"): if d.lower() == event.data.lower(): continue domains.append(d.lower().replace("*.", "")) if self.opts['fetchcerts'] and len(certs) > 0: self.info(f"Retrieving {len(set(certs))} certificates ...") # Fetch and store each certificate # Report SAN domains for each certificate for cert_id in set(certs): if self.checkForStop(): return if self.errorState: break params = { 'd': str(cert_id) } res = self.sf.fetchUrl( 'https://crt.sh/?' + urllib.parse.urlencode(params), timeout=30, useragent=self.opts['_useragent'] ) time.sleep(0.5) if not res or not res['content']: self.error(f"Error retrieving certificate with ID {cert_id}. No response from crt.sh") continue try: cert = self.sf.parseCert(str(res['content'])) except Exception as e: self.info(f"Error parsing certificate: {e}") continue cert_text = cert.get('text') if cert_text: evt = SpiderFootEvent("SSL_CERTIFICATE_RAW", str(cert_text), self.__name__, event) self.notifyListeners(evt) sans = cert.get('altnames', list()) if not sans: continue for san in sans: if san.lower() == event.data.lower(): continue domains.append(san.lower().replace("*.", "")) if self.opts['verify'] and len(domains) > 0: self.info(f"Resolving {len(set(domains))} domains ...") for domain in set(domains): if domain in self.results: continue if not self.sf.validHost(domain, self.opts['_internettlds']): continue if self.getTarget().matches(domain, includeChildren=True, includeParents=True): evt_type = 'INTERNET_NAME' if self.opts['verify'] and not self.sf.resolveHost(domain) and not self.sf.resolveHost6(domain): self.debug(f"Host {domain} could not be resolved") evt_type += '_UNRESOLVED' else: evt_type = 'CO_HOSTED_SITE' evt = SpiderFootEvent(evt_type, domain, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(domain, self.opts['_internettlds']): if evt_type == 'CO_HOSTED_SITE': evt = SpiderFootEvent('CO_HOSTED_SITE_DOMAIN', domain, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent('DOMAIN_NAME', domain, self.__name__, event) self.notifyListeners(evt) # End of sfp_crt class ================================================ FILE: modules/sfp_crxcavator.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_crxcavator # Purpose: Query CRXcavator for Chrome extensions. # # Author: # # Created: 2020-09-19 # Copyright: (c) bcoles 2019 # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_crxcavator(SpiderFootPlugin): meta = { 'name': "CRXcavator", 'summary': "Search CRXcavator for Chrome extensions.", 'flags': [], 'useCases': ["Investigate", "Footprint", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://crxcavator.io/", 'model': "FREE_NOAUTH_UNLIMITED", 'favIcon': "https://crxcavator.io/favicon-32x32.png", 'logo': "https://crxcavator.io/apple-touch-icon.png", 'description': "CRXcavator automatically scans the entire Chrome Web " "Store every 3 hours and produces a quantified risk score for " "each Chrome Extension based on several factors.", } } opts = { "verify": True, } optdescs = { "verify": "Verify identified hostnames resolve.", } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ 'DOMAIN_NAME' ] def producedEvents(self): return [ 'APPSTORE_ENTRY', 'INTERNET_NAME', 'INTERNET_NAME_UNRESOLVED', 'LINKED_URL_INTERNAL', 'AFFILIATE_INTERNET_NAME', 'AFFILIATE_INTERNET_NAME_UNRESOLVED', 'PHYSICAL_ADDRESS', 'RAW_RIR_DATA' ] def query(self, qry): params = urllib.parse.urlencode({ 'q': qry.encode('raw_unicode_escape').decode("ascii", errors='replace') }) res = self.sf.fetchUrl( f"https://api.crxcavator.io/v1/search?{params}", useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout'] ) time.sleep(1) if res['content'] is None: return None try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from CRXcavator: {e}") return None if not data: self.debug(f"No results found for {qry}") return None return data def queryExtension(self, extension_id): res = self.sf.fetchUrl( f"https://api.crxcavator.io/v1/report/{extension_id}", useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout'] ) time.sleep(1) if res['content'] is None: return None try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from CRXcavator: {e}") return None if not data: self.debug(f"No results found for extension {extension_id}") return None return data def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if eventName not in self.watchedEvents(): return self.results[eventData] = True domain_keyword = self.sf.domainKeyword(eventData, self.opts['_internettlds']) results = self.query(domain_keyword) if not results: self.info(f"No results found for {domain_keyword}") return evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(results), self.__name__, event) self.notifyListeners(evt) urls = list() hosts = list() locations = list() for result in results: if not isinstance(result, dict): continue extension_id = result.get('extension_id') if not extension_id: continue if '@' in extension_id: continue extensions = self.queryExtension(extension_id) if not extensions: continue evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(extensions), self.__name__, event) self.notifyListeners(evt) for extension in extensions: data = extension.get('data') if not data: continue manifest = data.get('manifest') if not manifest: continue version = manifest.get('version') if not version: continue webstore = data.get('webstore') if not webstore: continue name = webstore.get('name') if not name: continue app_full_name = f"{name} {version} ({extension_id})" privacy_policy = webstore.get('privacy_policy') support_site = webstore.get('support_site') offered_by = webstore.get('offered_by') website = webstore.get('website') if not privacy_policy and not support_site and not offered_by and not website: continue if ( not self.getTarget().matches(self.sf.urlFQDN(privacy_policy), includeChildren=True, includeParents=True) and not self.getTarget().matches(self.sf.urlFQDN(website), includeChildren=True, includeParents=True) and not self.getTarget().matches(self.sf.urlFQDN(offered_by), includeChildren=True, includeParents=True) and not self.getTarget().matches(self.sf.urlFQDN(support_site), includeChildren=True, includeParents=True) ): self.debug(f"Extension {app_full_name} does not match {eventData}, skipping") continue app_data = f"{name} {version}\nhttps://chrome.google.com/webstore/detail/{extension_id}" evt = SpiderFootEvent('APPSTORE_ENTRY', app_data, self.__name__, event) self.notifyListeners(evt) if privacy_policy: urls.append(privacy_policy) if support_site: urls.append(support_site) if website: urls.append(website) if offered_by: urls.append(offered_by) address = webstore.get('address') if address and len(address) > 10: locations.append(address) for url in set(urls): if not url: continue host = self.sf.urlFQDN(url) if not host: continue if self.getTarget().matches(host, includeChildren=True, includeParents=True): evt = SpiderFootEvent('LINKED_URL_INTERNAL', url, self.__name__, event) self.notifyListeners(evt) hosts.append(host) for host in set(hosts): if not host: continue if self.getTarget().matches(host, includeChildren=True, includeParents=True): evt_type = 'INTERNET_NAME' else: evt_type = 'AFFILIATE_INTERNET_NAME' if self.opts['verify'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host): self.debug(f"Host {host} could not be resolved") evt_type += '_UNRESOLVED' evt = SpiderFootEvent(evt_type, host, self.__name__, event) self.notifyListeners(evt) for location in set(locations): evt = SpiderFootEvent("PHYSICAL_ADDRESS", location, self.__name__, event) self.notifyListeners(evt) # End of sfp_crxcavator class ================================================ FILE: modules/sfp_customfeed.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_customfeed # Purpose: Checks if an ASN, IP, hostname or domain is listed as malicious # in a user-supplied data feed. # # Author: steve@binarypool.com # # Created: 11/11/2018 # Copyright: (c) Steve Micallef, 2018 # Licence: MIT # ------------------------------------------------------------------------------- import re from netaddr import IPAddress, IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin malchecks = { 'Custom Threat Data': { 'id': '_customfeed', 'checks': ['ip', 'netblock', 'asn', 'domain'], 'regex': '^{0}$' } } class sfp_customfeed(SpiderFootPlugin): meta = { 'name': "Custom Threat Feed", 'summary': "Check if a host/domain, netblock, ASN or IP is malicious according to your custom feed.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"] } # Default options opts = { 'checkaffiliates': True, 'checkcohosts': True, 'url': "", 'cacheperiod': 0 } # Option descriptions optdescs = { 'url': "The URL where the feed can be found. Exact matching is performed so the format must be a single line per host, ASN, domain, IP or netblock.", 'checkaffiliates': "Apply checks to affiliates?", 'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?", 'cacheperiod': "Maximum age of data in hours before re-downloading. 0 to always download." } # Be sure to completely clear any class variables in setup() # or you run the risk of data persisting between scan runs. results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False # Clear / reset any other class member variables here # or you risk them persisting between threads. for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input # * = be notified about all events. def watchedEvents(self): return ["INTERNET_NAME", "IP_ADDRESS", "AFFILIATE_INTERNET_NAME", "AFFILIATE_IPADDR", "CO_HOSTED_SITE"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_COHOST"] # Look up 'list' type resources def resourceList(self, replaceme_id, target, targetType): targetDom = '' # Get the base domain if we're supplied a domain if targetType == "domain": targetDom = self.sf.hostDomain(target, self.opts['_internettlds']) if not targetDom: return None for check in list(malchecks.keys()): cid = malchecks[check]['id'] url = self.opts['url'] if replaceme_id == cid: data = dict() data['content'] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0)) if data['content'] is None: data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if data['content'] is None: self.error("Unable to fetch " + url) return None self.sf.cachePut("sfmal_" + cid, data['content']) # If we're looking at netblocks if targetType == "netblock": iplist = list() # Get the regex, replace {0} with an IP address matcher to # build a list of IP. # Cycle through each IP and check if it's in the netblock. if 'regex' in malchecks[check]: rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)") pat = re.compile(rx, re.IGNORECASE) self.debug("New regex for " + check + ": " + rx) for line in data['content'].split('\n'): grp = re.findall(pat, line) if len(grp) > 0: # self.debug("Adding " + grp[0] + " to list.") iplist.append(grp[0]) else: iplist = data['content'].split('\n') for ip in iplist: if len(ip) < 8 or ip.startswith("#"): continue ip = ip.strip() try: if IPAddress(ip) in IPNetwork(target): self.debug(f"{ip} found within netblock/subnet {target} in {check}") return url except Exception as e: self.debug(f"Error encountered parsing: {e}") continue return None # If we're looking at hostnames/domains/IPs if 'regex' not in malchecks[check]: for line in data['content'].split('\n'): if line == target or (targetType == "domain" and line == targetDom): self.debug(target + "/" + targetDom + " found in " + check + " list.") return url else: # Check for the domain and the hostname try: rxDom = str(malchecks[check]['regex']).format(targetDom) rxTgt = str(malchecks[check]['regex']).format(target) for line in data['content'].split('\n'): if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \ re.match(rxTgt, line, re.IGNORECASE): self.debug(target + "/" + targetDom + " found in " + check + " list.") return url except Exception as e: self.debug("Error encountered parsing 2: " + str(e)) continue return None def lookupItem(self, resourceId, itemType, target): for check in list(malchecks.keys()): cid = malchecks[check]['id'] if cid == resourceId and itemType in malchecks[check]['checks']: self.debug("Checking maliciousness of " + target + " (" + itemType + ") with: " + cid) return self.resourceList(cid, target, itemType) return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.errorState: return if self.opts['url'] == "": self.error("You enabled sfp_customfeed but defined no custom feed URL!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False): return if eventName == 'AFFILIATE_IPADDR' \ and not self.opts.get('checkaffiliates', False): return if eventName == 'NETBLOCK_OWNER' and not self.opts.get('checknetblocks', False): return if eventName == 'NETBLOCK_MEMBER' and not self.opts.get('checksubnets', False): return for check in list(malchecks.keys()): cid = malchecks[check]['id'] if eventName in ['IP_ADDRESS', 'AFFILIATE_IPADDR']: typeId = 'ip' if eventName == 'IP_ADDRESS': evtType = 'MALICIOUS_IPADDR' else: evtType = 'MALICIOUS_AFFILIATE_IPADDR' if eventName in ['BGP_AS_OWNER', 'BGP_AS_MEMBER']: typeId = 'asn' evtType = 'MALICIOUS_ASN' if eventName in ['INTERNET_NAME', 'CO_HOSTED_SITE', 'AFFILIATE_INTERNET_NAME', ]: typeId = 'domain' if eventName == "INTERNET_NAME": evtType = "MALICIOUS_INTERNET_NAME" if eventName == 'AFFILIATE_INTERNET_NAME': evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME' if eventName == 'CO_HOSTED_SITE': evtType = 'MALICIOUS_COHOST' if eventName == 'NETBLOCK_OWNER': typeId = 'netblock' evtType = 'MALICIOUS_NETBLOCK' if eventName == 'NETBLOCK_MEMBER': typeId = 'netblock' evtType = 'MALICIOUS_SUBNET' url = self.lookupItem(cid, typeId, eventData) if self.checkForStop(): return # Notify other modules of what you've found if url is not None: text = f"{check} [{eventData}]\n{url}" evt = SpiderFootEvent(evtType, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_customfeed class ================================================ FILE: modules/sfp_cybercrimetracker.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_cybercrimetracker # Purpose: Check if a host/domain or IP address is malicious according to cybercrime-tracker.net. # # Author: steve@binarypool.com # # Created: 14/12/2013 # Copyright: (c) Steve Micallef, 2013 # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_cybercrimetracker(SpiderFootPlugin): meta = { 'name': "CyberCrime-Tracker.net", 'summary': "Check if a host/domain or IP address is malicious according to CyberCrime-Tracker.net.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://cybercrime-tracker.net/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://cybercrime-tracker.net/tools.php", "https://cybercrime-tracker.net/about.php" ], 'favIcon': "https://cybercrime-tracker.net/favicon.ico", 'logo': "https://cybercrime-tracker.net/favicon.ico", 'description': "CyberCrime is a C&C panel tracker, in other words, " "it lists the administration interfaces of certain in-the-wild botnets.", } } # Default options opts = { 'checkaffiliates': True, 'checkcohosts': True, 'cacheperiod': 18 } # Option descriptions optdescs = { 'checkaffiliates': "Apply checks to affiliates?", 'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?", 'cacheperiod': "Hours to cache list data before re-fetching." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "INTERNET_NAME", "IP_ADDRESS", "AFFILIATE_INTERNET_NAME", "AFFILIATE_IPADDR", "CO_HOSTED_SITE" ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_INTERNET_NAME", "BLACKLISTED_AFFILIATE_IPADDR", "BLACKLISTED_AFFILIATE_INTERNET_NAME", "BLACKLISTED_COHOST", "MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_COHOST" ] def queryBlacklist(self, target): blacklist = self.retrieveBlacklist() if not blacklist: return False if target.lower() in blacklist: self.debug(f"Host name {target} found in CyberCrime-Tracker.net blacklist.") return True return False def retrieveBlacklist(self): blacklist = self.sf.cacheGet('cybercrime-tracker', 24) if blacklist is not None: return self.parseBlacklist(blacklist) res = self.sf.fetchUrl( "https://cybercrime-tracker.net/all.php", timeout=10, useragent=self.opts['_useragent'], ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from CyberCrime-Tracker.net.") self.errorState = True return None if res['content'] is None: self.error("Received no content from CyberCrime-Tracker.net") self.errorState = True return None self.sf.cachePut("cybercrime-tracker", res['content']) return self.parseBlacklist(res['content']) def parseBlacklist(self, blacklist): """Parse plaintext blacklist Args: blacklist (str): plaintext blacklist from CyberCrime-Tracker.net Returns: list: list of blacklisted IP addresses and host names """ hosts = list() if not blacklist: return hosts for line in blacklist.split('\n'): if not line: continue if line.startswith('#'): continue # Note: URL parsing and validation with sf.validHost() is too slow to use here host = line.split("/")[0] if not host: continue if "." not in host: continue hosts.append(host.split(':')[0]) return hosts def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName == 'IP_ADDRESS': malicious_type = "MALICIOUS_IPADDR" blacklist_type = "BLACKLISTED_IPADDR" elif eventName == 'AFFILIATE_IPADDR': if not self.opts.get('checkaffiliates', False): return malicious_type = "MALICIOUS_AFFILIATE_IPADDR" blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" elif eventName == "INTERNET_NAME": malicious_type = "MALICIOUS_INTERNET_NAME" blacklist_type = "BLACKLISTED_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": if not self.opts.get('checkaffiliates', False): return malicious_type = "MALICIOUS_AFFILIATE_INTERNET_NAME" blacklist_type = "BLACKLISTED_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": if not self.opts.get('checkcohosts', False): return malicious_type = "MALICIOUS_COHOST" blacklist_type = "BLACKLISTED_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") return self.debug(f"Checking maliciousness of {eventData} ({eventName}) with CyberCrime-Tracker.net") if not self.queryBlacklist(eventData): return url = f"https://cybercrime-tracker.net/index.php?search={eventData}" text = f"CyberCrime-Tracker.net Malicious Submissions [{eventData}]\n{url}" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_cybercrimetracker class ================================================ FILE: modules/sfp_debounce.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_debounce # Purpose: Spiderfoot plugin to check if an email is # disposable using Debounce API. # # Author: Krishnasis Mandal # # Created: 2020-10-01 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_debounce(SpiderFootPlugin): meta = { 'name': "Debounce", 'summary': "Check whether an email is disposable", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://debounce.io/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://debounce.io/free-disposable-check-api/" ], 'favIcon': "https://debounce.io/wp-content/uploads/2018/01/favicon-2.png", 'logo': "https://debounce.io/wp-content/uploads/2018/01/debounce-logo-2.png", 'description': "DeBounce provides a free & powerful API endpoint for checking " "a domain or email address against a realtime up-to-date list of disposable domains." "CORS is enabled for all originating domains, " "so you can call the API directly from your client-side code.", } } opts = { } optdescs = { } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "EMAILADDR" ] def producedEvents(self): return [ "EMAILADDR_DISPOSABLE", "RAW_RIR_DATA" ] def queryEmailAddr(self, qry): res = self.sf.fetchUrl( f"https://disposable.debounce.io?email={qry}", timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot" ) if res['content'] is None: self.info(f"No Debounce info found for {qry}") return None try: return json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from Debounce: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") self.results[eventData] = True data = self.queryEmailAddr(eventData) if data is None: return isDisposable = data.get('disposable') if isDisposable == "true": evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent("EMAILADDR_DISPOSABLE", eventData, self.__name__, event) self.notifyListeners(evt) # End of sfp_debounce class ================================================ FILE: modules/sfp_dehashed.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_dehashed # Purpose: Gather breach data from Dehashed API. # # Author: # # Created: 16-01-2021 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import base64 import json import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_dehashed(SpiderFootPlugin): meta = { 'name': "Dehashed", 'summary': "Gather breach data from Dehashed API.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Leaks, Dumps and Breaches"], 'dataSource': { 'website': "https://www.dehashed.com/", 'model': "COMMERCIAL_ONLY", 'references': [ "https://www.dehashed.com/docs" ], 'apiKeyInstructions': [ "Visit https://www.dehashed.com/register" "Register a free account", "Visit https://www.dehashed.com/profile", "Your API key is listed under 'API Key'", ], 'favIcon': "https://www.dehashed.com/assets/img/favicon.ico", 'logo': "https://www.dehashed.com/assets/img/logo.png", 'description': "Have you been compromised? " "DeHashed provides free deep-web scans and protection against credential leaks. " "A modern personal asset search engine created for " "security analysts, journalists, security companies, " "and everyday people to help secure accounts and provide insight on compromised assets. " "Free breach alerts & breach notifications.", } } # Default options opts = { 'api_key_username': '', 'api_key': '', 'per_page': 10000, 'max_pages': 2, 'pause': 1 } # Option descriptions optdescs = { 'api_key_username': 'Dehashed username.', 'api_key': 'Dehashed API key.', 'per_page': 'Maximum number of results per page.(Max: 10000)', 'max_pages': 'Maximum number of pages to fetch(Max: 10 pages)', 'pause': 'Number of seconds to wait between each API call.' } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return [ "DOMAIN_NAME", "EMAILADDR" ] # What events this module produces def producedEvents(self): return [ 'EMAILADDR', 'EMAILADDR_COMPROMISED', 'PASSWORD_COMPROMISED', 'HASH_COMPROMISED', 'RAW_RIR_DATA' ] # Query Dehashed def query(self, event, per_page, start): if event.eventType == "EMAILADDR": queryString = f"https://api.dehashed.com/search?query=email:\"{event.data}\"&page={start}&size={self.opts['per_page']}" if event.eventType == "DOMAIN_NAME": queryString = f"https://api.dehashed.com/search?query=email:\"@{event.data}\"&page={start}&size={self.opts['per_page']}" token = (base64.b64encode(self.opts['api_key_username'].encode('utf8') + ":".encode('utf-8') + self.opts['api_key'].encode('utf-8'))).decode('utf-8') headers = { 'Accept': 'application/json', 'Authorization': f'Basic {token}' } res = self.sf.fetchUrl(queryString, headers=headers, timeout=15, useragent=self.opts['_useragent'], verify=True) time.sleep(self.opts['pause']) if res['code'] == "400": self.error("Too many requests were performed in a small amount of time. Please wait a bit before querying the API.") time.sleep(5) res = self.sf.fetchUrl(queryString, headers=headers, timeout=15, useragent=self.opts['_useragent'], verify=True) if res['code'] == "401": self.error("Invalid API credentials") self.errorState = True return None if res['code'] != "200": self.error("Unable to fetch data from Dehashed.") self.errorState = True return None if res['content'] is None: self.debug('No response from Dehashed') return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if srcModuleName == self.__name__: return if eventData in self.results: return if self.errorState: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "" or self.opts['api_key_username'] == "": self.error("You enabled sfp_dehashed but did not set an API key/API Key Username!") self.errorState = True return currentPage = 1 maxPages = self.opts['max_pages'] perPage = self.opts['per_page'] while currentPage <= maxPages: if self.checkForStop(): return if self.errorState: break data = self.query(event, perPage, currentPage) if not data: return breachResults = set() emailResults = set() if not data.get('entries'): return for row in data.get('entries'): email = row.get('email') password = row.get('password') passwordHash = row.get('hashed_password') leakSource = row.get('database_name', 'Unknown') if f"{email} [{leakSource}]" in breachResults: continue breachResults.add(f"{email} [{leakSource}]") if eventName == "EMAILADDR": if email == eventData: evt = SpiderFootEvent('EMAILADDR_COMPROMISED', f"{email} [{leakSource}]", self.__name__, event) self.notifyListeners(evt) if password: evt = SpiderFootEvent('PASSWORD_COMPROMISED', f"{email}:{password} [{leakSource}]", self.__name__, event) self.notifyListeners(evt) if passwordHash: evt = SpiderFootEvent('HASH_COMPROMISED', f"{email}:{passwordHash} [{leakSource}]", self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent('RAW_RIR_DATA', str(row), self.__name__, event) self.notifyListeners(evt) if eventName == "DOMAIN_NAME": pevent = SpiderFootEvent("EMAILADDR", email, self.__name__, event) if email not in emailResults: self.notifyListeners(pevent) emailResults.add(email) evt = SpiderFootEvent('EMAILADDR_COMPROMISED', f"{email} [{leakSource}]", self.__name__, pevent) self.notifyListeners(evt) if password: evt = SpiderFootEvent('PASSWORD_COMPROMISED', f"{email}:{password} [{leakSource}]", self.__name__, pevent) self.notifyListeners(evt) if passwordHash: evt = SpiderFootEvent('HASH_COMPROMISED', f"{email}:{passwordHash} [{leakSource}]", self.__name__, pevent) self.notifyListeners(evt) evt = SpiderFootEvent('RAW_RIR_DATA', str(row), self.__name__, pevent) self.notifyListeners(evt) currentPage += 1 if data.get('total') < self.opts['per_page']: break # End of sfp_dehashed class ================================================ FILE: modules/sfp_digitaloceanspace.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_digitaloceanspace # Purpose: SpiderFoot plug-in for identifying potential Digital Ocean spaces # related to the target. # # Author: Steve Micallef # # Created: 16/06/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import random import threading import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_digitaloceanspace(SpiderFootPlugin): meta = { 'name': "Digital Ocean Space Finder", 'summary': "Search for potential Digital Ocean Spaces associated with the target and attempt to list their contents.", 'flags': [], 'useCases': ["Footprint", "Passive"], 'categories': ["Crawling and Scanning"], 'dataSource': { 'website': "https://www.digitalocean.com/products/spaces/", 'model': "FREE_NOAUTH_UNLIMITED", 'favIcon': 'https://www.digitalocean.com/_next/static/media/favicon-32x32.b7ef9ede.png', 'logo': 'https://www.digitalocean.com/_next/static/media/logo.87a8f3b8.svg', 'description': "Store and deliver vast amounts of content." "S3-compatible object storage with a built-in CDN that makes scaling easy, reliable, and affordable." } } # Default options opts = { "endpoints": "nyc3.digitaloceanspaces.com,sgp1.digitaloceanspaces.com,ams3.digitaloceanspaces.com", "suffixes": "test,dev,web,beta,bucket,space,files,content,data,prod,staging,production,stage,app,media,development,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data,-prod,-staging,-production,-stage,-app,-media,-development", "_maxthreads": 20 } # Option descriptions optdescs = { "endpoints": "Different Digital Ocean locations to check where spaces may exist.", "suffixes": "List of suffixes to append to domains tried as space names", "_maxthreads": "Maximum threads" } results = None s3results = dict() lock = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.s3results = dict() self.results = self.tempStorage() self.lock = threading.Lock() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME", "LINKED_URL_EXTERNAL"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"] def checkSite(self, url): res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True) if not res['content']: return if "NoSuchBucket" in res['content']: self.debug(f"Not a valid bucket: {url}") return # Bucket found if res['code'] in ["301", "302", "200"]: # Bucket has files if "ListBucketResult" in res['content']: with self.lock: self.s3results[url] = res['content'].count("") else: # Bucket has no files with self.lock: self.s3results[url] = 0 def threadSites(self, siteList): self.s3results = dict() running = True i = 0 t = [] for site in siteList: if self.checkForStop(): return None self.info("Spawning thread to check bucket: " + site) tname = str(random.SystemRandom().randint(0, 999999999)) t.append(threading.Thread(name='thread_sfp_digitaloceanspaces_' + tname, target=self.checkSite, args=(site,))) t[i].start() i += 1 # Block until all threads are finished while running: found = False for rt in threading.enumerate(): if rt.name.startswith("thread_sfp_digitaloceanspaces_"): found = True if not found: running = False time.sleep(0.25) # Return once the scanning has completed return self.s3results def batchSites(self, sites): i = 0 res = list() siteList = list() for site in sites: if i >= self.opts['_maxthreads']: data = self.threadSites(siteList) if data is None: return res for ret in list(data.keys()): if data[ret]: # bucket:filecount res.append(ret + ":" + str(data[ret])) i = 0 siteList = list() siteList.append(site) i += 1 return res # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventName == "LINKED_URL_EXTERNAL": if ".digitaloceanspaces.com" in eventData: b = self.sf.urlFQDN(eventData) evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event) self.notifyListeners(evt) return targets = [eventData.replace('.', '')] kw = self.sf.domainKeyword(eventData, self.opts['_internettlds']) if kw: targets.append(kw) urls = list() for t in targets: for e in self.opts['endpoints'].split(','): suffixes = [''] + self.opts['suffixes'].split(',') for s in suffixes: if self.checkForStop(): return b = t + s + "." + e url = "https://" + b urls.append(url) # Batch the scans ret = self.batchSites(urls) for b in ret: bucket = b.split(":") evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event) self.notifyListeners(evt) if bucket[2] != "0": evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucket[0] + ":" + bucket[1] + ": " + bucket[2] + " files found.", self.__name__, evt) self.notifyListeners(evt) # End of sfp_digitaloceanspace class ================================================ FILE: modules/sfp_dns_for_family.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_dns_for_family # Purpose: Check if a host would be blocked by DNS for Family. # # Author: # # Created: 2021-10-11 # Copyright: (c) bcoles 2021 # Licence: MIT # ------------------------------------------------------------------------------- import dns.resolver from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_dns_for_family(SpiderFootPlugin): meta = { 'name': "DNS for Family", 'summary': "Check if a host would be blocked by DNS for Family.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://dnsforfamily.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://dnsforfamily.com/", ], 'favIcon': "https://dnsforfamily.com/Icons/favicon-32x32.png", 'logo': "https://dnsforfamily.com/Templates/assets/images/logo.svg", 'description': "DNS for Family aims to block websites which are considered as porn. " "So that you and your children can safely surf internet to their heart's desire, " "without worrying to be followed to harmful websites." } } opts = { } optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "CO_HOSTED_SITE" ] def producedEvents(self): return [ "BLACKLISTED_INTERNET_NAME", "BLACKLISTED_AFFILIATE_INTERNET_NAME", "BLACKLISTED_COHOST", ] def queryAddr(self, qaddr): if not qaddr: return None res = dns.resolver.Resolver() res.nameservers = ["94.130.180.225", "78.47.64.161"] try: return res.resolve(qaddr) except Exception: self.debug(f"Unable to resolve {qaddr}") return None def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: return self.results[eventData] = True if eventName == "INTERNET_NAME": blacklist_type = "BLACKLISTED_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": blacklist_type = "BLACKLISTED_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": blacklist_type = "BLACKLISTED_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") return res = self.queryAddr(eventData) if not res: return for result in res: k = str(result) if k != '159.69.10.249': continue self.debug(f"{eventData} blocked by DNS for Family") evt = SpiderFootEvent(blacklist_type, f"DNS for Family [{eventData}]", self.__name__, event) self.notifyListeners(evt) # End of sfp_dns_for_family class ================================================ FILE: modules/sfp_dnsbrute.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_dnsbrute # Purpose: SpiderFoot plug-in for attempting to resolve through brute-forcing # common hostnames. # # Author: Steve Micallef # # Created: 06/07/2017 # Copyright: (c) Steve Micallef 2017 # Licence: MIT # ------------------------------------------------------------------------------- import importlib import random import threading import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_dnsbrute(SpiderFootPlugin): meta = { 'name': "DNS Brute-forcer", 'summary': "Attempts to identify hostnames through brute-forcing common names and iterations.", 'flags': [], 'useCases': ["Footprint", "Investigate"], 'categories': ["DNS"] } # Default options opts = { "skipcommonwildcard": True, "domainonly": True, "commons": True, "top10000": False, "numbersuffix": True, "numbersuffixlimit": True, "_maxthreads": 100 } # Option descriptions optdescs = { 'skipcommonwildcard': "If wildcard DNS is detected, don't bother brute-forcing.", 'domainonly': "Only attempt to brute-force names on domain names, not hostnames (some hostnames are also sub-domains).", 'commons': "Try a list of about 750 common hostnames/sub-domains.", 'top10000': "Try a further 10,000 common hostnames/sub-domains. Will make the scan much slower.", 'numbersuffix': "For any host found, try appending 1, 01, 001, -1, -01, -001, 2, 02, etc. (up to 10)", 'numbersuffixlimit': "Limit using the number suffixes for hosts that have already been resolved? If disabled this will significantly extend the duration of scans.", "_maxthreads": "Maximum threads" } events = None sublist = None lock = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.sublist = self.tempStorage() self.events = self.tempStorage() self.__dataSource__ = "DNS" self.lock = threading.Lock() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] if self.opts['commons']: with importlib.resources.open_text('spiderfoot.dicts', 'subdomains.txt') as f: for s in f.readlines(): s = s.strip() self.sublist[s] = True if self.opts['top10000']: with importlib.resources.open_text('spiderfoot.dicts', 'subdomains-10000.txt') as f: for s in f.readlines(): s = s.strip() self.sublist[s] = True # What events is this module interested in for input def watchedEvents(self): ret = ['DOMAIN_NAME'] if not self.opts['domainonly'] or self.opts['numbersuffix']: ret.append('INTERNET_NAME') return ret # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["INTERNET_NAME"] def tryHost(self, name): try: if self.sf.resolveHost(name) or self.sf.resolveHost6(name): with self.lock: self.hostResults[name] = True except Exception: with self.lock: self.hostResults[name] = False def tryHostWrapper(self, hostList, sourceEvent): self.hostResults = dict() running = True i = 0 t = [] # Spawn threads for scanning self.info("Spawning threads to check hosts: " + str(hostList)) for name in hostList: tn = 'thread_sfp_dnsbrute_' + str(random.SystemRandom().randint(1, 999999999)) t.append(threading.Thread(name=tn, target=self.tryHost, args=(name,))) t[i].start() i += 1 # Block until all threads are finished while running: found = False for rt in threading.enumerate(): if rt.name.startswith("thread_sfp_dnsbrute_"): found = True if not found: running = False time.sleep(0.05) for res in self.hostResults: if self.hostResults.get(res, False): self.sendEvent(sourceEvent, res) # Store the result internally and notify listening modules def sendEvent(self, source, result): self.info("Found a brute-forced host: " + result) # Report the host evt = SpiderFootEvent("INTERNET_NAME", result, self.__name__, source) self.notifyListeners(evt) # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data eventDataHash = self.sf.hashstring(eventData) self.debug(f"Received event, {eventName}, from {srcModuleName}") if srcModuleName == "sfp_dnsbrute": return if eventDataHash in self.events: return self.events[eventDataHash] = True if eventName == "INTERNET_NAME" and not self.getTarget().matches(eventData, includeChildren=False): if not self.opts['numbersuffix']: return if self.checkForStop(): return h, dom = eventData.split(".", 1) # Try resolving common names wildcard = self.sf.checkDnsWildcard(dom) if self.opts['skipcommonwildcard'] and wildcard: self.debug("Wildcard DNS detected on " + dom + " so skipping host iteration.") return dom = "." + dom nextsubs = dict() for i in range(10): nextsubs[h + str(i) + dom] = True nextsubs[h + "0" + str(i) + dom] = True nextsubs[h + "00" + str(i) + dom] = True nextsubs[h + "-" + str(i) + dom] = True nextsubs[h + "-0" + str(i) + dom] = True nextsubs[h + "-00" + str(i) + dom] = True self.tryHostWrapper(list(nextsubs.keys()), event) # The rest of the module is for handling targets only return # Only for the target, from this point forward... if not self.getTarget().matches(eventData, includeChildren=False): return # Try resolving common names self.debug("Iterating through possible sub-domains.") wildcard = self.sf.checkDnsWildcard(eventData) if self.opts['skipcommonwildcard'] and wildcard: self.debug("Wildcard DNS detected.") return targetList = list() for sub in self.sublist: if self.checkForStop(): return name = f"{sub}.{eventData}" if len(targetList) <= self.opts['_maxthreads']: targetList.append(name) else: self.tryHostWrapper(targetList, event) targetList = list() # Scan whatever may be left over. if len(targetList) > 0: self.tryHostWrapper(targetList, event) if self.opts['numbersuffix'] and not self.opts['numbersuffixlimit']: nextsubs = dict() dom = "." + eventData for s in self.sublist: if self.checkForStop(): return for i in range(10): nextsubs[s + str(i) + dom] = True nextsubs[s + "0" + str(i) + dom] = True nextsubs[s + "00" + str(i) + dom] = True nextsubs[s + "-" + str(i) + dom] = True nextsubs[s + "-0" + str(i) + dom] = True nextsubs[s + "-00" + str(i) + dom] = True if len(list(nextsubs.keys())) >= self.opts['_maxthreads']: self.tryHostWrapper(list(nextsubs.keys()), event) nextsubs = dict() # Scan whatever may be left over. if len(nextsubs) > 0: self.tryHostWrapper(list(nextsubs.keys()), event) # End of sfp_dnsbrute class ================================================ FILE: modules/sfp_dnscommonsrv.py ================================================ # -*- coding: utf-8 -*- # ----------------------------------------------------------------------------- # Name: sfp_dnscommonsrv # Purpose: SpiderFoot plug-in for attempting to resolve through # brute-forcing common DNS SRV records. # # Author: Michael Scherer # # Created: 22/08/2017 # Copyright: (c) Michael Scherer 2017 # Licence: MIT # ----------------------------------------------------------------------------- import dns.resolver from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_dnscommonsrv(SpiderFootPlugin): meta = { 'name': "DNS Common SRV", 'summary': "Attempts to identify hostnames through brute-forcing common DNS SRV records.", 'flags': ["slow"], 'useCases': ["Footprint", "Investigate"], 'categories': ["DNS"] } opts = {} optdescs = {} events = None commonsrv = [ # LDAP/Kerberos, used for Active Directory # https://technet.microsoft.com/en-us/library/cc961719.aspx '_ldap._tcp', '_gc._msdcs', '_ldap._tcp.pdc._msdcs', '_ldap._tcp.gc._msdcs', '_kerberos._tcp.dc._msdcs', '_kerberos._tcp', '_kerberos._udp', '_kerberos-master._tcp', '_kerberos-master._udp', '_kpasswd._tcp', '_kpasswd._udp', '_ntp._udp', # SIP '_sip._tcp', '_sip._udp', '_sip._tls', '_sips._tcp', # STUN # https://tools.ietf.org/html/rfc5389 '_stun._tcp', '_stun._udp', '_stuns._tcp', # TURN # https://tools.ietf.org/html/rfc5928 '_turn._tcp', '_turn._udp', '_turns._tcp', # XMPP # http://xmpp.org/rfcs/rfc6120.html '_jabber._tcp', '_xmpp-client._tcp', '_xmpp-server._tcp' ] def setup(self, sfc, userOpts=dict()): self.sf = sfc self.events = self.tempStorage() self.__dataSource__ = "DNS" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ['INTERNET_NAME', 'DOMAIN_NAME'] def producedEvents(self): return ["INTERNET_NAME", "AFFILIATE_INTERNET_NAME"] def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if srcModuleName == "sfp_dnscommonsrv": self.debug(f"Ignoring {eventName}, from self.") return eventDataHash = self.sf.hashstring(eventData) parentEvent = event if eventDataHash in self.events: return self.events[eventDataHash] = True res = dns.resolver.Resolver() if self.opts.get('_dnsserver', "") != "": res.nameservers = [self.opts['_dnsserver']] self.debug("Iterating through possible SRV records.") # Try resolving common names for srv in self.commonsrv: if self.checkForStop(): return name = srv + "." + eventData # Skip hosts we've processed already if self.sf.hashstring(name) in self.events: continue try: answers = res.query(name, 'SRV', timeout=10) except Exception: answers = [] if not answers: continue evt = SpiderFootEvent( "DNS_SRV", name, self.__name__, parentEvent ) self.notifyListeners(evt) for a in answers: # Strip off the trailing . tgt_clean = a.target.to_text().rstrip(".") if self.getTarget().matches(tgt_clean): evt_type = "INTERNET_NAME" else: evt_type = "AFFILIATE_INTERNET_NAME" evt = SpiderFootEvent( evt_type, tgt_clean, self.__name__, parentEvent ) self.notifyListeners(evt) # End of sfp_dnscommonsrv class ================================================ FILE: modules/sfp_dnsdb.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_dnsdb # Purpose: SpiderFoot plug-in that resolves and gets history of domains and IPs # # Author: Filip Aleksić # # Created: 2020-09-09 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import re import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_dnsdb(SpiderFootPlugin): meta = { "name": "DNSDB", "summary": "Query FarSight's DNSDB for historical and passive DNS data.", 'flags': ["apikey"], "useCases": ["Passive", "Footprint", "Investigate"], "categories": ["Passive DNS"], "dataSource": { "website": "https://www.farsightsecurity.com", "model": "FREE_AUTH_LIMITED", "references": [ "https://docs.dnsdb.info/dnsdb-apiv2/", "https://www.farsightsecurity.com/get-started/" "https://www.farsightsecurity.com/solutions/dnsdb/", ], "apiKeyInstructions": [ "Visit https://www.farsightsecurity.com/get-started/", "Select the model that best fit your needs (free or premium)", "Fill in the form to get API key", "Check your email for your API Key ", ], "favIcon": "https://www.farsightsecurity.com/favicon.ico", "logo": "https://www.farsightsecurity.com/assets/media/svg/farsight-logo.svg", "description": "Farsight Security’s DNSDB is the world’s largest " "database of DNS resolution and change data. Started in 2010 and " "updated in real-time, DNSDB provides the most comprehensive " "history of domains and IP addresses worldwide.", }, } opts = { "api_key": "", "age_limit_days": 0, "verify": True, "cohostsamedomain": False, "maxcohost": 100, } optdescs = { "api_key": "DNSDB API Key.", "age_limit_days": "Ignore any DNSDB records older than this many days. 0 = unlimited.", "verify": "Verify co-hosts are valid by checking if they still resolve to the shared IP.", "cohostsamedomain": "Treat co-hosted sites on the same target domain as co-hosting?", "maxcohost": "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.", } results = None errorState = False cohostcount = 0 def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.cohostcount = 0 for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["IP_ADDRESS", "IPV6_ADDRESS", "DOMAIN_NAME"] # What events this module produces def producedEvents(self): return [ "RAW_RIR_DATA", "INTERNET_NAME", "INTERNET_NAME_UNRESOLVED", "PROVIDER_DNS", "DNS_TEXT", "PROVIDER_MAIL", "IP_ADDRESS", "IPV6_ADDRESS", "CO_HOSTED_SITE", ] def query(self, endpoint, queryType, query): if endpoint not in ("rrset", "rdata"): self.error(f"Endpoint MUST be rrset or rdata, you sent {endpoint}") return None if queryType not in ("name", "ip"): self.error(f"Query type MUST be name or ip, you sent {queryType}") return None headers = {"Accept": "application/x-ndjson", "X-API-Key": self.opts["api_key"]} res = self.sf.fetchUrl( f"https://api.dnsdb.info/dnsdb/v2/lookup/{endpoint}/{queryType}/{query}", timeout=30, useragent="SpiderFoot", headers=headers, ) if res["code"] == "429": self.error("You are being rate-limited by DNSDB") self.errorState = True return None if res["content"] is None: self.info(f"No DNSDB record found for {query}") return None splittedContent = res["content"].strip().split("\n") if len(splittedContent) == 2: self.info(f"No DNSDB record found for {query}") return None if len(splittedContent) < 2: self.info(f"Unexpected DNSDB response {query}") return None try: records = [] for content in splittedContent: records.append(json.loads(content)) except json.JSONDecodeError as e: self.error(f"Error processing JSON response from DNSDB: {e}") return None return records[1:-1] def isTooOld(self, lastSeen): ageLimitTs = int(time.time()) - (86400 * self.opts["age_limit_days"]) if self.opts["age_limit_days"] > 0 and lastSeen < ageLimitTs: self.debug("Record found but too old, skipping.") return True return False def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.error("You enabled sfp_dnsdb but did not set an API key!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True responseData = set() coHosts = set() if eventName == "DOMAIN_NAME": rrsetRecords = self.query("rrset", "name", eventData) if rrsetRecords is None: return evt = SpiderFootEvent("RAW_RIR_DATA", str(rrsetRecords), self.__name__, event) self.notifyListeners(evt) for record in rrsetRecords: record = record.get("obj") if self.checkForStop(): return if self.isTooOld(record.get("time_last", 0)): continue if record.get("rrtype") not in ( "A", "AAAA", "MX", "NS", "TXT", "CNAME", ): continue for data in record.get("rdata"): data = data.rstrip(".") if data in responseData: continue responseData.add(data) if record.get("rrtype") == "A": if not self.sf.validIP(data): self.debug(f"Skipping invalid IP address {data}") continue if self.opts["verify"] and not self.sf.validateIP( eventData, data ): self.debug( f"Host {eventData} no longer resolves to {data}" ) continue evt = SpiderFootEvent("IP_ADDRESS", data, self.__name__, event) if record.get("rrtype") == "AAAA": if not self.getTarget().matches( data, includeChildren=True, includeParents=True ): continue if not self.sf.validIP6(data): self.debug("Skipping invalid IPv6 address " + data) continue if self.opts["verify"] and not self.sf.validateIP( eventData, data ): self.debug( "Host " + eventData + " no longer resolves to " + data ) continue evt = SpiderFootEvent("IPV6_ADDRESS", data, self.__name__, event) elif record.get("rrtype") == "MX": data = re.sub(r'.*\s+(.*)', r'\1', data) evt = SpiderFootEvent("PROVIDER_MAIL", data, self.__name__, event) elif record.get("rrtype") == "NS": evt = SpiderFootEvent("PROVIDER_DNS", data, self.__name__, event) elif record.get("rrtype") == "TXT": data = data.replace('"', '') evt = SpiderFootEvent("DNS_TEXT", data, self.__name__, event) elif record.get("rrtype") == "CNAME": if not self.getTarget().matches(data): coHosts.add(data) self.notifyListeners(evt) rdataRecords = self.query("rdata", "name", eventData) if rdataRecords is None: return evt = SpiderFootEvent("RAW_RIR_DATA", str(rdataRecords), self.__name__, event) self.notifyListeners(evt) for record in rdataRecords: record = record.get("obj") if self.isTooOld(record.get("time_last", 0)): continue if record.get("rrtype") not in ("NS", "CNAME"): continue data = record.get("rrname").rstrip(".") if data in responseData: continue responseData.add(data) if record.get("rrtype") == "NS": evt = SpiderFootEvent("PROVIDER_DNS", data, self.__name__, event) elif record.get("rrtype") == "CNAME": if not self.getTarget().matches(data): coHosts.add(data) elif eventName in ("IP_ADDRESS", "IPV6_ADDRESS"): rdataRecords = self.query("rdata", "ip", eventData) if rdataRecords is None: return evt = SpiderFootEvent("RAW_RIR_DATA", str(rdataRecords), self.__name__, event) self.notifyListeners(evt) for record in rdataRecords: record = record.get("obj") if self.checkForStop(): return if self.isTooOld(record.get("time_last", 0)): continue if record.get("rrtype") not in ("A", "AAAA"): continue data = record.get("rrname").rstrip(".") if data in responseData: continue responseData.add(data) if not self.getTarget().matches(data): coHosts.add(data) continue if self.opts["verify"] and not self.sf.resolveHost(data) and not self.sf.resolveHost6(data): self.debug(f"Host {data} could not be resolved") evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", data, self.__name__, event) else: evt = SpiderFootEvent("INTERNET_NAME", data, self.__name__, event) self.notifyListeners(evt) for co in coHosts: if eventName == "IP_ADDRESS" and ( self.opts["verify"] and not self.sf.validateIP(co, eventData) ): self.debug("Host no longer resolves to our IP.") continue if not self.opts["cohostsamedomain"]: if self.getTarget().matches(co, includeParents=True): self.debug( "Skipping " + co + " because it is on the same domain." ) continue if self.cohostcount < self.opts["maxcohost"]: evt = SpiderFootEvent("CO_HOSTED_SITE", co, self.__name__, event) self.notifyListeners(evt) self.cohostcount += 1 # End of sfp_dnsdb class ================================================ FILE: modules/sfp_dnsdumpster.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_dnsdumpster # Purpose: SpiderFoot plug-in for subdomain enumeration using # dnsdumpster.com # # Author: TheTechromancer # # Created: 05/21/2021 # Copyright: (c) Steve Micallef 2021 # Licence: MIT # ------------------------------------------------------------------------------- import re from bs4 import BeautifulSoup from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_dnsdumpster(SpiderFootPlugin): meta = { "name": "DNSDumpster", "summary": "Passive subdomain enumeration using HackerTarget's DNSDumpster", "useCases": ["Investigate", "Footprint", "Passive"], "categories": ["Passive DNS"], "dataSource": { "website": "https://dnsdumpster.com/", "model": "FREE_NOAUTH_UNLIMITED", "description": "DNSdumpster.com is a FREE domain research tool that can discover hosts related to a domain.", } } # Default options opts = {} # Option descriptions optdescs = {} def setup(self, sfc, userOpts=dict()): self.sf = sfc self.debug("Setting up sfp_dnsdumpster") self.results = self.tempStorage() self.opts.update(userOpts) def watchedEvents(self): return ["DOMAIN_NAME", "INTERNET_NAME"] def producedEvents(self): return ["INTERNET_NAME", "INTERNET_NAME_UNRESOLVED"] def query(self, domain): ret = [] # first, get the CSRF tokens url = "https://dnsdumpster.com" res1 = self.sf.fetchUrl( url, useragent=self.opts.get("_useragent", "Spiderfoot") ) if res1["code"] not in ["200"]: self.error(f"Bad response code \"{res1['code']}\" from DNSDumpster") else: self.debug(f"Valid response code \"{res1['code']}\" from DNSDumpster") html = BeautifulSoup(str(res1["content"]), features="lxml") csrftoken = None csrfmiddlewaretoken = None try: for cookie in res1["headers"].get("set-cookie", "").split(";"): k, v = cookie.split('=', 1) if k == "csrftoken": csrftoken = str(v) csrfmiddlewaretoken = html.find("input", {"name": "csrfmiddlewaretoken"}).attrs.get("value", None) except Exception: pass # Abort if we didn't get the tokens if not csrftoken or not csrfmiddlewaretoken: self.error("Error obtaining CSRF tokens") self.errorState = True return ret self.debug("Successfully obtained CSRF tokens") # Otherwise, do the needful url = "https://dnsdumpster.com/" subdomains = set() res2 = self.sf.fetchUrl( url, cookies={ "csrftoken": csrftoken }, postData={ "csrfmiddlewaretoken": csrfmiddlewaretoken, "targetip": str(domain).lower(), "user": "free" }, headers={ "origin": "https://dnsdumpster.com", "referer": "https://dnsdumpster.com/" }, useragent=self.opts.get("_useragent", "Spiderfoot") ) if res2["code"] not in ["200"]: self.error(f"Bad response code \"{res2['code']}\" from DNSDumpster") return ret html = BeautifulSoup(str(res2["content"]), features="lxml") escaped_domain = re.escape(domain) match_pattern = re.compile(r"^[\w\.-]+\." + escaped_domain + r"$") for subdomain in html.findAll(text=match_pattern): subdomains.add(str(subdomain).strip().lower()) return list(subdomains) def sendEvent(self, source, host): if self.sf.resolveHost(host) or self.sf.resolveHost6(host): e = SpiderFootEvent("INTERNET_NAME", host, self.__name__, source) else: e = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host, self.__name__, source) self.notifyListeners(e) def handleEvent(self, event): query = str(event.data).lower() self.debug(f"Received event, {event.eventType}, from {event.module}") # skip if we've already processed this event (or its parent domain/subdomain) target = self.getTarget() eventDataHash = self.sf.hashstring(query) if eventDataHash in self.results or \ (target.matches(query, includeParents=True) and not target.matches(query, includeChildren=False)): self.debug(f"Skipping already-processed event, {event.eventType}, from {event.module}") return self.results[eventDataHash] = True for hostname in self.query(query): if target.matches(hostname, includeParents=True) and not \ target.matches(hostname, includeChildren=False): self.sendEvent(event, hostname) else: self.debug(f"Invalid subdomain: {hostname}") ================================================ FILE: modules/sfp_dnsgrep.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_dnsgrep # Purpose: SpiderFoot plug-in for retrieving domain names # from Rapid7 Sonar Project data sets using DNSGrep API. # - https://opendata.rapid7.com/about/ # - https://blog.erbbysam.com/index.php/2019/02/09/dnsgrep/ # - https://github.com/erbbysam/DNSGrep # # Author: # # Created: 2020-03-14 # Copyright: (c) bcoles 2020 # Licence: MIT # ------------------------------------------------------------------------------- import json import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_dnsgrep(SpiderFootPlugin): meta = { 'name': "DNSGrep", 'summary': "Obtain Passive DNS information from Rapid7 Sonar Project using DNSGrep API.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Passive DNS"], 'dataSource': { 'website': "https://opendata.rapid7.com/", 'model': "FREE_AUTH_UNLIMITED", 'references': [ "https://opendata.rapid7.com/apihelp/", "https://www.rapid7.com/about/research" ], 'apiKeyInstructions': [ "Visit https://opendata.rapid7.com/apihelp/", "Submit form requesting for access", "After getting access, navigate to https://insight.rapid7.com/platform#/apiKeyManagement", "Create an User Key", "The API key will be listed after creation" ], 'favIcon': "https://www.rapid7.com/includes/img/favicon.ico", 'logo': "https://www.rapid7.com/includes/img/Rapid7_logo.svg", 'description': "Offering researchers and community members open access to data from Project Sonar, " "which conducts internet-wide surveys to gain insights into global exposure " "to common vulnerabilities.", } } # Default options opts = { 'timeout': 30, 'dns_resolve': True } # Option descriptions optdescs = { 'timeout': "Query timeout, in seconds.", 'dns_resolve': "DNS resolve each identified domain." } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in userOpts.keys(): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME"] # What events this module produces def producedEvents(self): return ["INTERNET_NAME", "INTERNET_NAME_UNRESOLVED", "RAW_RIR_DATA"] # Query the DNSGrep REST API def query(self, qry): params = { 'q': '.' + qry.encode('raw_unicode_escape').decode("ascii", errors='replace') } res = self.sf.fetchUrl('https://dns.bufferover.run/dns?' + urllib.parse.urlencode(params), timeout=self.opts['timeout'], useragent=self.opts['_useragent']) if res['content'] is None: self.info("No results found for " + qry) return None if res['code'] != '200': self.debug("Error retrieving search results for " + qry) return None try: return json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from DNSGrep: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") data = self.query(eventData) if data is None: self.info("No DNS records found for " + eventData) return evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) domains = list() # Forward DNS A records fdns = data.get("FDNS_A") if fdns: for r in fdns: try: ip, domain = r.split(',') except Exception: continue domains.append(domain) # Reverse DNS records rdns = data.get("RDNS") if rdns: for r in rdns: try: ip, domain = r.split(',') except Exception: continue domains.append(domain) for domain in domains: if domain in self.results: continue if not self.getTarget().matches(domain, includeParents=True): continue evt_type = "INTERNET_NAME" if self.opts["dns_resolve"] and not self.sf.resolveHost(domain) and not self.sf.resolveHost6(domain): self.debug(f"Host {domain} could not be resolved") evt_type += "_UNRESOLVED" evt = SpiderFootEvent(evt_type, domain, self.__name__, event) self.notifyListeners(evt) # End of sfp_dnsgrep class ================================================ FILE: modules/sfp_dnsneighbor.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_dnsneighbor # Purpose: SpiderFoot plug-in for gathering IP addresses from sub-domains # and hostnames identified, and optionally affiliates. # # Author: Steve Micallef # # Created: 07/07/2017 # Copyright: (c) Steve Micallef 2017 # Licence: MIT # ------------------------------------------------------------------------------- import ipaddress from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_dnsneighbor(SpiderFootPlugin): meta = { 'name': "DNS Look-aside", 'summary': "Attempt to reverse-resolve the IP addresses next to your target to see if they are related.", 'flags': [], 'useCases': ["Footprint", "Investigate"], 'categories': ["DNS"] } # Default options opts = { 'lookasidebits': 4, 'validatereverse': True } # Option descriptions optdescs = { 'validatereverse': "Validate that reverse-resolved hostnames still resolve back to that IP before considering them as aliases of your target.", 'lookasidebits': "If look-aside is enabled, the netmask size (in CIDR notation) to check. Default is 4 bits (16 hosts)." } events = None domresults = None hostresults = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.events = self.tempStorage() self.domresults = self.tempStorage() self.hostresults = self.tempStorage() self.__dataSource__ = "DNS" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['IP_ADDRESS'] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["AFFILIATE_IPADDR", "IP_ADDRESS"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data eventDataHash = self.sf.hashstring(eventData) addrs = None parentEvent = event self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventDataHash in self.events: return self.events[eventDataHash] = True try: address = ipaddress.ip_address(eventData) netmask = address.max_prefixlen - min(address.max_prefixlen, max(1, int(self.opts.get("lookasidebits")))) network = ipaddress.ip_network(f"{eventData}/{netmask}", strict=False) except ValueError: self.error(f"Invalid IP address received: {eventData}") return self.debug(f"Lookaside max: {network.network_address}, min: {network.broadcast_address}") for ip in network: sip = str(ip) self.debug("Attempting look-aside lookup of: " + sip) if self.checkForStop(): return if sip in self.hostresults or sip == eventData: continue addrs = self.sf.resolveIP(sip) if not addrs: self.debug("Look-aside resolve for " + sip + " failed.") continue # Report addresses that resolve to hostnames on the same # domain or sub-domain as the target. if self.getTarget().matches(sip): affil = False else: affil = True for a in addrs: if self.getTarget().matches(a): affil = False # Generate the event for the look-aside IP, but don't let it re-trigger # this module by adding it to self.events first. self.events[sip] = True ev = self.processHost(sip, parentEvent, affil) if not ev: continue for addr in addrs: if self.checkForStop(): return if addr == sip: continue if self.sf.validIP(addr) or self.sf.validIP6(addr): parent = parentEvent else: # Hostnames from the IP need to be linked to the IP parent = ev if self.getTarget().matches(addr): # Generate an event for the IP, then # let the handling by this module take # care of follow-up processing. self.processHost(addr, parent, False) else: self.processHost(addr, parent, True) def processHost(self, host, parentEvent, affiliate=None): parentHash = self.sf.hashstring(parentEvent.data) if host not in self.hostresults: self.hostresults[host] = [parentHash] else: if parentHash in self.hostresults[host] or parentEvent.data == host: self.debug("Skipping host, " + host + ", already processed.") return None self.hostresults[host] = self.hostresults[host] + [parentHash] self.debug("Found host: " + host) # If the returned hostname is aliased to our # target in some way, flag it as an affiliate if affiliate is None: affil = True if self.getTarget().matches(host): affil = False else: # If the IP the host resolves to is in our # list of aliases, if not self.sf.validIP(host) and not self.sf.validIP6(host): hostips = self.sf.resolveHost(host) if hostips: for hostip in hostips: if self.getTarget().matches(hostip): affil = False break hostips6 = self.sf.resolveHost6(host) if hostips6: for hostip in hostips6: if self.getTarget().matches(hostip): affil = False break else: affil = affiliate if not self.sf.validIP(host): return None if affil: htype = "AFFILIATE_IPADDR" else: htype = "IP_ADDRESS" # If names were found, leave them to sfp_dnsresolve to resolve if not htype: return None # Report the host evt = SpiderFootEvent(htype, host, self.__name__, parentEvent) self.notifyListeners(evt) return evt # End of sfp_dnsneighbor class ================================================ FILE: modules/sfp_dnsraw.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_dnsraw # Purpose: SpiderFoot plug-in for collecting raw DNS records. # Also extracts hostnames from SPF records. # # Author: Steve Micallef # # Created: 07/07/2017 # Copyright: (c) Steve Micallef 2017 # Licence: MIT # ------------------------------------------------------------------------------- import re import dns.query import dns.rdatatype import dns.resolver from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_dnsraw(SpiderFootPlugin): meta = { 'name': "DNS Raw Records", 'summary': "Retrieves raw DNS records such as MX, TXT and others.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["DNS"] } # Default options opts = { 'verify': True, } # Option descriptions optdescs = { 'verify': "Verify identified hostnames resolve." } events = None checked = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.events = self.tempStorage() self.checked = self.tempStorage() self.__dataSource__ = "DNS" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['INTERNET_NAME', 'DOMAIN_NAME', 'DOMAIN_NAME_PARENT'] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["PROVIDER_MAIL", "PROVIDER_DNS", "RAW_DNS_RECORDS", "DNS_TEXT", "DNS_SPF", 'INTERNET_NAME', 'INTERNET_NAME_UNRESOLVED', 'AFFILIATE_INTERNET_NAME', 'AFFILIATE_INTERNET_NAME_UNRESOLVED'] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data eventDataHash = self.sf.hashstring(eventData) parentEvent = event self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventDataHash in self.events: self.debug("Skipping duplicate event for " + eventData) return self.events[eventDataHash] = True self.debug("Gathering DNS records for " + eventData) domains = list() # Process the raw data alone recs = { 'CNAME': r'\S+\s+(?:\d+)?\s+IN\s+CNAME\s+(\S+)\.', 'MX': r'\S+\s+(?:\d+)?\s+IN\s+MX\s+\d+\s+(\S+)\.', 'NS': r'\S+\s+(?:\d+)?\s+IN\s+NS\s+(\S+)\.', 'TXT': r'\S+\s+TXT\s+\"(.[^\"]*)"' } for rec in list(recs.keys()): if self.checkForStop(): return try: req = dns.message.make_query(eventData, dns.rdatatype.from_text(rec)) if self.opts.get('_dnsserver', "") != "": n = self.opts['_dnsserver'] else: ns = dns.resolver.get_default_resolver() n = ns.nameservers[0] res = dns.query.udp(req, n, timeout=30) if not len(res.answer): continue except Exception as e: self.error(f"Failed to obtain DNS response for {eventData} ({e})") continue # Iterate through DNS answers for x in res.answer: if str(x) in self.checked: continue self.checked[str(x)] = True evt = SpiderFootEvent("RAW_DNS_RECORDS", str(x), self.__name__, parentEvent) self.notifyListeners(evt) for rx in list(recs.keys()): self.debug("Checking " + str(x) + " + against " + recs[rx]) pat = re.compile(recs[rx], re.IGNORECASE | re.DOTALL) grps = re.findall(pat, str(x)) if len(grps) == 0: continue for m in grps: self.debug("Matched: " + m) strdata = str(m) if rx == "CNAME": domains.append(strdata.lower()) if rx == "MX": evt = SpiderFootEvent("PROVIDER_MAIL", strdata.lower(), self.__name__, parentEvent) self.notifyListeners(evt) domains.append(strdata.lower()) if rx == "NS": evt = SpiderFootEvent("PROVIDER_DNS", strdata.lower(), self.__name__, parentEvent) self.notifyListeners(evt) domains.append(strdata.lower()) if rx == "TXT": evt = SpiderFootEvent("DNS_TEXT", strdata, self.__name__, parentEvent) self.notifyListeners(evt) if "v=spf" in strdata or "spf2.0/" in strdata: evt = SpiderFootEvent("DNS_SPF", strdata, self.__name__, parentEvent) self.notifyListeners(evt) matches = re.findall(r'include:(.+?) ', strdata, re.IGNORECASE | re.DOTALL) if matches: for domain in matches: if '_' in domain: continue domains.append(domain.lower()) for domain in set(domains): if self.getTarget().matches(domain, includeChildren=True, includeParents=True): evt_type = 'INTERNET_NAME' else: evt_type = 'AFFILIATE_INTERNET_NAME' if self.opts['verify'] and not self.sf.resolveHost(domain) and not self.sf.resolveHost6(domain): self.debug(f"Host {domain} could not be resolved") evt_type += '_UNRESOLVED' evt = SpiderFootEvent(evt_type, domain, self.__name__, parentEvent) self.notifyListeners(evt) # End of sfp_dnsraw class ================================================ FILE: modules/sfp_dnsresolve.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_dnsresolve # Purpose: SpiderFoot plug-in for extracting hostnames from identified data # and resolving them. # # Author: Steve Micallef # # Created: 07/07/2017 # Copyright: (c) Steve Micallef 2017 # Licence: MIT # ------------------------------------------------------------------------------- import re import urllib from netaddr import IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_dnsresolve(SpiderFootPlugin): meta = { 'name': "DNS Resolver", 'summary': "Resolves hosts and IP addresses identified, also extracted from raw content.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["DNS"] } # Default options opts = { 'validatereverse': True, 'skipcommononwildcard': True, 'netblocklookup': True, 'maxnetblock': 24, 'maxv6netblock': 120, } # Option descriptions optdescs = { 'skipcommononwildcard': "If wildcard DNS is detected, only attempt to look up the first common sub-domain from the common sub-domain list.", 'validatereverse': "Validate that reverse-resolved hostnames still resolve back to that IP before considering them as aliases of your target.", 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible hosts on the same target subdomain/domain?", 'maxnetblock': "Maximum owned IPv4 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'maxv6netblock': "Maximum owned IPv6 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)" } events = None domresults = None hostresults = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.events = self.tempStorage() self.domresults = self.tempStorage() self.hostresults = self.tempStorage() self.__dataSource__ = "DNS" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def enrichTarget(self, target): ret = list() # If it's an IP, get the hostname it reverse resolves to self.info("Identifying aliases for specified target(s)") ret = self.resolveTargets(target, self.opts['validatereverse']) if not ret: return target for host in ret: self.debug("Found an alias: " + host) if self.sf.validIP(host): target.setAlias(host, "IP_ADDRESS") elif self.sf.validIP6(host): target.setAlias(host, "IPV6_ADDRESS") else: target.setAlias(host, "INTERNET_NAME") idnahost = host.encode("idna") if idnahost != host: target.setAlias(idnahost.decode('ascii', errors='replace'), "INTERNET_NAME") # If the target was a hostname/sub-domain, we can # add the domain as an alias for the target. But # not if the target was an IP or subnet. # if target.targetType == "INTERNET_NAME": # dom = self.sf.hostDomain(host, self.opts['_internettlds']) # target.setAlias(dom, "INTERNET_NAME") self.info(f"Target aliases identified: {target.targetAliases}") return target def resolveTargets(self, target, validateReverse: bool) -> list: """Resolve alternative names for a given target. Args: target (SpiderFootTarget): target object validateReverse (bool): validate domain names resolve Returns: list: list of domain names and IP addresses """ ret = list() if not target: return ret t = target.targetType v = target.targetValue if t in ["IP_ADDRESS", "IPV6_ADDRESS"]: r = self.sf.resolveIP(v) if r: ret.extend(r) if t == "INTERNET_NAME": r = self.sf.resolveHost(v) if r: ret.extend(r) r = self.sf.resolveHost6(v) if r: ret.extend(r) if t == "NETBLOCK_OWNER": max_netblock = self.opts['maxnetblock'] if IPNetwork(v).prefixlen < max_netblock: self.debug(f"Network size bigger than permitted: {IPNetwork(v).prefixlen} > {max_netblock}") return list(set(ret)) for addr in IPNetwork(v): if self.checkForStop(): return list(set(ret)) ipaddr = str(addr) if ipaddr.split(".")[3] in ['255', '0']: continue if '255' in ipaddr.split("."): continue ret.append(ipaddr) # Add the reverse-resolved hostnames as aliases too names = self.sf.resolveIP(ipaddr) if not names: continue if not validateReverse: ret.extend(names) continue for host in names: chk = self.sf.resolveHost(host) if chk and ipaddr in chk: ret.append(host) if t == "NETBLOCKV6_OWNER": max_netblock = self.opts['maxv6netblock'] if IPNetwork(v).prefixlen < max_netblock: self.debug(f"Network size bigger than permitted: {IPNetwork(v).prefixlen} > {max_netblock}") return list(set(ret)) for addr in IPNetwork(v): if self.checkForStop(): return list(set(ret)) ipaddr = str(addr) ret.append(ipaddr) # Add the reverse-resolved hostnames as aliases too names = self.sf.resolveIP(ipaddr) if not names: continue if not validateReverse: ret.extend(names) continue for host in names: chk = self.sf.resolveHost6(host) if chk and ipaddr in chk: ret.append(host) return list(set(ret)) # What events is this module interested in for input def watchedEvents(self): return [ # Events that need some kind of DNS treatment "CO_HOSTED_SITE", "AFFILIATE_INTERNET_NAME", "NETBLOCK_OWNER", "NETBLOCKV6_OWNER", "IP_ADDRESS", "IPV6_ADDRESS", "INTERNET_NAME", "AFFILIATE_IPADDR", "AFFILIATE_IPV6_ADDRESS", # Events that may contain hostnames in their content "TARGET_WEB_CONTENT", "BASE64_DATA", "AFFILIATE_DOMAIN_WHOIS", "CO_HOSTED_SITE_DOMAIN_WHOIS", "DOMAIN_WHOIS", "NETBLOCK_WHOIS", "LEAKSITE_CONTENT", "RAW_DNS_RECORDS", "RAW_FILE_META_DATA", "RAW_RIR_DATA", "SIMILARDOMAIN_WHOIS", "SSL_CERTIFICATE_RAW", "SSL_CERTIFICATE_ISSUED", "TCP_PORT_OPEN_BANNER", "WEBSERVER_BANNER", "WEBSERVER_HTTPHEADERS" ] # What events this module produces def producedEvents(self): return ["IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "AFFILIATE_IPADDR", "AFFILIATE_IPV6_ADDRESS", "DOMAIN_NAME", "IPV6_ADDRESS", "INTERNAL_IP_ADDRESS", "DOMAIN_NAME_PARENT", "CO_HOSTED_SITE_DOMAIN", "AFFILIATE_DOMAIN_NAME", "INTERNET_NAME_UNRESOLVED"] # Handle events sent to this module def handleEvent(self, event) -> None: eventName = event.eventType srcModuleName = event.module eventData = event.data eventDataHash = self.sf.hashstring(eventData) addrs = None parentEvent = event # Don't be recursive for names if srcModuleName in ["sfp_dnsresolve"] and "_NAME" in eventName: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventDataHash in self.events: self.debug("Skipping duplicate event.") return self.events[eventDataHash] = True # Parse Microsoft workaround "ipv6-literal.net" fake domain for IPv6 UNC paths # For internal use on Windows systems (should not resolve in DNS) if eventData.endswith(".ipv6-literal.net") and eventName == "AFFILIATE_INTERNET_NAME": ipv6 = eventData.split(".ipv6-literal.net")[0].replace('-', ':').replace('s', '%').split('%')[0] if self.sf.validIP6(ipv6): if self.getTarget().matches(ipv6): evt = SpiderFootEvent("IPV6_ADDRESS", ipv6, self.__name__, parentEvent) else: evt = SpiderFootEvent("AFFILIATE_IPV6_ADDRESS", ipv6, self.__name__, parentEvent) self.notifyListeners(evt) return # Convert ARPA in-addr.arpa address to IPv4 address if eventData.endswith(".in-addr.arpa") and eventName == "AFFILIATE_INTERNET_NAME": ipv4 = '.'.join(reversed(eventData.split('.in-addr.arpa')[0].split('.'))) if self.sf.validIP(ipv4): if self.getTarget().matches(ipv4): evt = SpiderFootEvent("IP_ADDRESS", ipv4, self.__name__, parentEvent) else: evt = SpiderFootEvent("AFFILIATE_IPADDR", ipv4, self.__name__, parentEvent) self.notifyListeners(evt) # Simply translates these to their domains if eventName in ["CO_HOSTED_SITE", "AFFILIATE_INTERNET_NAME"]: # If the co-host or affiliate is a domain name, generate # a domain event. if eventName == "AFFILIATE_INTERNET_NAME": ev = "AFFILIATE_DOMAIN_NAME" else: ev = "CO_HOSTED_SITE_DOMAIN" # What we've been provided might be a domain, so report it if self.sf.isDomain(eventData, self.opts['_internettlds']): evt = SpiderFootEvent(ev, eventData, self.__name__, parentEvent) self.notifyListeners(evt) # In case the domain of the provided host is different, report that too dom = self.sf.hostDomain(eventData, self.opts['_internettlds']) if dom and dom != eventData: evt = SpiderFootEvent(ev, dom, self.__name__, parentEvent) self.notifyListeners(evt) # Resolve host names if eventName in ["INTERNET_NAME", "AFFILIATE_INTERNET_NAME"]: addrs = list() addrs.extend(self.sf.resolveHost(eventData)) addrs.extend(self.sf.resolveHost6(eventData)) if not addrs: return addrs.append(eventData) # We now have a set of hosts/IPs to do something with. for addr in set(addrs): if self.checkForStop(): return # IP addresses resolved from hosts are assumed # to be part of the target (non-affiliates), # unless the source event was an AFFILIATE event. affiliate = False if self.getTarget().matches(addr): affiliate = False elif eventName.startswith("AFFILIATE_"): affiliate = True self.processHost(addr, parentEvent, affiliate) # Reverse resolve IP addresses elif eventName in ["IP_ADDRESS", "IPV6_ADDRESS", "AFFILIATE_IPADDR", "AFFILIATE_IPV6_ADDRESS"]: addrs = self.sf.resolveIP(eventData) if not addrs: return addrs.append(eventData) # We now have a set of hosts/IPs to do something with. for addr in set(addrs): if self.checkForStop(): return # IP addresses resolved from hosts are assumed # to be part of the target (non-affiliates), # unless the source event was an AFFILIATE event. affiliate = False if self.getTarget().matches(addr): affiliate = False elif eventName.startswith("AFFILIATE_"): affiliate = True self.processHost(addr, parentEvent, affiliate) elif eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']: if not self.opts['netblocklookup']: return if eventName == 'NETBLOCKV6_OWNER': max_netblock = self.opts['maxv6netblock'] else: max_netblock = self.opts['maxnetblock'] if IPNetwork(eventData).prefixlen < max_netblock: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}") return self.debug(f"Looking up IPs in owned netblock: {eventData}") for ip in IPNetwork(eventData): if self.checkForStop(): return ipaddr = str(ip) # Skip 0 and 255 for IPv4 addresses if self.sf.validIP(ipaddr): if ipaddr.split(".")[3] in ['255', '0']: continue if '255' in ipaddr.split("."): continue addrs = self.sf.resolveIP(ipaddr) if not addrs: continue self.debug(f"Found {len(addrs)} reversed hostnames from {ipaddr} ({addrs})") for addr in addrs: if self.checkForStop(): return # Generate an event for the IP, then # let the handling by this module take # care of follow-up processing. self.processHost(addr, parentEvent, False) # For everything else (ie, raw data), search for IPs/hosts else: # ignore co-hosted sites if eventName == "CO_HOSTED_SITE": return data = urllib.parse.unquote(eventData).lower() # We get literal \n from RAW_RIR_DATA in cases where JSON responses # have been str()'d, breaking interpretation of hostnames. if eventName == 'RAW_RIR_DATA': data = re.sub(r'(\\x[0-f]{2}|\\n|\\r)', '\n', data) for name in self.getTarget().getNames(): if self.checkForStop(): return offset = data.find(name) if offset < 0: continue pat = re.compile(r"[^a-z0-9\-\.]([a-z0-9\-\.]*\." + name + ")", re.DOTALL | re.MULTILINE) while offset >= 0: # If the target was found at the beginning of the content, skip past it if offset == 0: offset += len(name) continue if offset <= 100: # Start from the beginning of the text start = 0 else: # Start looking for a host 100 chars before the target name start = offset - 100 # Get up to 100 bytes before the name to try and get hostnames chunkhost = data[start:(offset + start + len(name) + 1)] matches = None try: matches = re.findall(pat, chunkhost) except Exception as e: self.error(f"Error applying regex to data ({e})") if matches: for match in matches: # Wildcard certs will come in as .blah.blah if match.startswith("."): m = match[1:] else: m = match self.processHost(m, parentEvent, False) offset = data.find(name, start + len(chunkhost)) # Process a host/IP, parentEvent is the event that represents this entity def processHost(self, host, parentEvent, affiliate=None) -> None: parentHash = self.sf.hashstring(parentEvent.data) if host in self.hostresults: if parentHash in self.hostresults[host] or parentEvent.data == host: self.debug(f"Skipping host, {host}, already processed.") return self.hostresults[host] = self.hostresults[host] + [parentHash] else: self.hostresults[host] = [parentHash] self.debug(f"Found host: {host}") # If the returned hostname is aliased to our # target in some way, flag it as an affiliate if affiliate is None: affil = True if self.getTarget().matches(host): affil = False # If the IP the host resolves to is in our # list of aliases, if not self.sf.validIP(host): hostips = self.sf.resolveHost(host) if hostips: for hostip in hostips: if self.getTarget().matches(hostip): affil = False break hostips6 = self.sf.resolveHost6(host) if hostips6: for hostip6 in hostips6: if self.getTarget().matches(hostip6): affil = False break else: affil = affiliate if affil: if self.sf.isValidLocalOrLoopbackIp(host): htype = "INTERNAL_IP_ADDRESS" elif self.sf.validIP(host): htype = "AFFILIATE_IPADDR" elif self.sf.validIP6(host): htype = "AFFILIATE_IPV6_ADDRESS" else: htype = "AFFILIATE_INTERNET_NAME" else: if self.sf.isValidLocalOrLoopbackIp(host): htype = "INTERNAL_IP_ADDRESS" elif self.sf.validIP(host): htype = "IP_ADDRESS" elif self.sf.validIP6(host): htype = "IPV6_ADDRESS" else: htype = "INTERNET_NAME" if htype in ["INTERNET_NAME", "AFFILIATE_INTERNET_NAME"]: if not self.sf.resolveHost(host) and not self.sf.resolveHost6(host): evt = SpiderFootEvent(f"{htype}_UNRESOLVED", host, self.__name__, parentEvent) self.notifyListeners(evt) return # Report the host if host != parentEvent.data: evt = SpiderFootEvent(htype, host, self.__name__, parentEvent) self.notifyListeners(evt) else: evt = parentEvent if htype == "INTERNET_NAME": dom = self.sf.hostDomain(host, self.opts['_internettlds']) if dom: self.processDomain(dom, evt, False, host) # Try obtain the IPv6 address ip6s = self.sf.resolveHost6(host) if not ip6s: return for ip6 in ip6s: parentHash = self.sf.hashstring(evt.data) if ip6 not in self.hostresults: self.hostresults[ip6] = [parentHash] else: if parentHash in self.hostresults[ip6] or evt.data == ip6: self.debug(f"Skipping host, {ip6}, already processed.") continue self.hostresults[ip6] = self.hostresults[ip6] + [parentHash] evt6 = SpiderFootEvent("IPV6_ADDRESS", ip6, self.__name__, evt) self.notifyListeners(evt6) if htype == "AFFILIATE_INTERNET_NAME": dom = self.sf.hostDomain(host, self.opts['_internettlds']) if not dom: return if dom == host and not self.sf.isDomain(dom, self.opts['_internettlds']): return self.processDomain(dom, evt, True, host) def processDomain(self, domainName, parentEvent, affil=False, host=None) -> None: if domainName in self.domresults: self.debug(f"Skipping domain, {domainName}, already processed.") return self.domresults[domainName] = True if affil: domevt = SpiderFootEvent("AFFILIATE_DOMAIN_NAME", domainName, self.__name__, parentEvent) self.notifyListeners(domevt) return if self.getTarget().matches(domainName): domevt = SpiderFootEvent("DOMAIN_NAME", domainName, self.__name__, parentEvent) self.notifyListeners(domevt) else: # Only makes sense to link this event with a source event # that sits on the parent domain. if not host: return if parentEvent.data.endswith("." + domainName): domevt = SpiderFootEvent("DOMAIN_NAME_PARENT", domainName, self.__name__, parentEvent) self.notifyListeners(domevt) # End of sfp_dnsresolve class ================================================ FILE: modules/sfp_dnszonexfer.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_dnszonexfer # Purpose: SpiderFoot plug-in for attempting a DNS zone transfer. # # Author: Steve Micallef # # Created: 22/08/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import re import dns.query import dns.zone from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_dnszonexfer(SpiderFootPlugin): meta = { 'name': "DNS Zone Transfer", 'summary': "Attempts to perform a full DNS zone transfer.", 'flags': [], 'useCases': ["Footprint", "Investigate"], 'categories': ["DNS"] } opts = { "timeout": 30 } optdescs = { "timeout": "Timeout in seconds" } events = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.events = self.tempStorage() self.__dataSource__ = "DNS" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ['PROVIDER_DNS'] def producedEvents(self): return ["RAW_DNS_RECORDS", "INTERNET_NAME"] def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data eventDataHash = self.sf.hashstring(eventData) parentEvent = event self.debug(f"Received event, {eventName}, from {srcModuleName}") if srcModuleName == "sfp_dnszonexfer": self.debug(f"Ignoring {eventName}, from self.") return if eventDataHash in self.events: self.debug("Skipping duplicate event for " + eventData) return self.events[eventDataHash] = True res = dns.resolver.Resolver() if self.opts.get('_dnsserver', "") != "": res.nameservers = [self.opts['_dnsserver']] # Get the name server's IP. This is to avoid DNS leaks # when attempting to resolve the name server during # the zone transfer. if not self.sf.validIP(eventData) and not self.sf.validIP6(eventData): nsips = self.sf.resolveHost(eventData) if not nsips: return if not nsips: self.error("Couldn't resolve the name server, so not attempting zone transfer.") return for n in nsips: if self.sf.validIP(n): nsip = n break else: nsip = eventData for name in self.getTarget().getNames(): self.debug("Trying for name: " + name) try: ret = list() z = dns.zone.from_xfr(dns.query.xfr(nsip, name, timeout=int(self.opts["timeout"]))) names = list(z.nodes.keys()) for n in names: ret.append(z[n].to_text(n)) evt = SpiderFootEvent("RAW_DNS_RECORDS", "\n".join(ret), self.__name__, parentEvent) self.notifyListeners(evt) # Try and pull out individual records for row in ret: pat = re.compile(r"^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL) grps = re.findall(pat, row) if len(grps) > 0: for strdata in grps: self.debug("Matched: " + strdata) if strdata.endswith("."): strdata = strdata[:-1] else: strdata = strdata + "." + name evt = SpiderFootEvent("INTERNET_NAME", strdata, self.__name__, parentEvent) self.notifyListeners(evt) except Exception as e: self.info(f"Unable to perform DNS zone transfer for {eventData} ({name}): {e}") # End of sfp_dnszonexfer class ================================================ FILE: modules/sfp_dronebl.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_dronebl # Purpose: SpiderFoot plug-in for looking up whether IPs/Netblocks/Domains # appear in the DroneBL blocklist, indicating potential open-relays, # open proxies, malicious servers, vulnerable servers, etc. # # Author: Steve Micallef # # Created: 07/01/2014 # Copyright: (c) Steve Micallef 2014 # Licence: MIT # ------------------------------------------------------------------------------- from netaddr import IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_dronebl(SpiderFootPlugin): meta = { 'name': "DroneBL", 'summary': "Query the DroneBL database for open relays, open proxies, vulnerable servers, etc.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://dronebl.org/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://dronebl.org/docs/howtouse", "https://dronebl.org/rpckey_signup", "https://dronebl.org/docs/rpc2" ], 'favIcon': "https://dronebl.org/images/favicon.ico", 'logo': "https://dronebl.org/images/dronebl-logo.svg", 'description': "DroneBL is a realtime monitor of abusable IPs, which has " "the goal of stopping abuse of infected machines.\n" "A real-time tracker of abusable IPs.", } } # Default options opts = { 'netblocklookup': True, 'maxnetblock': 24, 'subnetlookup': True, 'maxsubnet': 24 } # Option descriptions optdescs = { 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?", 'maxnetblock': "If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'subnetlookup': "Look up all IPs on subnets which your target is a part of for blacklisting?", 'maxsubnet': "If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)" } results = None checks = { "127.0.0.3": "dronebl.org - IRC Drone", "127.0.0.5": "dronebl.org - Bottler", "127.0.0.6": "dronebl.org - Unknown spambot or drone", "127.0.0.7": "dronebl.org - DDOS Drone", "127.0.0.8": "dronebl.org - SOCKS Proxy", "127.0.0.9": "dronebl.org - HTTP Proxy", "127.0.0.10": "dronebl.org - ProxyChain", "127.0.0.11": "dronebl.org - Web Page Proxy", "127.0.0.12": "dronebl.org - Open DNS Resolver", "127.0.0.13": "dronebl.org - Brute force attackers", "127.0.0.14": "dronebl.org - Open Wingate Proxy", "127.0.0.15": "dronebl.org - Compromised router / gateway", "127.0.0.16": "dronebl.org - Autorooting worms", "127.0.0.17": "dronebl.org - Automatically determined botnet IPs (experimental)", "127.0.0.18": "dronebl.org - Possibly compromised DNS/MX", "127.0.0.19": "dronebl.org - Abused VPN Service", "127.0.0.255": "dronebl.org - Unknown" } def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ 'IP_ADDRESS', 'AFFILIATE_IPADDR', 'NETBLOCK_OWNER', 'NETBLOCK_MEMBER' ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "BLACKLISTED_SUBNET", "BLACKLISTED_NETBLOCK", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK", "MALICIOUS_SUBNET", "VPN_HOST", "PROXY_HOST" ] # Swap 1.2.3.4 to 4.3.2.1 def reverseAddr(self, ipaddr): if not self.sf.validIP(ipaddr): self.debug(f"Invalid IPv4 address {ipaddr}") return None return '.'.join(reversed(ipaddr.split('.'))) def queryAddr(self, qaddr): """Query DroneBL DNS for an IPv4 address. Args: qaddr (str): IPv4 address. Returns: list: DroneBL DNS entries """ if not self.sf.validIP(qaddr): self.debug(f"Invalid IPv4 address {qaddr}") return None try: lookup = self.reverseAddr(qaddr) + '.dnsbl.dronebl.org' self.debug(f"Checking DroneBL blacklist: {lookup}") return self.sf.resolveHost(lookup) except Exception as e: self.debug(f"DroneBL did not resolve {qaddr} / {lookup}: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: return self.results[eventData] = True if eventName == 'NETBLOCK_OWNER': if not self.opts['netblocklookup']: return max_netblock = self.opts['maxnetblock'] if IPNetwork(eventData).prefixlen < max_netblock: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}") return if eventName == 'NETBLOCK_MEMBER': if not self.opts['subnetlookup']: return max_subnet = self.opts['maxsubnet'] if IPNetwork(eventData).prefixlen < max_subnet: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}") return if eventName == "AFFILIATE_IPADDR": malicious_type = "MALICIOUS_AFFILIATE_IPADDR" blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" elif eventName == "IP_ADDRESS": malicious_type = "MALICIOUS_IPADDR" blacklist_type = "BLACKLISTED_IPADDR" elif eventName == "NETBLOCK_OWNER": malicious_type = "MALICIOUS_NETBLOCK" blacklist_type = "BLACKLISTED_NETBLOCK" elif eventName == "NETBLOCK_MEMBER": malicious_type = "MALICIOUS_SUBNET" blacklist_type = "BLACKLISTED_SUBNET" else: self.debug(f"Unexpected event type {eventName}, skipping") return addrs = list() if eventName.startswith("NETBLOCK_"): for addr in IPNetwork(eventData): addrs.append(str(addr)) else: addrs.append(eventData) for addr in addrs: if self.checkForStop(): return res = self.queryAddr(addr) self.results[addr] = True if not res: continue self.debug(f"{addr} found in DroneBL DNS: {res}") for result in res: k = str(result) if k not in self.checks: if not result.endswith('.dnsbl.dronebl.org'): # This is an error. The "checks" dict may need to be updated. self.error(f"DroneBL resolved address {addr} to unknown IP address {result} not found in DroneBL list.") continue evt = SpiderFootEvent(blacklist_type, f"{self.checks[k]} [{addr}]", self.__name__, event) self.notifyListeners(evt) if k in [ "127.0.0.3", "127.0.0.5", "127.0.0.6", "127.0.0.7", "127.0.0.13", "127.0.0.15", "127.0.0.16", "127.0.0.17", "127.0.0.18", "127.0.0.19", ]: evt = SpiderFootEvent(malicious_type, f"{self.checks[k]} [{addr}]", self.__name__, event) self.notifyListeners(evt) if k in ["127.0.0.8", "127.0.0.9", "127.0.0.10", "127.0.0.11", "127.0.0.14"]: evt = SpiderFootEvent("PROXY_HOST", addr, self.__name__, event) self.notifyListeners(evt) if k == "127.0.0.19": evt = SpiderFootEvent("VPN_HOST", addr, self.__name__, event) self.notifyListeners(evt) # End of sfp_dronebl class ================================================ FILE: modules/sfp_duckduckgo.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_duckduckgo # Purpose: Queries DuckDuckGo's API for information abotut the target. # # Author: Steve Micallef # # Created: 21/07/2015 # Copyright: (c) Steve Micallef 2015 # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_duckduckgo(SpiderFootPlugin): meta = { 'name': "DuckDuckGo", 'summary': "Query DuckDuckGo's API for descriptive information about your target.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://duckduckgo.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://api.duckduckgo.com/api", "https://help.duckduckgo.com/company/partnerships/", "https://help.duckduckgo.com/duckduckgo-help-pages/" ], 'favIcon': "https://duckduckgo.com/favicon.ico", 'logo': "https://duckduckgo.com/assets/icons/meta/DDG-icon_256x256.png", 'description': "Our Instant Answer API gives you free access to many of our instant answers like: " "topic summaries , categories, disambiguation, and !bang redirects.", } } # Default options opts = { "affiliatedomains": True } # Option descriptions optdescs = { "affiliatedomains": "For affiliates, look up the domain name, not the hostname. This will usually return more meaningful information about the affiliate." } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME", "DOMAIN_NAME_PARENT", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["DESCRIPTION_CATEGORY", "DESCRIPTION_ABSTRACT", "AFFILIATE_DESCRIPTION_CATEGORY", "AFFILIATE_DESCRIPTION_ABSTRACT"] def handleEvent(self, event): eventName = event.eventType eventData = event.data if self.opts['affiliatedomains'] and "AFFILIATE_" in eventName: eventData = self.sf.hostDomain(eventData, self.opts['_internettlds']) if not eventData: return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True url = "https://api.duckduckgo.com/?q=" + eventData + "&format=json&pretty=1" res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot") if res['content'] is None: self.error(f"Unable to fetch {url}") return try: ret = json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from DuckDuckGo: {e}") return if not ret['Heading']: self.debug(f"No DuckDuckGo information for {eventData}") return abstract_text = ret.get('AbstractText') if abstract_text: event_type = "DESCRIPTION_ABSTRACT" if "AFFILIATE" in eventName: event_type = "AFFILIATE_" + event_type evt = SpiderFootEvent(event_type, str(abstract_text), self.__name__, event) self.notifyListeners(evt) related_topics = ret.get('RelatedTopics') if related_topics: event_type = "DESCRIPTION_CATEGORY" if "AFFILIATE" in eventName: event_type = "AFFILIATE_" + event_type for topic in related_topics: if not isinstance(topic, dict): self.debug("No category text found from DuckDuckGo.") continue category = topic.get('Text') if not category: self.debug("No category text found from DuckDuckGo.") continue evt = SpiderFootEvent(event_type, category, self.__name__, event) self.notifyListeners(evt) # End of sfp_duckduckgo class ================================================ FILE: modules/sfp_email.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_email # Purpose: SpiderFoot plug-in for scanning retrieved content by other # modules (such as sfp_spider) and identifying e-mail addresses # # Author: Steve Micallef # # Created: 06/04/2012 # Copyright: (c) Steve Micallef 2012 # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_email(SpiderFootPlugin): meta = { 'name': "E-Mail Address Extractor", 'summary': "Identify e-mail addresses in any obtained data.", 'useCases': ["Passive", "Investigate", "Footprint"], 'categories': ["Content Analysis"] } opts = { } optdescs = { } def setup(self, sfc, userOpts=dict()): self.sf = sfc for opt in userOpts.keys(): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["TARGET_WEB_CONTENT", "BASE64_DATA", "AFFILIATE_DOMAIN_WHOIS", "CO_HOSTED_SITE_DOMAIN_WHOIS", "DOMAIN_WHOIS", "NETBLOCK_WHOIS", "LEAKSITE_CONTENT", "RAW_DNS_RECORDS", "RAW_FILE_META_DATA", 'RAW_RIR_DATA', "SIMILARDOMAIN_WHOIS", "SSL_CERTIFICATE_RAW", "SSL_CERTIFICATE_ISSUED", "TCP_PORT_OPEN_BANNER", "WEBSERVER_BANNER", "WEBSERVER_HTTPHEADERS"] # What events this module produces def producedEvents(self): return ["EMAILADDR", "EMAILADDR_GENERIC", "AFFILIATE_EMAILADDR"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") emails = SpiderFootHelpers.extractEmailsFromText(eventData) for email in set(emails): evttype = "EMAILADDR" email = email.lower() # Get the domain and strip potential ending . mailDom = email.split('@')[1].strip('.') if not self.sf.validHost(mailDom, self.opts['_internettlds']): self.debug(f"Skipping {email} as not a valid e-mail.") continue if not self.getTarget().matches(mailDom, includeChildren=True, includeParents=True) and not self.getTarget().matches(email): self.debug("External domain, so possible affiliate e-mail") evttype = "AFFILIATE_EMAILADDR" if eventName.startswith("AFFILIATE_"): evttype = "AFFILIATE_EMAILADDR" if not evttype.startswith("AFFILIATE_") and email.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" self.info(f"Found e-mail address: {email}") mail = email.strip('.') evt = SpiderFootEvent(evttype, mail, self.__name__, event) if event.moduleDataSource: evt.moduleDataSource = event.moduleDataSource else: evt.moduleDataSource = "Unknown" self.notifyListeners(evt) # End of sfp_email class ================================================ FILE: modules/sfp_emailcrawlr.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_emailcrawlr # Purpose: Search EmailCrawlr for email addresses and phone numbers # associated with a domain. # # Authors: # # Created: 2020-06-19 # Copyright: (c) bcoles 2020 # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_emailcrawlr(SpiderFootPlugin): meta = { 'name': "EmailCrawlr", 'summary': "Search EmailCrawlr for email addresses and phone numbers associated with a domain.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://emailcrawlr.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://emailcrawlr.com/docs" ], 'apiKeyInstructions': [ "Visit https://emailcrawlr.com", "Sign up for free account", "Navigate to https://emailcrawlr.com/dashboard", "The API key is listed under 'API Key'" ], 'favIcon': "https://emailcrawlr.com/assets/fav-165eaa698b0dc774f0b250fadb2b41266e4c70dfbd7fb5fd4413e4bdecfd229d.png", 'logo': "https://emailcrawlr.com/assets/logo_black-d136617b2fc5d52df6eea245a4db78477d8d99f873e08c24a9c3c7defe1c1379.png", 'description': "By using the EmailCrawlr JSON API you can: " "Get key information about company websites.\n" "Find all email addresses associated with a domain.\n" "Get social accounts associated with an email.\n" "Verify email address deliverability.", } } # Default options opts = { "api_key": "", "delay": 1, } # Option descriptions optdescs = { "api_key": "EmailCrawlr API key.", "delay": "Delay between requests, in seconds.", } results = None errorState = False # Initialize module and module options def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in userOpts.keys(): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME"] # What events this module produces def producedEvents(self): return ["RAW_RIR_DATA", "EMAILADDR", "EMAILADDR_GENERIC", "PHONE_NUMBER", "GEOINFO"] # Query domain # https://emailcrawlr.com/docs def queryDomain(self, qry): params = { 'domain': qry.encode('raw_unicode_escape').decode("ascii", errors='replace') } headers = { 'Accept': "application/json", 'x-api-key': self.opts['api_key'] } res = self.sf.fetchUrl( f"https://api.emailcrawlr.com/v2/domain?{urllib.parse.urlencode(params)}", headers=headers, timeout=15, useragent=self.opts['_useragent'] ) time.sleep(self.opts['delay']) return self.parseApiResponse(res) # Parse API response def parseApiResponse(self, res: dict): if not res: self.error("No response from EmailCrawlr.") return None if res['code'] == '404': self.debug("No results for query") return None if res['code'] == "401": self.error("Invalid EmailCrawlr API key.") self.errorState = True return None if res['code'] == '429': self.error("You are being rate-limited by EmailCrawlr") self.errorState = True return None if res['code'] == '503': self.error("EmailCrawlr service unavailable") self.errorState = True return None # Catch all other non-200 status codes, and presume something went wrong if res['code'] != '200': self.error("Failed to retrieve content from EmailCrawlr") self.errorState = True return None if res['content'] is None: return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if eventData in self.results: return if self.opts['api_key'] == "": self.error("You enabled sfp_emailcrawlr but did not set an API key!") self.errorState = True return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventName in ["DOMAIN_NAME"]: data = self.queryDomain(eventData) if data is None: self.debug(f"No information found for domain {eventData}") return evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) emails = data.get("emails") if not emails: self.info(f"No emails found for domain {eventData}") return for res in emails: email = res.get('email') if email: mail_domain = email.lower().split('@')[1] if self.getTarget().matches(mail_domain, includeChildren=True): if email.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" evt = SpiderFootEvent(evttype, email, self.__name__, event) self.notifyListeners(evt) name = res.get('name') if name: full_name = name.get('name') if full_name and len(full_name) > 3: evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {full_name}", self.__name__, event) self.notifyListeners(evt) phone_numbers = res.get('numbers') if phone_numbers: for number in phone_numbers: if number: evt = SpiderFootEvent("PHONE_NUMBER", number, self.__name__, event) self.notifyListeners(evt) location = res.get('location') if location: loc = ', '.join([_f for _f in [location.get('city'), location.get('country')] if _f]) if loc: evt = SpiderFootEvent("GEOINFO", loc, self.__name__, event) self.notifyListeners(evt) # End of sfp_emailcrawlr class ================================================ FILE: modules/sfp_emailformat.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_emailformat # Purpose: SpiderFoot plug-in for retrieving e-mail addresses # belonging to your target from email-format.com. # # Author: # # Created: 29/09/2018 # Copyright: (c) bcoles 2018 # Licence: MIT # ------------------------------------------------------------------------------- import re from bs4 import BeautifulSoup from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_emailformat(SpiderFootPlugin): meta = { 'name': "EmailFormat", 'summary': "Look up e-mail addresses on email-format.com.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://www.email-format.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://www.email-format.com/i/api_access/", "https://www.email-format.com/i/api_v2/", "https://www.email-format.com/i/api_v1/" ], 'favIcon': "https://www.google.com/s2/favicons?domain=https://www.email-format.com/", 'logo': "https://www.google.com/s2/favicons?domain=https://www.email-format.com/", 'description': "Save time and energy - find the email address formats in use at thousands of companies.", } } opts = { } optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ['INTERNET_NAME', "DOMAIN_NAME"] def producedEvents(self): return ["EMAILADDR", "EMAILADDR_GENERIC"] def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") # Get e-mail addresses on this domain res = self.sf.fetchUrl(f"https://www.email-format.com/d/{eventData}/", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if res['content'] is None: return html = BeautifulSoup(res["content"], features="lxml") if not html: return tbody = html.find('tbody') if tbody: data = str(tbody.contents) else: # fall back to raw page contents data = res["content"] emails = SpiderFootHelpers.extractEmailsFromText(data) for email in emails: # Skip unrelated emails mailDom = email.lower().split('@')[1] if not self.getTarget().matches(mailDom): self.debug(f"Skipped address: {email}") continue # Skip masked emails if re.match(r"^[0-9a-f]{8}\.[0-9]{7}@", email): self.debug(f"Skipped address: {email}") continue self.info(f"Found e-mail address: {email}") if email.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" evt = SpiderFootEvent(evttype, email, self.__name__, event) self.notifyListeners(evt) # End of sfp_emailformat class ================================================ FILE: modules/sfp_emailrep.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_emailrep # Purpose: Searches EmailRep.io for email address reputation. # # Author: # # Created: 2019-08-07 # Copyright: (c) bcoles 2019 # Licence: MIT # ------------------------------------------------------------------------------- import json import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_emailrep(SpiderFootPlugin): meta = { 'name': "EmailRep", 'summary': "Search EmailRep.io for email address reputation.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://emailrep.io/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://docs.emailrep.io/" ], 'apiKeyInstructions': [ "Visit https://emailrep.io/free", "Request a free API Key", "The API key will be sent to registered email account on approval" ], 'favIcon': "https://emailrep.io/assets/img/favicon.png", 'logo': "https://emailrep.io/assets/img/logo-light.png", 'description': "Illuminate the \"reputation\" behind an email address.\n" "EmailRep uses hundreds of factors like domain age, traffic rankings, " "presence on social media sites, professional networking sites, personal connections, " "public records, deliverability, data breaches, dark web credential leaks, " "phishing emails, threat actor emails, and more to answer these types of questions.", } } opts = { 'api_key': '', } optdescs = { 'api_key': 'EmailRep API key.', } results = None errorState = False errorWarned = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ['EMAILADDR'] def producedEvents(self): return ['RAW_RIR_DATA', 'EMAILADDR_COMPROMISED', 'MALICIOUS_EMAILADDR'] # https://emailrep.io/docs/ def query(self, qry): headers = { 'Accept': "application/json" } if self.opts['api_key'] != '': headers['Key'] = self.opts['api_key'] res = self.sf.fetchUrl( 'https://emailrep.io/' + qry, headers=headers, useragent='SpiderFoot', timeout=self.opts['_fetchtimeout'] ) # Documentation does not indicate rate limit threshold (50 queries/day) time.sleep(1) if res['content'] is None: return None if res['code'] == '400': self.error('API error: Bad request') self.errorState = True return None if res['code'] == '401': self.error('API error: Invalid API key') self.errorState = True return None if res['code'] == '429': self.error('API error: Too Many Requests') self.errorState = True return None if res['code'] != '200': self.error('Unexpected reply from EmailRep.io: ' + res['code']) self.errorState = True return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == '' and not self.errorWarned: self.error("Warning: You enabled sfp_emailrep but did not set an API key! Queries will be rate limited.") self.errorWarned = True res = self.query(eventData) if res is None: return details = res.get('details') if not details: return credentials_leaked = details.get('credentials_leaked') if credentials_leaked: evt = SpiderFootEvent('EMAILADDR_COMPROMISED', eventData + " [Unknown]", self.__name__, event) self.notifyListeners(evt) malicious_activity = details.get('malicious_activity') if malicious_activity: evt = SpiderFootEvent('MALICIOUS_EMAILADDR', 'EmailRep [' + eventData + ']', self.__name__, event) self.notifyListeners(evt) if malicious_activity or credentials_leaked: evt = SpiderFootEvent('RAW_RIR_DATA', str(res), self.__name__, event) self.notifyListeners(evt) # End of sfp_emailrep class ================================================ FILE: modules/sfp_emergingthreats.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_emergingthreats # Purpose: Checks if an IP address or netblock is malicious according to # EmergingThreats.net. # # Author: steve@binarypool.com # # Created: 16/05/2020 # Copyright: (c) Steve Micallef, 2020 # Licence: MIT # ------------------------------------------------------------------------------- from netaddr import IPAddress, IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_emergingthreats(SpiderFootPlugin): meta = { 'name': "Emerging Threats", 'summary': "Check if a netblock or IP address is malicious according to EmergingThreats.net.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://rules.emergingthreats.net/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://doc.emergingthreats.net/" ], 'favIcon': "https://doc.emergingthreats.net/pub/Main/WebPreferences/favicon.ico", 'logo': "https://doc.emergingthreats.net/logo.png", 'description': "Emerging Threats delivers the most timely and accurate threat intelligence.\n" "Emerging Threat (ET) intelligence helps prevent attacks and reduce risk by " "helping you understand the historical context of where these threats originated, " "who is behind them, when have they attacked, what methods they used, and what they're after. " "Get on-demand access to current and historical metadata on IPs, domains, " "and other related threat intelligence to help research threats and investigate incidents.", } } opts = { 'checkaffiliates': True, 'cacheperiod': 18, 'checknetblocks': True, 'checksubnets': True } optdescs = { 'checkaffiliates': "Apply checks to affiliate IP addresses?", 'cacheperiod': "Hours to cache list data before re-fetching.", 'checknetblocks': "Report if any malicious IPs are found within owned netblocks?", 'checksubnets': "Check if any malicious IPs are found within the same subnet of the target?" } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "AFFILIATE_IPADDR", "NETBLOCK_MEMBER", "NETBLOCK_OWNER", ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "BLACKLISTED_SUBNET", "BLACKLISTED_NETBLOCK", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_SUBNET", "MALICIOUS_NETBLOCK", ] def query(self, qry, targetType): cid = "_emergingthreats" url = "https://rules.emergingthreats.net/blockrules/compromised-ips.txt" data = dict() data["content"] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0)) if data["content"] is None: data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if data["code"] != "200": self.error(f"Unable to fetch {url}") self.errorState = True return None if data["content"] is None: self.error(f"Unable to fetch {url}") self.errorState = True return None self.sf.cachePut("sfmal_" + cid, data['content']) for line in data["content"].split('\n'): ip = line.strip().lower() if targetType == "netblock": try: if IPAddress(ip) in IPNetwork(qry): self.debug(f"{ip} found within netblock/subnet {qry} in EmergingThreats.net list.") return url except Exception as e: self.debug(f"Error encountered parsing: {e}") continue if targetType == "ip": if qry.lower() == ip: self.debug(f"{qry} found in EmergingThreats.net list.") return url return None def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName == 'IP_ADDRESS': targetType = 'ip' malicious_type = "MALICIOUS_IPADDR" blacklist_type = "BLACKLISTED_IPADDR" elif eventName == 'AFFILIATE_IPADDR': if not self.opts.get('checkaffiliates', False): return targetType = 'ip' malicious_type = "MALICIOUS_AFFILIATE_IPADDR" blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" elif eventName == 'NETBLOCK_OWNER': if not self.opts.get('checknetblocks', False): return targetType = 'netblock' malicious_type = "MALICIOUS_NETBLOCK" blacklist_type = "BLACKLISTED_NETBLOCK" elif eventName == 'NETBLOCK_MEMBER': if not self.opts.get('checksubnets', False): return targetType = 'netblock' malicious_type = "MALICIOUS_SUBNET" blacklist_type = "BLACKLISTED_SUBNET" else: self.debug(f"Unexpected event type {eventName}, skipping") return self.debug(f"Checking maliciousness of {eventData} with EmergingThreats.net") url = self.query(eventData, targetType) if not url: return text = f"EmergingThreats.net [{eventData}]\n{url}" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_emergingthreats class ================================================ FILE: modules/sfp_errors.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_errors # Purpose: Identify common error messages in content like SQL errors, etc. # # Author: Steve Micallef # # Created: 18/01/2015 # Copyright: (c) Steve Micallef 2015 # Licence: MIT # ------------------------------------------------------------------------------- import re from spiderfoot import SpiderFootEvent, SpiderFootPlugin # Taken from Google Dorks on exploit-db.com regexps = dict({ "PHP Error": ["PHP pase error", "PHP warning", "PHP error", "unexpected T_VARIABLE", "warning: failed opening", "include_path="], "Generic Error": ["Internal Server Error", "Incorrect syntax"], "Oracle Error": [r"ORA-\d+", "TNS:.?no listen"], "ASP Error": ["NET_SessionId"], "MySQL Error": [r"mysql_query\(", r"mysql_connect\("], "ODBC Error": [r"\[ODBC SQL"] }) class sfp_errors(SpiderFootPlugin): meta = { 'name': "Error String Extractor", 'summary': "Identify common error messages in content like SQL errors, etc.", 'flags': [], 'useCases': ["Footprint", "Passive"], 'categories': ["Content Analysis"] } # Default options opts = {} # Option descriptions optdescs = { # For each option in opts you should have a key/value pair here # describing it. It will end up in the UI to explain the option # to the end-user. } # Target results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.__dataSource__ = "Target Website" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input # * = be notified about all events. def watchedEvents(self): return ["TARGET_WEB_CONTENT"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["ERROR_MESSAGE"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data # We only want web content from the target if srcModuleName != "sfp_spider": return eventSource = event.actualSource self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventSource not in list(self.results.keys()): self.results[eventSource] = list() # We only want web content for pages on the target site if not self.getTarget().matches(self.sf.urlFQDN(eventSource)): self.debug("Not collecting web content information for external sites.") return for regexpGrp in list(regexps.keys()): if regexpGrp in self.results[eventSource]: continue for regex in regexps[regexpGrp]: pat = re.compile(regex, re.IGNORECASE) matches = re.findall(pat, eventData) if len(matches) > 0 and regexpGrp not in self.results[eventSource]: self.info("Matched " + regexpGrp + " in content from " + eventSource) self.results[eventSource] = self.results[eventSource] + [regexpGrp] evt = SpiderFootEvent("ERROR_MESSAGE", regexpGrp, self.__name__, event) self.notifyListeners(evt) # End of sfp_errors class ================================================ FILE: modules/sfp_ethereum.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_ethereum # Purpose: SpiderFoot plug-in for scanning retrieved content by other # modules (such as sfp_spider) and identifying ethereum addresses. # # Author: Steve Micallef # # Created: 03/09/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import re from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_ethereum(SpiderFootPlugin): meta = { 'name': "Ethereum Address Extractor", 'summary': "Identify ethereum addresses in scraped webpages.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Content Analysis"] } # Default options opts = {} optdescs = {} results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["TARGET_WEB_CONTENT"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["ETHEREUM_ADDRESS"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data sourceData = self.sf.hashstring(eventData) if sourceData in self.results: return self.results[sourceData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") # thanks to https://stackoverflow.com/questions/21683680/regex-to-match-ethereum-addresses matches = re.findall(r"[\s:=\>](0x[a-fA-F0-9]{40})", eventData) for m in matches: self.debug("Ethereum address match: " + m) evt = SpiderFootEvent("ETHEREUM_ADDRESS", m, self.__name__, event) self.notifyListeners(evt) # End of sfp_ethereum class ================================================ FILE: modules/sfp_etherscan.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_etherscan # Purpose: SpiderFoot plug-in to look up a ethereum wallet's balance by # querying etherscan.io. # # Author: Krishnasis Mandal # # Created: 26/01/2021 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_etherscan(SpiderFootPlugin): meta = { 'name': "Etherscan", 'summary': "Queries etherscan.io to find the balance of identified ethereum wallet addresses.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Public Registries"], 'dataSource': { 'website': "https://etherscan.io", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://etherscan.io/apis" ], 'apiKeyInstructions': [ "Visit https://etherscan.io", "Register a free account", "Browse to https://etherscan.io/myapikey", "Click on Add beside API Key", "Your API Key will be listed under API Key Token", ], 'favIcon': "https://etherscan.io/images/favicon3.ico", 'logo': "https://etherscan.io/images/brandassets/etherscan-logo-circle.png", 'description': "Etherscan allows you to explore and search the Ethereum blockchain " "for transactions, addresses, tokens, prices and other activities taking place on Ethereum (ETH)", } } # Default options opts = { 'api_key': '', 'pause': 1 } # Option descriptions optdescs = { 'api_key': "API Key for etherscan.io", 'pause': "Number of seconds to wait between each API call." } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return [ "ETHEREUM_ADDRESS" ] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return [ "ETHEREUM_BALANCE", "RAW_RIR_DATA" ] def query(self, qry): queryString = f"https://api.etherscan.io/api?module=account&action=balance&address={qry}&tag=latest&apikey={self.opts['api_key']}" # Wallet balance res = self.sf.fetchUrl(queryString, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) time.sleep(self.opts['pause']) if res['content'] is None: self.info(f"No Etherscan data found for {qry}") return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.errorState: return if self.opts['api_key'] == "": self.error("You enabled sfp_etherscan but did not set an API key!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True data = self.query(eventData) if data is None: self.info(f"No Etherscan data found for {eventData}") return # Value returned by etherscan was too large in comparison to actual wallet balance balance = float(data.get('result')) / 1000000000000000000 evt = SpiderFootEvent("ETHEREUM_BALANCE", f"{str(balance)} ETH", self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) # End of sfp_etherscan class ================================================ FILE: modules/sfp_filemeta.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_filemeta # Purpose: From Spidering and from searching search engines, extracts file # meta data from files matching certain file extensions. # # Author: Steve Micallef # # Created: 25/04/2014 # Copyright: (c) Steve Micallef 2014 # Licence: MIT # ------------------------------------------------------------------------------- import io import mimetypes import PyPDF2 import docx import exifread import pptx from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_filemeta(SpiderFootPlugin): meta = { 'name': "File Metadata Extractor", 'summary': "Extracts meta data from documents and images.", 'flags': [], 'useCases': ["Footprint"], 'categories': ["Content Analysis"] } # Default options opts = { 'fileexts': ["docx", "pptx", 'pdf', 'jpg', 'jpeg', 'tiff', 'tif'], 'timeout': 300 } # Option descriptions optdescs = { 'fileexts': "File extensions of files you want to analyze the meta data of (only PDF, DOCX, XLSX and PPTX are supported.)", 'timeout': "Download timeout for files, in seconds." } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.__dataSource__ = "Target Website" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["LINKED_URL_INTERNAL", "INTERESTING_FILE"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["RAW_FILE_META_DATA", "SOFTWARE_USED"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: return self.results[eventData] = True for fileExt in self.opts['fileexts']: if self.checkForStop(): return if "." + fileExt.lower() in eventData.lower(): # Fetch the file, allow much more time given that these files are # typically large. ret = self.sf.fetchUrl(eventData, timeout=self.opts['timeout'], useragent=self.opts['_useragent'], disableContentEncoding=True, sizeLimit=10000000, verify=False) if ret['content'] is None: self.error(f"Unable to fetch file for meta analysis: {eventData}") return if len(ret['content']) < 512: self.error(f"Strange content encountered, size of {len(ret['content'])}") return meta = None data = None # Based on the file extension, handle it if fileExt.lower() == "pdf": try: raw = io.BytesIO(ret['content']) # data = metapdf.MetaPdfReader().read_metadata(raw) pdf = PyPDF2.PdfFileReader(raw, strict=False) data = pdf.getDocumentInfo() meta = str(data) self.debug("Obtained meta data from " + eventData) except Exception as e: self.error(f"Unable to parse meta data from: {eventData} ({e})") return if fileExt.lower() in ["docx"]: try: c = io.BytesIO(ret['content']) doc = docx.Document(c) mtype = mimetypes.guess_type(eventData)[0] self.debug("Office type: " + str(mtype)) a = doc.core_properties.author c = doc.core_properties.comments data = [_f for _f in [a, c] if _f] meta = ", ".join(data) except Exception as e: self.error(f"Unable to process file: {eventData} ({e})") return if fileExt.lower() in ["pptx"]: try: c = io.BytesIO(ret['content']) doc = pptx.Presentation(c) mtype = mimetypes.guess_type(eventData)[0] self.debug("Office type: " + str(mtype)) a = doc.core_properties.author c = doc.core_properties.comments data = [_f for _f in [a, c] if _f] meta = ", ".join(data) except Exception as e: self.error(f"Unable to process file: {eventData} ({e})") return if fileExt.lower() in ["jpg", "jpeg", "tiff"]: try: raw = io.BytesIO(ret['content']) data = exifread.process_file(raw) if data is None or len(data) == 0: continue meta = str(data) except Exception as e: self.error(f"Unable to parse meta data from: {eventData} ({e})") return if meta is not None and data is not None: rawevt = SpiderFootEvent("RAW_FILE_META_DATA", meta, self.__name__, event) self.notifyListeners(rawevt) val = list() try: if "/Producer" in data: val.append(str(data['/Producer'])) if "/Creator" in data: val.append(str(data['/Creator'])) if "Application" in data: val.append(str(data['Application'])) if "Image Software" in data: val.append(str(data['Image Software'])) except Exception as e: self.error("Failed to parse PDF, " + eventData + ": " + str(e)) return for v in val: if v and not isinstance(v, PyPDF2.generic.NullObject): self.debug("VAL: " + str(val)) # Strip non-ASCII v = ''.join([i if ord(i) < 128 else ' ' for i in v]) evt = SpiderFootEvent("SOFTWARE_USED", v, self.__name__, rawevt) self.notifyListeners(evt) ================================================ FILE: modules/sfp_flickr.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_flickr # Purpose: Search Flickr API for domains, URLs and emails related to the # specified domain. # # Author: # # Created: 2018-10-08 # Copyright: (c) bcoles 2018 # Licence: MIT # ------------------------------------------------------------------------------- import json import re import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_flickr(SpiderFootPlugin): meta = { 'name': "Flickr", 'summary': "Search Flickr for domains, URLs and emails related to the specified domain.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Social Media"], 'dataSource': { 'website': "https://www.flickr.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://www.flickr.com/services/api/", "https://www.flickr.com/services/developer/api/", "https://code.flickr.net/" ], 'favIcon': "https://combo.staticflickr.com/pw/favicon.ico", 'logo': "https://combo.staticflickr.com/pw/favicon.ico", 'description': "Flickr is almost certainly the best online photo management and sharing application in the world.\n " "On Flickr, members upload photos, share them securely, supplement their photos with " "metadata like license information, geo-location, people, tags, etc., " "and interact with their family, friends, contacts or anyone in the community. " "Practically all the features on Flickr's various platforms -- web, mobile and desktop -- " "are accompanied by a longstanding API program. " "Since 2005, developers have collaborated on top of Flickr's APIs to build fun, creative, " "and gorgeous experiences around photos that extend beyond Flickr.", } } # Default options opts = { 'pause': 1, 'per_page': 100, 'maxpages': 20, 'dns_resolve': True, } # Option descriptions optdescs = { 'pause': "Number of seconds to pause between fetches.", 'per_page': "Maximum number of results per page.", 'maxpages': "Maximum number of pages of results to fetch.", 'dns_resolve': "DNS resolve each identified domain.", } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME"] # What events this module produces def producedEvents(self): return ["EMAILADDR", "EMAILADDR_GENERIC", "INTERNET_NAME", "DOMAIN_NAME", "LINKED_URL_INTERNAL"] # Retrieve API key def retrieveApiKey(self): res = self.sf.fetchUrl("https://www.flickr.com/", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if res['content'] is None: return None keys = re.findall(r'YUI_config.flickr.api.site_key = "([a-zA-Z0-9]+)"', str(res['content'])) if not keys: return None return keys[0] # Query the REST API def query(self, qry, api_key, page=1, per_page=200): params = { "sort": "relevance", "parse_tags": "1", "content_type": "7", "extras": "description,owner_name,path_alias,realname", "hermes": "1", "hermesClient": "1", "reqId": "", "nojsoncallback": "1", "viewerNSID": "", "method": "flickr.photos.search", "csrf": "", "lang": "en-US", "per_page": str(per_page), "page": str(page), "text": qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), "api_key": api_key, "format": "json" } res = self.sf.fetchUrl("https://api.flickr.com/services/rest?" + urllib.parse.urlencode(params), useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout']) time.sleep(self.opts['pause']) try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: self.debug(f"Skipping {eventData}, already checked") return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if srcModuleName == 'sfp_flickr': self.debug(f"Ignoring {eventData}, from self.") return # Retrieve API key api_key = self.retrieveApiKey() if not api_key: self.error("Failed to obtain API key") return self.debug(f"Retrieved API key: {api_key}") # Query API for event data hosts = list() page = 1 pages = self.opts['maxpages'] per_page = self.opts['per_page'] while page <= pages: if self.checkForStop(): return if self.errorState: return data = self.query(eventData, api_key, page=page, per_page=per_page) if data is None: return # Check the response is ok if data.get('stat') != "ok": self.debug("Error retrieving search results.") return photos = data.get('photos') if not photos: self.debug("No search results.") return # Calculate number of pages to retrieve result_pages = int(photos.get('pages', 0)) if result_pages < pages: pages = result_pages if 'max_allowed_pages' in photos: allowed_pages = int(photos.get('max_allowed_pages', 0)) if pages > allowed_pages: pages = allowed_pages self.info(f"Parsing page {page} of {pages}") # Extract data for photo in photos.get('photo', list()): emails = SpiderFootHelpers.extractEmailsFromText(str(photo)) for email in emails: if email in self.results: continue mail_domain = email.lower().split('@')[1] if not self.getTarget().matches(mail_domain, includeChildren=True, includeParents=True): self.debug(f"Skipped unrelated address: {email}") continue self.info("Found e-mail address: " + email) if email.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" evt = SpiderFootEvent(evttype, email, self.__name__, event) self.notifyListeners(evt) self.results[email] = True links = SpiderFootHelpers.extractUrlsFromText(str(photo)) for link in links: if link in self.results: continue host = self.sf.urlFQDN(link) if not self.getTarget().matches(host, includeChildren=True, includeParents=True): self.debug(f"Skipped unrelated URL: {link}") continue hosts.append(host) self.debug(f"Found a URL: {link}") evt = SpiderFootEvent('LINKED_URL_INTERNAL', link, self.__name__, event) self.notifyListeners(evt) self.results[link] = True page += 1 for host in set(hosts): if self.checkForStop(): return if self.errorState: return if self.opts['dns_resolve'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host): self.debug(f"Host {host} could not be resolved") evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host, self.__name__, event) self.notifyListeners(evt) continue evt = SpiderFootEvent("INTERNET_NAME", host, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(host, self.opts["_internettlds"]): evt = SpiderFootEvent("DOMAIN_NAME", host, self.__name__, event) self.notifyListeners(evt) # End of sfp_flickr class ================================================ FILE: modules/sfp_focsec.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_focsec # Purpose: Look up IP address information from Focsec. # # Author: # # Created: 2021-10-09 # Copyright: (c) bcoles 2021 # Licence: MIT # ------------------------------------------------------------------------------- import json import urllib from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_focsec(SpiderFootPlugin): meta = { 'name': "Focsec", 'summary': "Look up IP address information from Focsec.", 'flags': ['apikey'], 'useCases': ["Passive", "Footprint", "Investigate"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://focsec.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://docs.focsec.com/#ip", ], "apiKeyInstructions": [ "Visit https://focsec.com/signup", "Register an account", "Visit https://focsec.com/account/dashboard and use the API key provided", ], 'favIcon': "https://focsec.com/static/favicon.png", 'logo': "https://focsec.com/static/web/images/logo.png", 'description': "Our API lets you know if a user's IP address is associated with a VPN, Proxy, TOR or malicious bots." "Take your applications security to the next level by detecting suspicious activity early on." } } opts = { "api_key": "", } optdescs = { "api_key": "Focsec API Key.", } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.errorState = False self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "IPV6_ADDRESS" ] def producedEvents(self): return [ "RAW_RIR_DATA", "GEOINFO", "MALICIOUS_IPADDR", "PROXY_HOST", "VPN_HOST", "TOR_EXIT_NODE", ] def query(self, qry): """Retrieve IP address information from Focsec. Args: qry (str): IPv4/IPv6 address Returns: dict: JSON formatted results """ params = urllib.parse.urlencode({ 'api_key': self.opts["api_key"], }) res = self.sf.fetchUrl( f"https://api.focsec.com/v1/ip/{qry}?{params}", timeout=self.opts["_fetchtimeout"], useragent=self.opts['_useragent'] ) if not res: self.error("No response from Focsec.") return None if res['code'] == "400": self.error("Bad request.") self.errorState = True return None if res['code'] == "401": self.error("Unauthorized - Invalid API key.") self.errorState = True return None if res['code'] == "402": self.error("Unauthorized - Payment Required. Subscription or trial period expired.") self.errorState = True return None if res['code'] == "404": self.debug(f"No results for {qry}") return None # Future proofing - Focsec does not implement rate limiting if res['code'] == "429": self.error("You are being rate-limited by Focsec.") return None if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from Focsec.") return None if not res['content']: self.debug("No results from Focsec.") return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.error( f"You enabled {self.__class__.__name__} but did not set an API key!" ) self.errorState = True return data = self.query(eventData) if not data: self.debug(f"Found no results for {eventData}") return e = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(e) is_bot = data.get('is_bot') if is_bot: e = SpiderFootEvent("MALICIOUS_IPADDR", f"Focsec [{eventData}]", self.__name__, event) self.notifyListeners(e) is_tor = data.get('is_tor') if is_tor: e = SpiderFootEvent("TOR_EXIT_NODE", eventData, self.__name__, event) self.notifyListeners(e) is_vpn = data.get('is_vpn') if is_vpn: e = SpiderFootEvent("VPN_HOST", eventData, self.__name__, event) self.notifyListeners(e) is_proxy = data.get('is_proxy') if is_proxy: e = SpiderFootEvent("PROXY_HOST", eventData, self.__name__, event) self.notifyListeners(e) location = ', '.join( filter( None, [ data.get('city'), data.get('country'), ] ) ) if location: e = SpiderFootEvent("GEOINFO", location, self.__name__, event) self.notifyListeners(e) # End of sfp_focsec class ================================================ FILE: modules/sfp_fortinet.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_fortinet # Purpose: Check if an IP address is malicious according to FortiGuard Antispam. # # Author: steve@binarypool.com # # Created: 14/12/2013 # Copyright: (c) Steve Micallef, 2013 # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_fortinet(SpiderFootPlugin): meta = { 'name': "FortiGuard Antispam", 'summary': "Check if an IP address is malicious according to FortiGuard Antispam.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://www.fortiguard.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://www.fortiguard.com/learnmore#as", ], 'favIcon': "https://www.fortiguard.com/static/images/favicon.ico", 'logo': "https://www.fortiguard.com/static/images/Fortinet-logo%20white.png?v=880", 'description': "FortiGuard Antispam provides a comprehensive and multi-layered approach to detect and filter spam processed by organizations." } } opts = { 'checkaffiliates': True } optdescs = { 'checkaffiliates': "Apply checks to affiliates?" } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "IPV6_ADDRESS", "AFFILIATE_IPADDR", "AFFILIATE_IPV6_ADDRESS", ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", ] def query(self, ip): if not ip: return None res = self.sf.fetchUrl( f"https://www.fortiguard.com/search?q={ip}&engine=8", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from FortiGuard Antispam.") self.errorState = True return None if res['content'] is None: self.error("Received no content from FortiGuard Antispam") self.errorState = True return None return res['content'] def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']: malicious_type = 'MALICIOUS_IPADDR' blacklist_type = 'BLACKLISTED_IPADDR' elif eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS']: if not self.opts.get('checkaffiliates', False): return malicious_type = 'MALICIOUS_AFFILIATE_IPADDR' blacklist_type = 'BLACKLISTED_AFFILIATE_IPADDR' else: self.debug(f"Unexpected event type {eventName}, skipping") return data = self.query(eventData) if not data: return if "Your signature is on the blocklist" not in data: return url = f"https://www.fortiguard.com/search?q={eventData}&engine=8" text = f"FortiGuard Antispam [{eventData}]\n{url}" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_fortinet class ================================================ FILE: modules/sfp_fraudguard.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_fraudguard # Purpose: Query fraudguard.io using their API # # Author: Steve Micallef # # Created: 18/06/2017 # Copyright: (c) Steve Micallef 2017 # Licence: MIT # ------------------------------------------------------------------------------- import base64 import json import time from datetime import datetime from netaddr import IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_fraudguard(SpiderFootPlugin): meta = { 'name': "Fraudguard", 'summary': "Obtain threat information from Fraudguard.io", 'flags': ["apikey"], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://fraudguard.io/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://docs.fraudguard.io/", "https://faq.fraudguard.io/" ], 'apiKeyInstructions': [ "Visit https://app.fraudguard.io/register", "Register a free account", "Navigate to https://app.fraudguard.io/keys", "The API key combination is listed under Username and Password" ], 'favIcon': "https://fraudguard.io/img/favicon.ico", 'logo': "https://s3.amazonaws.com/fraudguard.io/img/header.png", 'description': "FraudGuard is a service designed to provide an easy way to validate usage " "by continuously collecting and analyzing real-time internet traffic. " "Utilizing just a few simple API endpoints we make integration as simple as possible " "and return data such as: Risk Level, Threat Type, Geo Location, etc. Super fast, super simple.\n" "Lookup any IP address by querying our threat engine.", } } # Default options opts = { "fraudguard_api_key_account": "", "fraudguard_api_key_password": "", "age_limit_days": 90, 'netblocklookup': True, 'maxnetblock': 24, 'maxv6netblock': 120, 'subnetlookup': True, 'maxsubnet': 24, 'maxv6subnet': 120, 'checkaffiliates': True } # Option descriptions optdescs = { "fraudguard_api_key_account": "Fraudguard.io API username.", "fraudguard_api_key_password": "Fraudguard.io API password.", "age_limit_days": "Ignore any records older than this many days. 0 = unlimited.", 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?", 'maxnetblock': "If looking up owned netblocks, the maximum IPv4 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'maxv6netblock': "If looking up owned netblocks, the maximum IPv6 netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'subnetlookup': "Look up all IPs on subnets which your target is a part of for blacklisting?", 'maxsubnet': "If looking up subnets, the maximum IPv4 subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'maxv6subnet': "If looking up subnets, the maximum IPv6 subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'checkaffiliates': "Apply checks to affiliates?" } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.errorState = False self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "IPV6_ADDRESS", "AFFILIATE_IPADDR", "AFFILIATE_IPV6_ADDRESS", "NETBLOCK_MEMBER", "NETBLOCKV6_MEMBER", "NETBLOCK_OWNER", "NETBLOCKV6_OWNER", ] def producedEvents(self): return [ "GEOINFO", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_SUBNET", "MALICIOUS_NETBLOCK" ] def query(self, qry): """Query IP address Args: qry (str): IPv4/IPv6 address Returns: dict: JSON formatted results """ fraudguard_url = "https://api.fraudguard.io/ip/" + qry api_key_account = self.opts['fraudguard_api_key_account'] if type(api_key_account) == str: api_key_account = api_key_account.encode('utf-8') api_key_password = self.opts['fraudguard_api_key_password'] if type(api_key_password) == str: api_key_password = api_key_password.encode('utf-8') token = base64.b64encode(api_key_account + ':'.encode('utf-8') + api_key_password) headers = { 'Authorization': "Basic " + token.decode('utf-8') } res = self.sf.fetchUrl( fraudguard_url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=headers ) if res['code'] in ["400", "429", "500", "403"]: self.error("Fraudguard.io API key seems to have been rejected or you have exceeded usage limits for the month.") self.errorState = True return None if res['content'] is None: self.info(f"No Fraudguard.io info found for {qry}") return None try: return json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from Fraudguard.io: {e}") return None def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['fraudguard_api_key_account'] == "" or self.opts['fraudguard_api_key_password'] == "": self.error("You enabled sfp_fraudguard but did not set an API username/password!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if eventName.startswith("AFFILIATE") and not self.opts['checkaffiliates']: return if eventName in ['NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']: if not self.opts['netblocklookup']: return if eventName == 'NETBLOCKV6_OWNER': max_netblock = self.opts['maxv6netblock'] else: max_netblock = self.opts['maxnetblock'] if IPNetwork(eventData).prefixlen < max_netblock: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}") return if eventName in ['NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']: if not self.opts['subnetlookup']: return if eventName == 'NETBLOCKV6_MEMBER': max_subnet = self.opts['maxv6subnet'] else: max_subnet = self.opts['maxsubnet'] if IPNetwork(eventData).prefixlen < max_subnet: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}") return if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS', 'NETBLOCK_OWNER', 'NETBLOCKV6_OWNER']: evtType = 'MALICIOUS_IPADDR' elif eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS', 'NETBLOCK_MEMBER', 'NETBLOCKV6_MEMBER']: evtType = 'MALICIOUS_AFFILIATE_IPADDR' else: self.debug(f"Unexpected event type {eventName}, skipping") return qrylist = list() if eventName.startswith("NETBLOCK"): for ipaddr in IPNetwork(eventData): qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) self.results[eventData] = True for addr in qrylist: if self.checkForStop(): return data = self.query(addr) if not data: continue self.debug(f"Found results for {addr} in Fraudguard.io") # Format: 2016-12-24T07:25:35+00:00' created_dt = datetime.strptime(data.get('discover_date'), '%Y-%m-%d %H:%M:%S') created_ts = int(time.mktime(created_dt.timetuple())) age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days']) if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts: self.debug(f"Record found but too old ({created_dt}), skipping.") continue # For netblocks, we need to create the IP address event so that # the threat intel event is more meaningful. if eventName == 'NETBLOCK_OWNER': pevent = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) elif eventName == 'NETBLOCKV6_OWNER': pevent = SpiderFootEvent("IPV6_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) elif eventName == 'NETBLOCK_MEMBER': pevent = SpiderFootEvent("AFFILIATE_IPADDR", addr, self.__name__, event) self.notifyListeners(pevent) elif eventName == 'NETBLOCKV6_MEMBER': pevent = SpiderFootEvent("AFFILIATE_IPV6_ADDRESS", addr, self.__name__, event) self.notifyListeners(pevent) else: pevent = event geoinfo = [ _f for _f in [ data.get('state'), data.get('city'), data.get('postal_code'), data.get('country') ] if _f and _f != "unknown" ] if geoinfo: location = ', '.join(filter(None, geoinfo)) e = SpiderFootEvent("GEOINFO", location, self.__name__, pevent) self.notifyListeners(e) threat = data.get('threat') if threat and threat != "unknown": risk_level = data.get('risk_level') e = SpiderFootEvent(evtType, f"{threat} (risk level: {risk_level}) [{addr}]", self.__name__, pevent) self.notifyListeners(e) # End of sfp_fraudguard class ================================================ FILE: modules/sfp_fsecure_riddler.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_fsecure_riddler # Purpose: Query F-Secure Riddler.io API. # # Author: # # Created: 2019-09-16 # Copyright: (c) bcoles 2019 # Licence: MIT # ------------------------------------------------------------------------------- import json import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_fsecure_riddler(SpiderFootPlugin): meta = { 'name': "F-Secure Riddler.io", 'summary': "Obtain network information from F-Secure Riddler.io API.", 'flags': ["apikey"], 'useCases': ["Investigate", "Footprint", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://riddler.io/", 'model': "PRIVATE_ONLY", 'references': [ "https://riddler.io/help/api", "https://riddler.io/help/search", "https://riddler.io/static/riddler_white_paper.pdf", "https://www.f-secure.com/en/business/products/vulnerability-management/radar" ], 'apiKeyInstructions': [ "Registration is disabled for new accounts" ], 'favIcon': "https://riddler.io/static/images/favicon.png", 'logo': "https://riddler.io/static/images/logo.png", 'description': "Riddler.io allows you to search in a high quality dataset with more than 396,831,739 hostnames. " "Unlike others, we do not rely on simple port scanning techniques - we crawl the web, " "ensuring an in-depth quality data set you will not find anywhere else.\n" "Use Riddler to enumerate possible attack vectors during your pen-test or use the very same data " "to monitor potential threats before it is too late.", } } opts = { 'verify': True, 'username': '', 'password': '' } optdescs = { 'verify': 'Verify host names resolve', 'username': 'F-Secure Riddler.io username', 'password': 'F-Secure Riddler.io password' } results = None token = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ['DOMAIN_NAME', 'INTERNET_NAME', 'INTERNET_NAME_UNRESOLVED', 'IP_ADDRESS'] def producedEvents(self): return ['INTERNET_NAME', 'AFFILIATE_INTERNET_NAME', 'INTERNET_NAME_UNRESOLVED', 'AFFILIATE_INTERNET_NAME_UNRESOLVED', 'DOMAIN_NAME', 'AFFILIATE_DOMAIN_NAME', 'IP_ADDRESS', 'PHYSICAL_COORDINATES', 'RAW_RIR_DATA'] # https://riddler.io/help/api def login(self): params = { 'email': self.opts['username'].encode('raw_unicode_escape').decode("ascii"), 'password': self.opts['password'].encode('raw_unicode_escape').decode("ascii") } headers = { 'Content-Type': 'application/json', } res = self.sf.fetchUrl('https://riddler.io/auth/login', postData=json.dumps(params), headers=headers, useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout']) if res['content'] is None: return try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from F-Secure Riddler: {e}") return try: token = data.get('response').get('user').get('authentication_token') except Exception: self.error('Login failed') self.errorState = True return if not token: self.error('Login failed') self.errorState = True return self.token = token # https://riddler.io/help/search def query(self, qry): params = { 'query': qry.encode('raw_unicode_escape').decode("ascii", errors='replace') } headers = { 'Authentication-Token': self.token, 'Content-Type': 'application/json', } res = self.sf.fetchUrl('https://riddler.io/api/search', postData=json.dumps(params), headers=headers, useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout']) time.sleep(1) if res['code'] in ["400", "401", "402", "403", "500"]: self.error(f"Unexpected HTTP response code {res['code']} from F-Secure Riddler") self.errorState = True return None if res['content'] is None: return None try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from F-Secure Riddler: {e}") return None if not data: self.debug(f"No results found for {qry}") return None return data def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if srcModuleName == 'sfp_fsecure_riddler': self.debug(f"Ignoring {eventName}, from self.") return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.opts['username'] == '' or self.opts['password'] == '': self.error('You enabled sfp_fsecure_riddler but did not set an API username/password!') self.errorState = True return if not self.token: self.login() if not self.token: self.error('Could not login to F-Secure Riddler') self.errorState = True return self.results[eventData] = True data = None if eventName in ['INTERNET_NAME', 'DOMAIN_NAME']: data = self.query("pld:" + eventData) elif eventName == 'IP_ADDRESS': data = self.query("ip:" + eventData) if not data: self.info(f"No results found for {eventData}") return e = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(e) hosts = list() addrs = list() coords = list() for result in data: host = result.get('host') if not host: continue if not self.getTarget().matches(host, includeChildren=True, includeParents=True): continue hosts.append(host) addr = result.get('addr') if addr: addrs.append(addr) coord = result.get('cordinates') if coord and len(coord) == 2: coords.append(str(coord[0]) + ', ' + str(coord[1])) if self.opts['verify'] and len(hosts) > 0: self.info("Resolving " + str(len(set(hosts))) + " domains ...") for host in set(hosts): if self.getTarget().matches(host, includeChildren=True, includeParents=True): evt_type = 'INTERNET_NAME' else: evt_type = 'AFFILIATE_INTERNET_NAME' if self.opts['verify'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host): self.debug(f"Host {host} could not be resolved") evt_type += '_UNRESOLVED' evt = SpiderFootEvent(evt_type, host, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(host, self.opts['_internettlds']): if evt_type.startswith('AFFILIATE'): evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', host, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent('DOMAIN_NAME', host, self.__name__, event) self.notifyListeners(evt) for addr in set(addrs): if self.sf.validIP(addr): evt = SpiderFootEvent('IP_ADDRESS', addr, self.__name__, event) self.notifyListeners(evt) for coord in set(coords): evt = SpiderFootEvent('PHYSICAL_COORDINATES', coord, self.__name__, event) self.notifyListeners(evt) # End of sfp_fsecure_riddler class ================================================ FILE: modules/sfp_fullcontact.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_fullcontact # Purpose: Gather domain and e-mail information from FullContact.com API. # # Author: Steve Micallef # # Created: 06/02/2018 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import time from datetime import datetime from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_fullcontact(SpiderFootPlugin): meta = { 'name': "FullContact", 'summary': "Gather domain and e-mail information from FullContact.com API.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://www.fullcontact.com", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://dashboard.fullcontact.com/api-ref", "https://www.fullcontact.com/developer-portal/", "https://www.fullcontact.com/insights-bundles/", "https://dashboard.fullcontact.com/docs", "https://www.fullcontact.com/faq/" ], 'apiKeyInstructions': [ "Visit https://fullcontact.com", "Register a free account", "Navigate to https://dashboard.fullcontact.com", "Click on 'Get an API Key'", "Verify your account using your contact number", "The API Key will be listed under 'Your API Keys'" ], 'favIcon': "https://1a3asl4eps7u26kl661u3bi9-wpengine.netdna-ssl.com/wp-content/uploads/2019/11/cropped-full-contact-isologo-32x32.png", 'logo': "https://1a3asl4eps7u26kl661u3bi9-wpengine.netdna-ssl.com/wp-content/themes/fc-theme/assets/images/common/full-contact-logo.svg?1574450351", 'description': "Connecting data. Consolidating identities. Applying insights. Amplifying media reach. " "We provide person-centered identity resolution to improve your customer interactions with a simple, " "real-time API integration.\n" "FullContact is a privacy-safe Identity Resolution company building trust between people and brands. " "We deliver the capabilities needed to create tailored customer experiences by unifying data and " "applying insights in the moments that matter.", } } opts = { "api_key": "", "max_age_days": "365" } optdescs = { "api_key": "FullContact.com API key.", "max_age_days": "Maximum number of age in days for a record before it's considered invalid and not reported." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["DOMAIN_NAME", "EMAILADDR"] def producedEvents(self): return [ "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA", "PHONE_NUMBER", "GEOINFO", "PHYSICAL_ADDRESS" ] def query(self, url, data, failcount=0): headers = { 'Authorization': f"Bearer {self.opts['api_key']}" } res = self.sf.fetchUrl( url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", postData=json.dumps(data), headers=headers ) if res['code'] in ["401", "400"]: self.error("API key rejected by FullContact") self.errorState = True return None if res['code'] == "403": if failcount == 3: self.error("Throttled or other blocking by FullContact") return None time.sleep(2) failcount += 1 return self.query(url, data, failcount) if not res['content']: self.error("No content returned from FullContact") return None try: ret = json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from FullContact: {e}") return None if "updated" in ret and int(self.opts['max_age_days']) > 0: last_dt = datetime.strptime(ret['updated'], '%Y-%m-%d') last_ts = int(time.mktime(last_dt.timetuple())) age_limit_ts = int(time.time()) - (86400 * int(self.opts['max_age_days'])) if last_ts < age_limit_ts: self.debug("FullContact record found but too old.") return None return ret def queryCompany(self, domain): url = "https://api.fullcontact.com/v3/company.enrich" if not domain: return None return self.query(url, {"domain": domain}) def queryPersonByEmail(self, email): url = "https://api.fullcontact.com/v3/person.enrich" if not email: return None return self.query(url, {'email': email}) def queryPersonByName(self, name): url = "https://api.fullcontact.com/v3/person.enrich" if not name: return None return self.query(url, {'fullName': name}) def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if self.opts["api_key"] == "": self.error( f"You enabled {self.__class__.__name__} but did not set an API key!" ) self.errorState = True return self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == "EMAILADDR": data = self.queryPersonByEmail(eventData) if not data: return full_name = data.get('fullName') if full_name: e = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {full_name}", self.__name__, event) self.notifyListeners(e) return if eventName == "DOMAIN_NAME": data = self.queryCompany(eventData) if not data: return if data.get("details"): data = data['details'] if data.get("emails"): for r in data['emails']: email = r.get('value') if not email: continue if email.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" e = SpiderFootEvent(evttype, email, self.__name__, event) self.notifyListeners(e) if data.get("phones"): for r in data['phones']: phone = r.get('value') if not phone: continue e = SpiderFootEvent("PHONE_NUMBER", phone, self.__name__, event) self.notifyListeners(e) if data.get("locations"): for r in data['locations']: location = ', '.join([_f for _f in [r.get('city'), r.get('country')] if _f]) if location: e = SpiderFootEvent( "GEOINFO", location, self.__name__, event ) self.notifyListeners(e) if r.get("formatted"): # Seems to contain some junk sometimes if len(r['formatted']) > 10: e = SpiderFootEvent( "PHYSICAL_ADDRESS", r['formatted'], self.__name__, event ) self.notifyListeners(e) if data.get("keyPeople"): for r in data['keyPeople']: full_name = r.get('fullName') if full_name: e = SpiderFootEvent( "RAW_RIR_DATA", f"Possible full name: {full_name}", self.__name__, event ) self.notifyListeners(e) # End of sfp_fullcontact class ================================================ FILE: modules/sfp_fullhunt.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_fullhunt # Purpose: Identify domain attack surface using FullHunt API. # # Author: # # Created: 2021-10-26 # Copyright: (c) bcoles 2021 # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_fullhunt(SpiderFootPlugin): meta = { 'name': "FullHunt", 'summary': "Identify domain attack surface using FullHunt API.", 'flags': ['apikey'], 'useCases': ["Passive", "Footprint", "Investigate"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://fullhunt.io/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://api-docs.fullhunt.io/", ], 'apiKeyInstructions': [ "Visit https://fullhunt.io/", "Register a free account", "Navigate to https://fullhunt.io/user/settings/", "Your API key is listed under 'API Access'" ], 'favIcon': "https://fullhunt.io/static/theme/images/logo/favicon.ico", 'logo': "https://fullhunt.io/static/theme/images/logo/Icon.png", 'description': "Discover, monitor, and secure your attack surface. " "FullHunt delivers the best platform in the market for attack surface security." } } opts = { "api_key": "", } optdescs = { "api_key": "FullHunt API key.", } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.errorState = False self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "DOMAIN_NAME", ] def producedEvents(self): return [ "INTERNET_NAME", "INTERNET_NAME_UNRESOLVED", "AFFILIATE_INTERNET_NAME", "AFFILIATE_INTERNET_NAME_UNRESOLVED", "TCP_PORT_OPEN", "PROVIDER_DNS", "PROVIDER_MAIL", "RAW_RIR_DATA" ] def queryDomainDetails(self, qry): """Search for hosts on a domain. Args: qry (str): domain name Returns: dict: search results """ headers = { 'X-API-KEY': self.opts['api_key'] } res = self.sf.fetchUrl( f"https://fullhunt.io/api/v1/domain/{qry}/details", timeout=30, headers=headers, useragent=self.opts['_useragent'] ) return self.parseApiResponse(res) def parseApiResponse(self, res: dict): if not res: self.error("No response from FullHunt.") return None if res['code'] == "400": self.error("Bad Request -- Your request is invalid.") return None if res['code'] == "401": self.errorState = True self.error("Unauthorized -- Your API key is wrong.") return None if res['code'] == "403": self.errorState = True self.error("Forbidden -- The requested resource is forbidden.") return None if res['code'] == "404": self.error("Not Found -- The requested resource could not be found.") return None if res['code'] == "429": self.errorState = True self.error("Too Many Requests -- You are sending too many requests.") return None try: results = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None return results.get('hosts') def handleEvent(self, event): eventName = event.eventType eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {event.module}") if self.opts["api_key"] == "": self.error( f"You enabled {self.__class__.__name__} but did not set an API key!" ) self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True res = self.queryDomainDetails(eventData) if not res: self.debug(f"Found no results for {eventData}") return e = SpiderFootEvent("RAW_RIR_DATA", str(res), self.__name__, event) self.notifyListeners(e) hosts = list() name_servers = list() mail_servers = list() for record in res: host = record.get('host') if not host: continue hosts.append(host) dns = record.get('dns') if dns: mx = dns.get('mx') if mx: for mail_server in mx: mail_servers.append(mail_server.rstrip(".")) ns = dns.get('ns') if ns: for name_server in ns: name_servers.append(name_server.rstrip(".")) cname = dns.get('cname') if cname: for c in cname: hosts.append(c.rstrip(".")) network_ports = record.get('network_ports') if network_ports: for port in network_ports: e = SpiderFootEvent("TCP_PORT_OPEN", f"{host}:{port}", self.__name__, event) self.notifyListeners(e) for host in set(mail_servers): if not host: continue hosts.append(host) e = SpiderFootEvent("PROVIDER_MAIL", host, self.__name__, event) self.notifyListeners(e) for host in set(name_servers): if not host: continue hosts.append(host) e = SpiderFootEvent("PROVIDER_DNS", host, self.__name__, event) self.notifyListeners(e) for host in set(hosts): if not host: continue if host in self.results: continue self.results[host] = True if self.getTarget().matches(host, includeChildren=True): evt_type = "INTERNET_NAME" else: evt_type = "AFFILIATE_INTERNET_NAME" if not self.sf.resolveHost(host) and not self.sf.resolveHost6(host): self.debug(f"Host {host} could not be resolved") evt_type += "_UNRESOLVED" evt = SpiderFootEvent(evt_type, host, self.__name__, event) self.notifyListeners(evt) # End of sfp_fullhunt class ================================================ FILE: modules/sfp_github.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_github # Purpose: Identifies public code repositories in Github associated with # your target. # # Author: Steve Micallef # # Created: 21/07/2015 # Copyright: (c) Steve Micallef 2015 # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_github(SpiderFootPlugin): meta = { 'name': "Github", 'summary': "Identify associated public code repositories on Github.", 'flags': [], 'useCases': ["Footprint", "Passive"], 'categories': ["Social Media"], 'dataSource': { 'website': "https://github.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://developer.github.com/" ], 'favIcon': "https://github.githubassets.com/favicons/favicon.png", 'logo': "https://github.githubassets.com/favicons/favicon.png", 'description': "GitHub brings together the world's largest community of " "developers to discover, share, and build better software.", } } # Default options opts = { 'namesonly': True } # Option descriptions optdescs = { 'namesonly': "Match repositories by name only, not by their descriptions. Helps reduce false positives." } def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME", "USERNAME", "SOCIAL_MEDIA"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["RAW_RIR_DATA", "GEOINFO", "PUBLIC_CODE_REPO"] # Build up repo info for use as an event def buildRepoInfo(self, item): # Get repos matching the name name = item.get('name') if name is None: self.debug("Incomplete Github information found (name).") return None html_url = item.get('html_url') if html_url is None: self.debug("Incomplete Github information found (url).") return None description = item.get('description') if description is None: self.debug("Incomplete Github information found (description).") return None return "\n".join([f"Name: {name}", f"URL: {html_url}", f"Description: {description}"]) def handleEvent(self, event): eventName = event.eventType eventData = event.data srcModuleName = event.module self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Already did a search for {eventData}, skipping.") return self.results[eventData] = True # Extract name and location from profile if eventName == "SOCIAL_MEDIA": try: network = eventData.split(": ")[0] url = eventData.split(": ")[1].replace("", "").replace("", "") except Exception as e: self.debug(f"Unable to parse SOCIAL_MEDIA: {eventData} ({e})") return if network != "Github": self.debug(f"Skipping social network profile, {url}, as not a GitHub profile") return try: urlParts = url.split("/") username = urlParts[len(urlParts) - 1] except Exception: self.debug(f"Couldn't get a username out of {url}") return res = self.sf.fetchUrl( f"https://api.github.com/users/{username}", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'] ) if res['content'] is None: return try: json_data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return if not json_data.get('login'): self.debug(f"{username} is not a valid GitHub profile") return full_name = json_data.get('name') if not full_name: self.debug(f"{username} is not a valid GitHub profile") return e = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {full_name}", self.__name__, event) self.notifyListeners(e) location = json_data.get('location') if location is None: return if len(location) < 3 or len(location) > 100: self.debug(f"Skipping likely invalid location: {location}") return e = SpiderFootEvent("GEOINFO", location, self.__name__, event) self.notifyListeners(e) return if eventName == "DOMAIN_NAME": username = self.sf.domainKeyword(eventData, self.opts['_internettlds']) if not username: return if eventName == "USERNAME": username = eventData self.debug(f"Looking at {username}") failed = False # Get all the repositories based on direct matches with the # name identified url = f"https://api.github.com/search/repositories?q={username}" res = self.sf.fetchUrl( url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'] ) if res['content'] is None: self.error(f"Unable to fetch {url}") failed = True if not failed: try: ret = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from GitHub: {e}") ret = None if ret is None: self.error(f"Unable to process empty response from Github for: {username}") failed = True if not failed: if ret.get('total_count', "0") == "0" or len(ret['items']) == 0: self.debug(f"No Github information for {username}") failed = True if not failed: for item in ret['items']: repo_info = self.buildRepoInfo(item) if repo_info is not None: if self.opts['namesonly'] and username != item['name']: continue evt = SpiderFootEvent("PUBLIC_CODE_REPO", repo_info, self.__name__, event) self.notifyListeners(evt) # Now look for users matching the name found failed = False url = f"https://api.github.com/search/users?q={username}" res = self.sf.fetchUrl( url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'] ) if res['content'] is None: self.error(f"Unable to fetch {url}") failed = True if not failed: try: ret = json.loads(res['content']) if ret is None: self.error(f"Unable to process empty response from Github for: {username}") failed = True except Exception: self.error(f"Unable to process invalid response from Github for: {username}") failed = True if not failed: if ret.get('total_count', "0") == "0" or len(ret['items']) == 0: self.debug("No Github information for " + username) failed = True if not failed: # For each user matching the username, get their repos for item in ret['items']: if item.get('repos_url') is None: self.debug("Incomplete Github information found (repos_url).") continue url = item['repos_url'] res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if res['content'] is None: self.error(f"Unable to fetch {url}") continue try: repret = json.loads(res['content']) except Exception as e: self.error(f"Invalid JSON returned from Github: {e}") continue if repret is None: self.error(f"Unable to process empty response from Github for: {username}") continue for item in repret: if type(item) != dict: self.debug("Encountered an unexpected or empty response from Github.") continue repo_info = self.buildRepoInfo(item) if repo_info is not None: if self.opts['namesonly'] and item['name'] != username: continue if eventName == "USERNAME" and "/" + username + "/" not in item.get('html_url', ''): continue evt = SpiderFootEvent("PUBLIC_CODE_REPO", repo_info, self.__name__, event) self.notifyListeners(evt) # End of sfp_github class ================================================ FILE: modules/sfp_gleif.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_gleif # Purpose: Search the Global LEI Index for company information using # Global Legal Entity Identifier Foundation (GLEIF) search API. # # Author: # # Created: 2021-06-21 # Copyright: (c) bcoles 2021 # Licence: MIT # ------------------------------------------------------------------------------- import json import urllib from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_gleif(SpiderFootPlugin): meta = { 'name': "GLEIF", 'summary': "Look up company information from Global Legal Entity Identifier Foundation (GLEIF).", 'flags': [], 'useCases': ["Passive", "Footprint", "Investigate"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://search.gleif.org/", 'model': "FREE_NOAUTH_LIMITED", 'references': [ "https://www.gleif.org/en/lei-data/gleif-api", "https://api.gleif.org/docs", ], 'favIcon': "https://www.gleif.org/favicon.ico", 'logo': "https://search.gleif.org/static/img/gleif-logo.svg", 'description': "The Global Legal Entity Identifier Foundation (GLEIF) Global LEI Index contains " "historical and current LEI records including related reference data in one authoritative, central repository." } } opts = { } optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["COMPANY_NAME", "LEI"] def producedEvents(self): return ["COMPANY_NAME", "LEI", "PHYSICAL_ADDRESS", "RAW_RIR_DATA"] def searchLegalName(self, qry): """Fuzzy search for legal entity by name Args: qry (str): legal entity name Returns: dict: search results """ params = urllib.parse.urlencode({ 'q': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), 'field': "entity.legalName" }) headers = { 'Accept': 'application/vnd.api+json' } res = self.sf.fetchUrl( f"https://api.gleif.org/api/v1/fuzzycompletions?{params}", timeout=30, headers=headers, useragent=self.opts['_useragent'] ) if res['code'] == "429": self.error("You are being rate-limited by GLEIF.") return None try: results = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None data = results.get('data') if not data: return None if not len(data): return None return data def searchAutocompletions(self, qry): """Search for legal entity name autocompletions Args: qry (str): legal entity name Returns: dict: search results """ params = urllib.parse.urlencode({ 'q': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), 'field': "fulltext" }) headers = { 'Accept': 'application/vnd.api+json' } res = self.sf.fetchUrl( f"https://api.gleif.org/api/v1/autocompletions?{params}", timeout=30, headers=headers, useragent=self.opts['_useragent'] ) if res['code'] == "429": self.error("You are being rate-limited by GLEIF.") return None try: results = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None data = results.get('data') if not data: return None if not len(data): return None return data def retrieveRecord(self, lei): headers = { 'Accept': 'application/vnd.api+json' } res = self.sf.fetchUrl( f"https://api.gleif.org/api/v1/lei-records/{lei}", timeout=self.opts['_fetchtimeout'], headers=headers, useragent=self.opts['_useragent'] ) if res['code'] == "404": self.error(f"No record for LEI: {lei}") return None if res['code'] == "429": self.error("You are being rate-limited by GLEIF.") return None try: results = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None data = results.get('data') if not len(data): return None if not data: return None return data def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.debug(f"Received event, {eventName}, from {srcModuleName}") leis = list() if eventName == "LEI": leis.append(eventData) elif eventName == "COMPANY_NAME": self.results[eventData] = True res = self.searchAutocompletions(eventData) if res is None: self.debug(f"Found no results for {eventData}") return e = SpiderFootEvent("RAW_RIR_DATA", str(res), self.__name__, event) self.notifyListeners(e) for record in res: relationships = record.get('relationships') if not relationships: continue lei_records = relationships.get('lei-records') if not lei_records: continue data = lei_records.get('data') if not data: continue lei = data.get('id') if not SpiderFootHelpers.validLEI(lei): continue leis.append(lei) self.info(f"Found {len(leis)} LEIs matching {eventData}") for lei in set(leis): if lei in self.results: continue if not SpiderFootHelpers.validLEI(lei): continue self.results[lei] = True e = SpiderFootEvent("LEI", lei, self.__name__, event) self.notifyListeners(e) self.results[lei] = True res = self.retrieveRecord(lei) if not res: self.debug(f"Found no results for {eventData}") continue attributes = res.get('attributes') if not attributes: continue entity = attributes.get('entity') if not entity: continue legal_name = entity.get('legalName') if legal_name: entity_name = legal_name.get('value') if entity_name: e = SpiderFootEvent("COMPANY_NAME", entity_name, self.__name__, event) self.notifyListeners(e) addresses = list() address = entity.get('legalAddress') if address.get('addressLines'): address_lines = ', '.join(filter(None, address.get('addressLines'))) location = ', '.join( filter( None, [ address_lines, address.get('city'), address.get('region'), address.get('country'), address.get('postalCode') ] ) ) if location: addresses.append(location) address = entity.get('headquartersAddress') if address.get('addressLines'): address_lines = ', '.join(filter(None, address.get('addressLines'))) location = ', '.join( filter( None, [ address_lines, address.get('city'), address.get('region'), address.get('country'), address.get('postalCode') ] ) ) if location: addresses.append(location) for address in set(addresses): e = SpiderFootEvent("PHYSICAL_ADDRESS", address, self.__name__, event) self.notifyListeners(e) # End of sfp_gleif class ================================================ FILE: modules/sfp_google_tag_manager.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_google_tag_manager # Purpose: Search Google Tag Manager (GTM) for hosts sharing the same GTM code. # # Author: # # Created: 2023-10-29 # Copyright: (c) bcoles 2023 # Licence: MIT # ------------------------------------------------------------------------------- import re import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_google_tag_manager(SpiderFootPlugin): meta = { 'name': "Google Tag Manager", 'summary': "Search Google Tag Manager (GTM) for hosts sharing the same GTM code.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Passive DNS"], 'dataSource': { 'website': "https://tagmanager.google.com", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://marketingplatform.google.com/about/tag-manager/", "https://developers.google.com/tag-manager/quickstart", "https://developers.google.com/tag-manager/devguide" ], 'favIcon': "https://google.com/favicon.ico", 'logo': "https://google.com/favicon.ico", 'description': "Manage all your website tags without editing code. Google Tag Manager " "delivers simple, reliable, easily integrated tag management solutions for free." } } opts = { "verify": True } optdescs = { "verify": "Verify identified hostnames resolve to an IP address." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ['WEB_ANALYTICS_ID'] def producedEvents(self): return [ 'DOMAIN_NAME', 'INTERNET_NAME', 'AFFILIATE_DOMAIN_NAME', 'AFFILIATE_INTERNET_NAME', ] # from: https://stackoverflow.com/a/43211062 def is_valid_hostname(self, hostname: str = None) -> bool: if not hostname: return False if len(hostname) > 255: return False hostname = hostname.rstrip(".") allowed = re.compile("(?!-)[A-Z0-9-_]{1,63}(? set: if not tag_id: return None params = urllib.parse.urlencode({ 'id': tag_id, }) res = self.sf.fetchUrl( f"https://googletagmanager.com/gtm.js?{params}", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'] ) if res['code'] != "200": self.debug(f"Invalid GTM tag id: {tag_id}") return None data = res['content'] if not data: self.debug(f"Invalid GTM tag id: {tag_id}") return None hosts = list() for host in re.findall(r'"map","key","(.+?)"', data): if '.' not in host: continue if self.is_valid_hostname(host): hosts.append(host) for host in re.findall(r',"arg1":"(.+?)"', data): if '.' not in host: continue if self.is_valid_hostname(host): hosts.append(host) for url in SpiderFootHelpers.extractUrlsFromText(str(data).replace("\\/", "/")): host = self.sf.urlFQDN(url) if not host: continue if '.' not in host: continue hosts.append(host) return set(hosts) def handleEvent(self, event): self.debug(f"Received event, {event.eventType}, from {event.module}") if self.errorState: return if event.data in self.results: return self.results[event.data] = True try: network = event.data.split(": ")[0] tag_id = event.data.split(": ")[1] except Exception as e: self.error(f"Unable to parse WEB_ANALYTICS_ID: {event.data} ({e})") return if network != 'Google Tag Manager': return hosts = self.queryGoogleTagId(tag_id) if not hosts: self.info(f"No hosts found for {tag_id}") return self.info(f"Retrieved {len(hosts)} results") for host in hosts: # we ignore unresolved hosts due to large number of false positives if self.opts['verify'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host): self.debug(f"Potential host name '{host}' could not be resolved") continue if self.getTarget().matches(host, includeChildren=True, includeParents=True): evt_type = 'INTERNET_NAME' else: evt_type = 'AFFILIATE_INTERNET_NAME' evt = SpiderFootEvent(evt_type, host, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(host, self.opts['_internettlds']): if evt_type.startswith('AFFILIATE'): evt_type = 'AFFILIATE_DOMAIN_NAME' else: evt_type = 'DOMAIN_NAME' evt = SpiderFootEvent(evt_type, host, self.__name__, event) self.notifyListeners(evt) # End of sfp_google_tag_manager class ================================================ FILE: modules/sfp_googlemaps.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_googlemaps # Purpose: SpiderFoot plug-in to identify historical certificates for a domain # from googlemaps.sh, and from this identify hostnames. # # Author: Steve Micallef # # Created: 18/03/2017 # Copyright: (c) Steve Micallef 2017 # Licence: MIT # ------------------------------------------------------------------------------- import json import urllib from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_googlemaps(SpiderFootPlugin): meta = { 'name': "Google Maps", 'summary': "Identifies potential physical addresses and latitude/longitude coordinates.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Real World"], 'dataSource': { 'website': "https://cloud.google.com/maps-platform/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://developers.google.com/maps/documentation/?_ga=2.135220017.1220421370.1587340370-900596925.1587340370" ], 'apiKeyInstructions': [ "Visit https://cloud.google.com/maps-platform/", "Register a free Google account", "Click on 'Get Started'", "Click on 'API'", "Select the type of API", "Navigate to https://console.cloud.google.com/apis/credentials", "Click on 'Credentials'", "The API Key will be listed under 'API Keys'" ], 'favIcon': "https://www.gstatic.com/devrel-devsite/prod/v2210deb8920cd4a55bd580441aa58e7853afc04b39a9d9ac4198e1cd7fbe04ef/cloud/images/favicons/onecloud/favicon.ico", 'logo': "https://www.gstatic.com/devrel-devsite/prod/v2210deb8920cd4a55bd580441aa58e7853afc04b39a9d9ac4198e1cd7fbe04ef/cloud/images/cloud-logo.svg", 'description': "Explore where real-world insights and immersive location experiences can take your business.\n" "Build with reliable, comprehensive data for over 200 countries and territories.\n" "has been done here. If line breaks are needed for breaking up\n" "Scale confidently, backed by our infrastructure.", } } opts = { "api_key": "" } optdescs = { "api_key": "Google Geocoding API Key." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ['DOMAIN_NAME', 'PHYSICAL_ADDRESS'] def producedEvents(self): return ["PHYSICAL_ADDRESS", "PHYSICAL_COORDINATES", "RAW_RIR_DATA"] def query(self, address): params = urllib.parse.urlencode({ 'key': self.opts['api_key'], 'address': address.encode('raw_unicode_escape').decode("ascii", errors='replace') }) res = self.sf.fetchUrl( f"https://maps.googleapis.com/maps/api/geocode/json?{params}", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'] ) if res['content'] is None: self.info(f"No location info found for {address}") return None return res def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.error( f"You enabled {self.__class__.__name__} but did not set an API key!" ) self.errorState = True return res = self.query(eventData) if not res: self.debug(f"No information found for {eventData}") return evt = SpiderFootEvent( "RAW_RIR_DATA", res['content'], self.__name__, event ) self.notifyListeners(evt) try: data = json.loads(res['content'])['results'][0] except Exception as e: self.debug(f"Error processing JSON response: {e}") return if srcModuleName == "sfp_googlemaps": return geometry = data.get('geometry') if geometry: location = data.get('location') if location: lat = location.get('lat') lng = location.get('lng') if lat and lng: evt = SpiderFootEvent( "PHYSICAL_COORDINATES", f"{lat},{lng}", self.__name__, event ) self.notifyListeners(evt) formatted_address = data.get('formatted_address') if formatted_address: evt = SpiderFootEvent( "PHYSICAL_ADDRESS", data['formatted_address'], self.__name__, event ) self.notifyListeners(evt) # End of sfp_googlemaps class ================================================ FILE: modules/sfp_googleobjectstorage.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_googleobjectstorage # Purpose: SpiderFoot plug-in for identifying potential Google Object Storage # buckets related to the target. # # Author: Steve Micallef # # Created: 24/01/2020 # Copyright: (c) Steve Micallef 2020 # Licence: MIT # ------------------------------------------------------------------------------- import random import threading import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_googleobjectstorage(SpiderFootPlugin): meta = { 'name': "Google Object Storage Finder", 'summary': "Search for potential Google Object Storage buckets associated with the target and attempt to list their contents.", 'flags': [], 'useCases': ["Footprint", "Passive"], 'categories': ["Crawling and Scanning"], 'dataSource': { 'website': "https://cloud.google.com/storage", 'model': "FREE_NOAUTH_UNLIMITED", 'favIcon': 'https://www.gstatic.com/devrel-devsite/prod/v4c1e9ea53f4457a5de1027b4eeb4608c1000a427e20261ba1771dd3fc26d5df8/cloud/images/favicons/onecloud/favicon.ico', 'logo': 'https://www.gstatic.com/devrel-devsite/prod/v4c1e9ea53f4457a5de1027b4eeb4608c1000a427e20261ba1771dd3fc26d5df8/cloud/images/cloud-logo.svg', 'description': "Object storage for companies of all sizes." "Secure, durable, and with low latency. Store any amount of data." "Retrieve it as often as you'd like." } } # Default options opts = { "suffixes": "test,dev,web,beta,bucket,space,files,content,data,prod,staging,production,stage,app,media,development,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data,-prod,-staging,-production,-stage,-app,-media,-development", "_maxthreads": 20 } # Option descriptions optdescs = { "suffixes": "List of suffixes to append to domains tried as bucket names", "_maxthreads": "Maximum threads" } results = None gosresults = dict() lock = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.gosresults = dict() self.results = self.tempStorage() self.lock = threading.Lock() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME", "LINKED_URL_EXTERNAL"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"] def checkSite(self, url): res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True) if not res['content']: return if "NoSuchBucket" in res['content']: self.debug(f"Not a valid bucket: {url}") return # Bucket found if res['code'] in ["301", "302", "200"]: # Bucket has files if "ListBucketResult" in res['content']: with self.lock: self.gosresults[url] = res['content'].count("") else: # Bucket has no files with self.lock: self.gosresults[url] = 0 def threadSites(self, siteList): self.gosresults = dict() running = True i = 0 t = [] for site in siteList: if self.checkForStop(): return None self.info("Spawning thread to check bucket: " + site) tname = str(random.SystemRandom().randint(0, 999999999)) t.append(threading.Thread(name='thread_sfp_googleobjectstorage_' + tname, target=self.checkSite, args=(site,))) t[i].start() i += 1 # Block until all threads are finished while running: found = False for rt in threading.enumerate(): if rt.name.startswith("thread_sfp_googleobjectstorage_"): found = True if not found: running = False time.sleep(0.25) # Return once the scanning has completed return self.gosresults def batchSites(self, sites): i = 0 res = list() siteList = list() for site in sites: if i >= self.opts['_maxthreads']: data = self.threadSites(siteList) if data is None: return res for ret in list(data.keys()): if data[ret]: # bucket:filecount res.append(ret + ":" + str(data[ret])) i = 0 siteList = list() siteList.append(site) i += 1 return res # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventName == "LINKED_URL_EXTERNAL": if ".storage.googleapis.com" in eventData: b = self.sf.urlFQDN(eventData) evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event) self.notifyListeners(evt) return targets = [eventData.replace('.', '')] kw = self.sf.domainKeyword(eventData, self.opts['_internettlds']) if kw: targets.append(kw) urls = list() for t in targets: suffixes = [''] + self.opts['suffixes'].split(',') for s in suffixes: if self.checkForStop(): return b = t + s + ".storage.googleapis.com" url = "https://" + b urls.append(url) # Batch the scans ret = self.batchSites(urls) for b in ret: bucket = b.split(":") evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event) self.notifyListeners(evt) if bucket[2] != "0": bucketname = bucket[1].replace("//", "") evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucketname + ": " + bucket[2] + " files found.", self.__name__, evt) self.notifyListeners(evt) # End of sfp_googleobjectstorage class ================================================ FILE: modules/sfp_googlesafebrowsing.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_googlesafebrowsing # Purpose: SpiderFoot plug-in to check if the URL is included on any of the # Google Safe Browsing lists # # Author: Filip Aleksić # # Created: 2020-08-18 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_googlesafebrowsing(SpiderFootPlugin): meta = { "name": "Google SafeBrowsing", "summary": "Check if the URL is included on any of the Safe Browsing lists.", 'flags': ["slow", "apikey"], "useCases": ["Passive", "Investigate"], "categories": ["Reputation Systems"], "dataSource": { "website": "https://developers.google.com/safe-browsing/v4/lookup-api", "model": "FREE_AUTH_UNLIMITED", "references": [ "https://developers.google.com/safe-browsing/v4/reference/rest" ], "apiKeyInstructions": [ "Visit https://console.developers.google.com/", "Register a free Google account or sign in", "Create or select existing Google Developer Console project", "Go to the Cloud Console API Library https://console.cloud.google.com/apis/library", "From the projects list, select the project you want to use", "In the API Library select 'Safe Browsing APIs'", "On the API page, click ENABLE", "Navigate to the APIs & Services→Credentials panel in Cloud Console", "Select Create credentials, then select API key from the dropdown menu", "The API key created dialog box displays your newly created key", ], "favIcon": "https://www.gstatic.com/devrel-devsite/prod/v1241c04ebcb2127897d6c18221acbd64e7ed5c46e5217fd83dd808e592c47bf6/developers/images/favicon.png", "logo": "https://developers.google.com/safe-browsing/images/SafeBrowsing_Icon.png", "description": "The Safe Browsing APIs (v4) let your client applications check URLs " "against Google's constantly updated lists of unsafe web resources. " "Any URL found on a Safe Browsing list is considered unsafe.", }, } opts = {"api_key": ""} optdescs = { "api_key": "Google Safe Browsing API key.", } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return [ "INTERNET_NAME", "IP_ADDRESS", "AFFILIATE_INTERNET_NAME", "AFFILIATE_IPADDR", "CO_HOSTED_SITE", ] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return [ "MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_COHOST", "RAW_RIR_DATA", ] def query(self, qry): headers = {"Content-Type": "application/json"} url = ( "https://safebrowsing.googleapis.com/v4/threatMatches" f":find?key={self.opts['api_key']}" ) payload = { "client": {"clientId": "SpiderFoot", "clientVersion": "3.2"}, "threatInfo": { "threatTypes": [ "THREAT_TYPE_UNSPECIFIED", "MALWARE", "SOCIAL_ENGINEERING", "UNWANTED_SOFTWARE", "POTENTIALLY_HARMFUL_APPLICATION", ], "platformTypes": ["PLATFORM_TYPE_UNSPECIFIED", "ANY_PLATFORM"], "threatEntryTypes": [ "THREAT_ENTRY_TYPE_UNSPECIFIED", "URL", "EXECUTABLE", ], "threatEntries": [ { "url": qry.encode("raw_unicode_escape").decode( "ascii", errors="replace" ) } ], }, } res = self.sf.fetchUrl( url, timeout=self.opts["_fetchtimeout"], useragent=self.opts["_useragent"], headers=headers, postData=json.dumps(payload), ) if res["code"] == "400": self.error("Invalid request payload on Google Safe Browsing API") self.errorState = True return None if res["code"] == "429": self.error("Reaching rate limit on Google Safe Browsing API") self.errorState = True return None if res["code"] == "403": self.error( "Permission denied, invalid API key on Google Safe Browsing API" ) self.errorState = True return None if res["code"] in ["500", "503", "504"]: self.error( "Google Safe Browsing API is having some troubles or unavailable." ) self.errorState = True return None try: info = json.loads(res["content"]) if info == {}: self.info("No Google Safe Browsing matches found for " + qry) return None except Exception as e: self.error(f"Error processing JSON response from SHODAN: {e}") return None return info def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.error( "You enabled sfp_googlesafebrowsing but did not set an API key!" ) self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True evtType = "" if eventName in ["IP_ADDRESS", "AFFILIATE_IPADDR"]: if eventName == "IP_ADDRESS": evtType = "MALICIOUS_IPADDR" else: evtType = "MALICIOUS_AFFILIATE_IPADDR" if eventName in ["INTERNET_NAME", "CO_HOSTED_SITE", "AFFILIATE_INTERNET_NAME"]: if eventName == "INTERNET_NAME": evtType = "MALICIOUS_INTERNET_NAME" if eventName == "AFFILIATE_INTERNET_NAME": evtType = "MALICIOUS_AFFILIATE_INTERNET_NAME" if eventName == "CO_HOSTED_SITE": evtType = "MALICIOUS_COHOST" rec = self.query(eventData) if rec is None: return evt = SpiderFootEvent("RAW_RIR_DATA", str(rec), self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent( evtType, "Google SafeBrowsing [" + eventData + "]", self.__name__, event ) self.notifyListeners(evt) # End of sfp_googlesafebrowsing class ================================================ FILE: modules/sfp_googlesearch.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_googlesearch # Purpose: Searches Google for content related to the domain in question. # # Author: Steve Micallef # # Created: 07/05/2012 # Copyright: (c) Steve Micallef 2012 # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_googlesearch(SpiderFootPlugin): meta = { 'name': "Google", 'summary': "Obtain information from the Google Custom Search API to identify sub-domains and links.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://developers.google.com/custom-search", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://developers.google.com/custom-search/v1", "https://developers.google.com/custom-search/docs/overview", "https://cse.google.com/cse" ], 'apiKeyInstructions': [ "Visit https://developers.google.com/custom-search/v1/introduction", "Register a free Google account", "Click on 'Get A Key'", "Connect a Project", "The API Key will be listed under 'YOUR API KEY'" ], 'favIcon': "https://www.gstatic.com/devrel-devsite/prod/v2210deb8920cd4a55bd580441aa58e7853afc04b39a9d9ac4198e1cd7fbe04ef/developers/images/favicon.png", 'logo': "https://www.gstatic.com/devrel-devsite/prod/v2210deb8920cd4a55bd580441aa58e7853afc04b39a9d9ac4198e1cd7fbe04ef/developers/images/favicon.png", 'description': "Google Custom Search enables you to create a search engine for your website, your blog, or a collection of websites. " "You can configure your engine to search both web pages and images. " "You can fine-tune the ranking, add your own promotions and customize the look and feel of the search results. " "You can monetize the search by connecting your engine to your Google AdSense account.", } } # Default options opts = { "api_key": "", "cse_id": "013611106330597893267:tfgl3wxdtbp" } # Option descriptions optdescs = { "api_key": "Google API Key for Google search.", "cse_id": "Google Custom Search Engine ID." } # Target results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["INTERNET_NAME"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["LINKED_URL_INTERNAL", "RAW_RIR_DATA"] def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "": self.error("You enabled sfp_googlesearch but did not set a Google API key!") self.errorState = True return if eventData in self.results: self.debug("Already did a search for " + eventData + ", skipping.") return self.results[eventData] = True # Sites hosted on the domain res = self.sf.googleIterate( searchString="site:" + eventData, opts={ "timeout": self.opts["_fetchtimeout"], "useragent": self.opts["_useragent"], "api_key": self.opts["api_key"], "cse_id": self.opts["cse_id"], }, ) if res is None: # Failed to talk to the Google API or no results returned return urls = res["urls"] new_links = list(set(urls) - set(self.results.keys())) # Add new links to results for link in new_links: self.results[link] = True internal_links = [ link for link in new_links if self.sf.urlFQDN(link).endswith(eventData) ] for link in internal_links: self.debug("Found a link: " + link) evt = SpiderFootEvent("LINKED_URL_INTERNAL", link, self.__name__, event) self.notifyListeners(evt) if internal_links: evt = SpiderFootEvent( "RAW_RIR_DATA", str(res), self.__name__, event ) self.notifyListeners(evt) # End of sfp_googlesearch class ================================================ FILE: modules/sfp_gravatar.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_gravatar # Purpose: SpiderFoot plug-in to search Gravatar API for an email address # and retrieve user information, including username, name, phone # numbers, additional email addresses, and social media usernames. # # Author: # # Created: 2019-05-26 # Copyright: (c) bcoles 2019 # Licence: MIT # ------------------------------------------------------------------------------- import hashlib import json import time from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_gravatar(SpiderFootPlugin): meta = { 'name': "Gravatar", 'summary': "Retrieve user information from Gravatar API.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Social Media"], 'dataSource': { 'website': "https://secure.gravatar.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://secure.gravatar.com/site/implement/" ], 'favIcon': "https://secure.gravatar.com/favicon.ico", 'logo': "https://secure.gravatar.com/favicon.ico", 'description': "Your Gravatar is an image that follows you from site to site " "appearing beside your name when you do things like comment or post on a blog.\n" "A Gravatar is a Globally Recognized Avatar. You upload it and create your profile just once, " "and then when you participate in any Gravatar-enabled site, your Gravatar image will automatically follow you there.", } } # Default options opts = { } # Option descriptions optdescs = { } results = None reportedUsers = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.reportedUsers = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['EMAILADDR'] # What events this module produces def producedEvents(self): return ['RAW_RIR_DATA', 'USERNAME', 'EMAILADDR', 'EMAILADDR_GENERIC', 'PHONE_NUMBER', 'GEOINFO', 'ACCOUNT_EXTERNAL_OWNED', 'SOCIAL_MEDIA'] # Query Gravatar API for the specified email address # https://secure.gravatar.com/site/implement/ # https://secure.gravatar.com/site/implement/profiles/ def query(self, qry): email_hash = hashlib.md5(qry.encode('utf-8', errors='replace').lower()).hexdigest() # noqa: DUO130 output = 'json' res = self.sf.fetchUrl("https://secure.gravatar.com/" + email_hash + '.' + output, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) time.sleep(1) if res['content'] is None: self.debug('No response from gravatar.com') return None if res['code'] != '200': return None try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None if data.get('entry') is None or len(data.get('entry')) == 0: return None return data.get('entry')[0] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") data = self.query(eventData) if data is None: self.debug("No user information found for " + eventData) return evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) if data.get('preferredUsername') is not None: un = data.get('preferredUsername') evt = SpiderFootEvent("USERNAME", un, self.__name__, event) self.notifyListeners(evt) self.reportedUsers[un] = True names = list() if data.get('name') is not None: if type(data.get('name')) != list: names.append(data.get('name')) else: names = data.get('name') for name in names: full_name = name.get('formatted') if full_name: evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {full_name}", self.__name__, event) self.notifyListeners(evt) # TODO: re-enable once location validation is implemented # location can not be trusted # if data.get('currentLocation') is not None: # location = data.get('currentLocation') # if len(location) < 3 or len(location) > 100: # self.debug("Skipping likely invalid location.") # else: # evt = SpiderFootEvent("GEOINFO", location, self.__name__, event) # self.notifyListeners(evt) if data.get('phoneNumbers') is not None: for number in data.get('phoneNumbers'): if number.get('value') is not None: evt = SpiderFootEvent("PHONE_NUMBER", number.get('value'), self.__name__, event) self.notifyListeners(evt) if data.get('emails') is not None: for email in data.get('emails'): em = email.get('value') if not em: continue if SpiderFootHelpers.validEmail(em) and em != eventData: if em.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" evt = SpiderFootEvent(evttype, em, self.__name__, event) self.notifyListeners(evt) if data.get('ims') is not None: for im in data.get('ims'): v = im.get('value') if v is None: continue t = im.get('type').capitalize() + " (Instant Messenger)\n" + v evt = SpiderFootEvent("ACCOUNT_EXTERNAL_OWNED", t, self.__name__, event) self.notifyListeners(evt) if v not in self.reportedUsers: evt = SpiderFootEvent("USERNAME", v, self.__name__, event) self.notifyListeners(evt) self.reportedUsers[v] = True if data.get('accounts') is not None: for account in data.get('accounts'): url = account.get('url') platform = account.get('shortname') if platform is not None and url is not None: t = platform.capitalize() + ": " + url + "" evt = SpiderFootEvent("SOCIAL_MEDIA", t, self.__name__, event) self.notifyListeners(evt) # End of sfp_gravatar class ================================================ FILE: modules/sfp_grayhatwarfare.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_grayhatwarfare # Purpose: Find bucket names matching the keyword extracted from a domain from Grayhat API. # # Author: # # Created: 24-01-2021 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_grayhatwarfare(SpiderFootPlugin): meta = { 'name': "Grayhat Warfare", 'summary': "Find bucket names matching the keyword extracted from a domain from Grayhat API.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://buckets.grayhatwarfare.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://buckets.grayhatwarfare.com/docs/api/v1" ], 'apiKeyInstructions': [ "Visit https://grayhatwarfare.com/register", "Register an account", "Visit https://grayhatwarfare.com/account/settings", "Your API key is listed under 'Api Key'", ], 'favIcon': "https://buckets.grayhatwarfare.com/assets/template/images/favicon.png", 'logo': "https://buckets.grayhatwarfare.com/assets/images/logo/logo-sm.png", 'description': "It is a searchable database of open buckets." "Has up to million results of each bucket." "Full text search with binary logic (can search for keywords and also stopwords)", } } # Default options opts = { 'api_key': '', 'per_page': 1000, 'max_pages': 2, 'pause': 1 } # Option descriptions optdescs = { 'api_key': 'Grayhat Warfare API key.', 'per_page': 'Maximum number of results per page (Max: 1000).', 'max_pages': 'Maximum number of pages to fetch.', 'pause': 'Number of seconds to wait between each API call.' } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return [ "DOMAIN_NAME", ] # What events this module produces def producedEvents(self): return [ 'CLOUD_STORAGE_BUCKET', 'CLOUD_STORAGE_BUCKET_OPEN', 'RAW_RIR_DATA' ] # Query Grayhat Warfare def query(self, keyword, start): params = urllib.parse.urlencode({ 'keywords': keyword.encode('raw_unicode_escape'), 'access_token': self.opts['api_key'] }) headers = { 'Accept': 'application/json', } res = self.sf.fetchUrl( f"https://buckets.grayhatwarfare.com/api/v1/buckets/{start}/{self.opts['per_page']}?{params}", headers=headers, timeout=15, useragent=self.opts['_useragent'], verify=True ) time.sleep(self.opts['pause']) if res['code'] != "200": self.error("Unable to fetch data from Grayhat Warfare API.") self.errorState = True return None if res['content'] is None: self.debug('No response from Grayhat Warfare API.') return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return if self.errorState: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "": self.error("You enabled sfp_grayhatwarfare but did not set an API key!") self.errorState = True return currentIndex = 0 currentPage = 0 maxPages = self.opts['max_pages'] perPage = self.opts['per_page'] keyword = self.sf.domainKeyword(eventData, self.opts['_internettlds']) while currentPage < maxPages: currentIndex = currentPage * perPage if self.checkForStop(): return if self.errorState: break data = self.query(keyword=keyword, start=currentIndex) if not data: return for row in data.get('buckets'): bucketName = row.get('bucket') bucketKeyword = bucketName.split('.')[0] self.debug(bucketKeyword) if bucketKeyword.startswith(keyword) or bucketKeyword.endswith(keyword): evt = SpiderFootEvent('CLOUD_STORAGE_BUCKET', bucketName, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent('CLOUD_STORAGE_BUCKET_OPEN', f"{bucketName}: {row.get('fileCount')} files found.", self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent('RAW_RIR_DATA', str(row), self.__name__, event) self.notifyListeners(evt) currentPage += 1 if data.get('buckets_count') < perPage: break # End of sfp_grayhatwarfare class ================================================ FILE: modules/sfp_greensnow.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_greensnow # Purpose: Checks if an IP address or netblock is malicious according to greensnow.co. # # Author: steve@binarypool.com # # Created: 16/05/2020 # Copyright: (c) Steve Micallef, 2020 # Licence: MIT # ------------------------------------------------------------------------------- from netaddr import IPAddress, IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_greensnow(SpiderFootPlugin): meta = { 'name': "Greensnow", 'summary': "Check if a netblock or IP address is malicious according to greensnow.co.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://greensnow.co/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://blocklist.greensnow.co/greensnow.txt", "https://greensnow.co/faq" ], 'favIcon': "https://greensnow.co/favicon.ico", 'logo': "https://greensnow.co/img/logo.png", 'description': "GreenSnow is a team consisting of the best specialists in computer security, " "we harvest a large number of IPs from different computers located around the world. " "GreenSnow is comparable with SpamHaus.org for attacks of any kind except for spam. " "Our list is updated automatically and you can withdraw at any time your IP address if it has been listed.", } } opts = { 'checkaffiliates': True, 'cacheperiod': 18, 'checknetblocks': True, 'checksubnets': True } optdescs = { 'checkaffiliates': "Apply checks to affiliate IP addresses?", 'cacheperiod': "Hours to cache list data before re-fetching.", 'checknetblocks': "Report if any malicious IPs are found within owned netblocks?", 'checksubnets': "Check if any malicious IPs are found within the same subnet of the target?" } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ 'IP_ADDRESS', 'AFFILIATE_IPADDR', 'NETBLOCK_OWNER', 'NETBLOCK_MEMBER', ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "BLACKLISTED_SUBNET", "BLACKLISTED_NETBLOCK", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK", "MALICIOUS_SUBNET", ] def query(self, qry, targetType): cid = "_greensnow" url = "https://blocklist.greensnow.co/greensnow.txt" data = dict() data["content"] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0)) if data["content"] is None: data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if data["code"] != "200": self.error(f"Unable to fetch {url}") self.errorState = True return None if data["content"] is None: self.error(f"Unable to fetch {url}") self.errorState = True return None self.sf.cachePut("sfmal_" + cid, data['content']) for line in data["content"].split('\n'): ip = line.strip().lower() if targetType == "netblock": try: if IPAddress(ip) in IPNetwork(qry): self.debug(f"{ip} found within netblock/subnet {qry} in greensnow.co list.") return f"https://greensnow.co/view/{ip}" except Exception as e: self.debug(f"Error encountered parsing: {e}") continue if targetType == "ip": if qry.lower() == ip: self.debug(f"{qry} found in greensnow.co list.") return f"https://greensnow.co/view/{ip}" return None def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName == 'IP_ADDRESS': targetType = 'ip' malicious_type = "MALICIOUS_IPADDR" blacklist_type = "BLACKLISTED_IPADDR" elif eventName == 'AFFILIATE_IPADDR': if not self.opts.get('checkaffiliates', False): return targetType = 'ip' malicious_type = "MALICIOUS_AFFILIATE_IPADDR" blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" elif eventName == 'NETBLOCK_OWNER': if not self.opts.get('checknetblocks', False): return targetType = 'netblock' malicious_type = "MALICIOUS_NETBLOCK" blacklist_type = "BLACKLISTED_NETBLOCK" elif eventName == 'NETBLOCK_MEMBER': if not self.opts.get('checksubnets', False): return targetType = 'netblock' malicious_type = "MALICIOUS_SUBNET" blacklist_type = "BLACKLISTED_SUBNET" else: self.debug(f"Unexpected event type {eventName}, skipping") return self.debug(f"Checking maliciousness of {eventData} ({eventName}) with greensnow.co") url = self.query(eventData, targetType) if not url: return text = f"greensnow.co [{eventData}]\n{url}" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_greensnow class ================================================ FILE: modules/sfp_grep_app.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_grep_app # Purpose: Searches grep.app API for domains, URLs and emails related to the # specified domain. # # Author: # # Created: 2020-04-12 # Copyright: (c) bcoles 2020 # Licence: MIT # ------------------------------------------------------------------------------- import json import math import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_grep_app(SpiderFootPlugin): meta = { 'name': "grep.app", 'summary': "Search grep.app API for links and emails related to the specified domain.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://grep.app/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [], 'favIcon': "https://grep.app/favicon-16x16.png", 'logo': "https://grep.app/apple-touch-icon.png", 'description': "grep.app searches code from over a half million public repositories on GitHub.\n" "It searches for the exact string you enter, including any punctuation or other characters.\n" "You can also search by regular expression, using the RE2 syntax.", } } # Default options opts = { 'max_pages': 20, 'dns_resolve': True, } # Option descriptions optdescs = { 'max_pages': "Maximum number of pages of results to fetch.", 'dns_resolve': "DNS resolve each identified domain." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["DOMAIN_NAME"] def producedEvents(self): return ["EMAILADDR", "EMAILADDR_GENERIC", "DOMAIN_NAME", "INTERNET_NAME", "RAW_RIR_DATA", "INTERNET_NAME_UNRESOLVED", "LINKED_URL_INTERNAL"] def query(self, qry, page): params = { 'q': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), 'page': str(page) } res = self.sf.fetchUrl("https://grep.app/api/search?" + urllib.parse.urlencode(params), useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout']) time.sleep(1) if res['content'] is None: return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if srcModuleName == 'sfp_grep_app': self.debug("Ignoring " + eventData + ", from self.") return hosts = list() page = 1 per_page = 10 pages = self.opts['max_pages'] while page <= pages: if self.checkForStop(): return if self.errorState: return res = self.query(eventData, page) if res is None: return facets = res.get('facets') if facets is None: return count = facets.get('count') if count is None: return last_page = math.ceil(count / per_page) if last_page is None: pages = 0 if last_page < pages: pages = last_page self.info("Parsing page " + str(page) + " of " + str(pages)) page += 1 hits = res.get('hits') if hits is None: return data = hits.get('hits') if data is None: return for result in data: if result is None: continue evt = SpiderFootEvent("RAW_RIR_DATA", str(result), self.__name__, event) self.notifyListeners(evt) content = result.get('content') if content is None: continue snippet = content.get('snippet') if snippet is None: continue links = self.sf.extractUrlsFromText(snippet.replace('', '').replace('', '')) if links: for link in links: if link in self.results: continue host = self.sf.urlFQDN(link) if not self.getTarget().matches(host, includeChildren=True, includeParents=True): continue hosts.append(host) if not self.getTarget().matches(self.sf.urlFQDN(link), includeChildren=True, includeParents=True): self.debug("Skipped unrelated link: " + link) continue self.debug('Found a link: ' + link) evt = SpiderFootEvent('LINKED_URL_INTERNAL', link, self.__name__, event) self.notifyListeners(evt) self.results[link] = True emails = SpiderFootHelpers.extractEmailsFromText(snippet.replace('', '').replace('', '')) if emails: for email in emails: if email in self.results: continue mail_domain = email.lower().split('@')[1] if not self.getTarget().matches(mail_domain, includeChildren=True, includeParents=True): self.debug("Skipped unrelated email address: " + email) continue self.info("Found e-mail address: " + email) if email.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" evt = SpiderFootEvent(evttype, email, self.__name__, event) self.notifyListeners(evt) self.results[email] = True for host in set(hosts): if self.checkForStop(): return if self.errorState: return if self.opts['dns_resolve'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host): self.debug(f"Host {host} could not be resolved") evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host, self.__name__, event) self.notifyListeners(evt) continue evt = SpiderFootEvent("INTERNET_NAME", host, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(host, self.opts["_internettlds"]): evt = SpiderFootEvent("DOMAIN_NAME", host, self.__name__, event) self.notifyListeners(evt) # End of sfp_grep_app class ================================================ FILE: modules/sfp_greynoise.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_greynoise # Purpose: Query GreyNoise's API # # Author: Steve Micallef # Updated By: Brad Chiappetta, GreyNoise # # Created: 20/11/2018 # Updated: 31-Aug-2022 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import time from datetime import datetime from netaddr import IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_greynoise(SpiderFootPlugin): meta = { "name": "GreyNoise", "summary": "Obtain IP enrichment data from GreyNoise", "flags": ["apikey"], "useCases": ["Investigate", "Passive"], "categories": ["Reputation Systems"], "dataSource": { "website": "https://greynoise.io/", "model": "FREE_AUTH_LIMITED", "references": ["https://docs.greynoise.io/", "https://viz.greynoise.io/signup"], "apiKeyInstructions": [ "Visit https://viz.greynoise.io/signup", "Sign up for a free account", "Navigate to https://viz.greynoise.io/account", "The API key is listed under 'API Key'", ], "favIcon": "https://viz.greynoise.io/favicon.ico", "logo": "https://viz.greynoise.io/_nuxt/img/greynoise-logo.dccd59d.png", "description": "At GreyNoise, we collect and analyze untargeted, widespread, " "and opportunistic scan and attack activity that reaches every server directly connected to the Internet. " "Mass scanners (such as Shodan and Censys), search engines, bots, worms, " "and crawlers generate logs and events omnidirectionally on every IP address in the IPv4 space. " "GreyNoise gives you the ability to filter this useless noise out.", }, } # Default options opts = { "api_key": "", "age_limit_days": 30, "netblocklookup": True, "maxnetblock": 24, "subnetlookup": True, "maxsubnet": 24 # 'asnlookup': True } # Option descriptions optdescs = { "api_key": "GreyNoise API Key.", "age_limit_days": "Ignore any records older than this many days. 0 = unlimited.", "netblocklookup": "Look up netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?", "maxnetblock": "If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", "subnetlookup": "Look up subnets which your target is a part of for blacklisting?", "maxsubnet": "If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)" # 'asnlookup': "Look up ASNs that your target is a member of?" } # Be sure to completely clear any class variables in setup() # or you run the risk of data persisting between scan runs. results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() # Clear / reset any other class member variables here # or you risk them persisting between threads. for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["IP_ADDRESS", "AFFILIATE_IPADDR", "NETBLOCK_MEMBER", "NETBLOCK_OWNER"] # What events this module produces def producedEvents(self): return [ "MALICIOUS_IPADDR", "MALICIOUS_ASN", "MALICIOUS_SUBNET", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK", "COMPANY_NAME", "GEOINFO", "BGP_AS_MEMBER", "OPERATING_SYSTEM", "RAW_RIR_DATA", ] def queryIP(self, qry, qry_type): gn_context_url = "https://api.greynoise.io/v2/noise/context/" gn_gnql_url = "https://api.greynoise.io/v2/experimental/gnql?query=" headers = {"key": self.opts["api_key"]} res = {} if qry_type == "ip": self.debug(f"Querying GreyNoise for IP: {qry}") res = {} ip_response = self.sf.fetchUrl( gn_context_url + qry, timeout=self.opts["_fetchtimeout"], useragent="greynoise-spiderfoot-v1.2.0", headers=headers, ) if ip_response["code"] == "200": res = json.loads(ip_response["content"]) else: self.debug(f"Querying GreyNoise for Netblock: {qry}") query_response = self.sf.fetchUrl( gn_gnql_url + qry, timeout=self.opts["_fetchtimeout"], useragent="greynoise-spiderfoot-v1.1.0", headers=headers, ) if query_response["code"] == "200": res = json.loads(query_response["content"]) if not res: self.error("Greynoise API key seems to have been rejected or you have exceeded usage limits.") self.errorState = True return None return res # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.error("You enabled sfp_greynoise but did not set an API key!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == "NETBLOCK_OWNER": if not self.opts["netblocklookup"]: return max_netblock = self.opts['maxnetblock'] if IPNetwork(eventData).prefixlen < max_netblock: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}") return if eventName == "NETBLOCK_MEMBER": if not self.opts["subnetlookup"]: return max_subnet = self.opts['maxsubnet'] if IPNetwork(eventData).prefixlen < max_subnet: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}") return if eventName == "IP_ADDRESS": evtType = "MALICIOUS_IPADDR" qryType = "ip" if eventName.startswith("NETBLOCK_"): evtType = "MALICIOUS_IPADDR" qryType = "netblock" if eventName == "AFFILIATE_IPADDR": evtType = "MALICIOUS_AFFILIATE_IPADDR" qryType = "ip" ret = self.queryIP(eventData, qryType) if not ret: return if "data" not in ret and "seen" not in ret: return if "data" in ret and len(ret["data"]) > 0: for rec in ret["data"]: if rec.get("seen", None): self.debug(f"Found threat info in Greynoise: {rec['ip']}") lastseen = rec.get("last_seen", "1970-01-01") lastseen_dt = datetime.strptime(lastseen, "%Y-%m-%d") lastseen_ts = int(time.mktime(lastseen_dt.timetuple())) age_limit_ts = int(time.time()) - (86400 * self.opts["age_limit_days"]) if self.opts["age_limit_days"] > 0 and lastseen_ts < age_limit_ts: self.debug(f"Record [{rec['ip']}] found but too old, skipping.") return # Only report meta data about the target, not affiliates if rec.get("metadata") and eventName == "IP_ADDRESS": met = rec.get("metadata") if met.get("country", "unknown") != "unknown": loc = "" if met.get("city"): loc = met.get("city") + ", " loc += met.get("country") e = SpiderFootEvent("GEOINFO", loc, self.__name__, event) self.notifyListeners(e) if met.get("asn", "unknown") != "unknown": asn = met.get("asn").replace("AS", "") e = SpiderFootEvent("BGP_AS_MEMBER", asn, self.__name__, event) self.notifyListeners(e) if met.get("organization", "unknown") != "unknown": e = SpiderFootEvent("COMPANY_NAME", met.get("organization"), self.__name__, event) self.notifyListeners(e) if met.get("os", "unknown") != "unknown": e = SpiderFootEvent("OPERATING_SYSTEM", met.get("os"), self.__name__, event) self.notifyListeners(e) e = SpiderFootEvent("RAW_RIR_DATA", str(rec), self.__name__, event) self.notifyListeners(e) if rec.get("classification"): descr = ( "GreyNoise - Mass-Scanning IP Detected [" + rec.get("ip") + "]\n - Classification: " + rec.get("classification") ) if rec.get("tags"): descr += "\n - " + "Scans For Tags: " + ", ".join(rec.get("tags")) if rec.get("cve"): descr += "\n - " + "Scans For CVEs: " + ", ".join(rec.get("cve")) if rec.get("raw_data") and not (rec.get("tags") or ret.get("cve")): descr += "\n - " + "Raw data: " + str(rec.get("raw_data")) descr += "\nhttps://viz.greynoise.io/ip/" + rec.get("ip") + "" e = SpiderFootEvent(evtType, descr, self.__name__, event) self.notifyListeners(e) if "seen" in ret: if ret.get("seen", None): lastseen = ret.get("last_seen", "1970-01-01") lastseen_dt = datetime.strptime(lastseen, "%Y-%m-%d") lastseen_ts = int(time.mktime(lastseen_dt.timetuple())) age_limit_ts = int(time.time()) - (86400 * self.opts["age_limit_days"]) if self.opts["age_limit_days"] > 0 and lastseen_ts < age_limit_ts: self.debug("Record found but too old, skipping.") return # Only report meta data about the target, not affiliates if ret.get("metadata") and eventName == "IP_ADDRESS": met = ret.get("metadata") if met.get("country", "unknown") != "unknown": loc = "" if met.get("city"): loc = met.get("city") + ", " loc += met.get("country") e = SpiderFootEvent("GEOINFO", loc, self.__name__, event) self.notifyListeners(e) if met.get("asn", "unknown") != "unknown": asn = met.get("asn").replace("AS", "") e = SpiderFootEvent("BGP_AS_MEMBER", asn, self.__name__, event) self.notifyListeners(e) if met.get("organization", "unknown") != "unknown": e = SpiderFootEvent("COMPANY_NAME", met.get("organization"), self.__name__, event) self.notifyListeners(e) if met.get("os", "unknown") != "unknown": e = SpiderFootEvent("OPERATING_SYSTEM", met.get("os"), self.__name__, event) self.notifyListeners(e) e = SpiderFootEvent("RAW_RIR_DATA", str(ret), self.__name__, event) self.notifyListeners(e) if ret.get("classification"): descr = ( "GreyNoise - Mass-Scanning IP Detected [" + eventData + "]\n - Classification: " + ret.get("classification") ) if ret.get("tags"): descr += "\n - " + "Scans For Tags: " + ", ".join(ret.get("tags")) if ret.get("cve"): descr += "\n - " + "Scans For CVEs: " + ", ".join(ret.get("cve")) if ret.get("raw_data") and not (ret.get("tags") or ret.get("cve")): descr += "\n - " + "Raw data: " + str(ret.get("raw_data")) descr += "\nhttps://viz.greynoise.io/ip/" + ret.get("ip") + "" e = SpiderFootEvent(evtType, descr, self.__name__, event) self.notifyListeners(e) # End of sfp_greynoise class ================================================ FILE: modules/sfp_greynoise_community.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_greynoise_community # Purpose: Query GreyNoise's Community API # # Author: Brad Chiappetta, GreyNoise # Updated By: Brad Chiappetta, GreyNoise # # Created: 31-Aug-2022 # Updated: 31-Aug-2022 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import time from datetime import datetime from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_greynoise_community(SpiderFootPlugin): meta = { "name": "GreyNoise Community", "summary": "Obtain IP enrichment data from GreyNoise Community API", "flags": ["apikey"], "useCases": ["Investigate", "Passive"], "categories": ["Reputation Systems"], "dataSource": { "website": "https://greynoise.io/", "model": "FREE_AUTH_LIMITED", "references": ["https://docs.greynoise.io/reference/get_v3-community-ip", "https://viz.greynoise.io/signup"], "apiKeyInstructions": [ "Visit https://viz.greynoise.io/signup", "Sign up for a free account", "Navigate to https://viz.greynoise.io/account/", "The API key is listed under 'API Key'", ], "favIcon": "https://viz.greynoise.io/favicon.ico", "logo": "https://viz.greynoise.io/_nuxt/img/greynoise-logo.dccd59d.png", "description": "At GreyNoise, we collect and analyze untargeted, widespread, " "and opportunistic scan and attack activity that reaches every server directly connected to the Internet. " "Mass scanners (such as Shodan and Censys), search engines, bots, worms, " "and crawlers generate logs and events omnidirectionally on every IP address in the IPv4 space. " "GreyNoise gives you the ability to filter this useless noise out.", }, } # Default options opts = { "api_key": "", "age_limit_days": 30, } # Option descriptions optdescs = { "api_key": "GreyNoise Community API Key.", "age_limit_days": "Ignore any records older than this many days. 0 = unlimited.", } # Be sure to completely clear any class variables in setup() # or you run the risk of data persisting between scan runs. results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() # Clear / reset any other class member variables here # or you risk them persisting between threads. for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["IP_ADDRESS", "AFFILIATE_IPADDR", "NETBLOCK_MEMBER", "NETBLOCK_OWNER"] # What events this module produces def producedEvents(self): return [ "MALICIOUS_IPADDR", "COMPANY_NAME", "RAW_RIR_DATA", ] def queryIP(self, qry, qry_type): gn_community_url = "https://api.greynoise.io/v3/community/" headers = {"key": self.opts["api_key"]} res = {} if qry_type == "ip": self.debug(f"Querying GreyNoise Community API for IP: {qry}") ip_res = {} ip_response = self.sf.fetchUrl( gn_community_url + qry, timeout=self.opts["_fetchtimeout"], useragent="greynoise-spiderfoot-community-v1.2.0", headers=headers, ) if ip_response["code"] == "200": ip_res = json.loads(ip_response["content"]) res = ip_res if not res: self.error("Greynoise API key seems to have been rejected or you have exceeded usage limits.") self.errorState = True return None return res # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.error("You enabled sfp_greynoise_community but did not set an API key!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == "IP_ADDRESS": evtType = "MALICIOUS_IPADDR" qryType = "ip" if eventName == "AFFILIATE_IPADDR": evtType = "MALICIOUS_AFFILIATE_IPADDR" qryType = "ip" ret = self.queryIP(eventData, qryType) if not ret: return if "data" not in ret and "noise" not in ret: return if "noise" in ret: if ret.get("noise", None): lastseen = ret.get("last_seen", "1970-01-01") lastseen_dt = datetime.strptime(lastseen, "%Y-%m-%d") lastseen_ts = int(time.mktime(lastseen_dt.timetuple())) age_limit_ts = int(time.time()) - (86400 * self.opts["age_limit_days"]) if self.opts["age_limit_days"] > 0 and lastseen_ts < age_limit_ts: self.debug("Record found but too old, skipping.") return e = SpiderFootEvent("RAW_RIR_DATA", str(ret), self.__name__, event) self.notifyListeners(e) # Only report meta data about the target, not affiliates if ret.get("name", "unknown") != "unknown": e = SpiderFootEvent("COMPANY_NAME", ret.get("name"), self.__name__, event) self.notifyListeners(e) if ret.get("classification"): descr = ( "GreyNoise - Mass-Scanning IP Detected [" + eventData + "]\n - Classification: " + ret.get("classification") ) descr += "\nhttps://viz.greynoise.io/ip/" + ret.get("ip") + "" e = SpiderFootEvent(evtType, descr, self.__name__, event) self.notifyListeners(e) # End of sfp_greynoise_community class ================================================ FILE: modules/sfp_h1nobbdde.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_h1.nobbd.de # Purpose: Query the the unofficial HackerOne disclosure timeline database # to see if our target appears. # # Author: Dhiraj Mishra # Created: 28/10/2018 # Copyright: (c) Dhiraj Mishra # Licence: MIT # ------------------------------------------------------------------------------- import re from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_h1nobbdde(SpiderFootPlugin): meta = { 'name': "HackerOne (Unofficial)", 'summary': "Check external vulnerability scanning/reporting service h1.nobbd.de to see if the target is listed.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Leaks, Dumps and Breaches"], 'dataSource': { 'website': "http://www.nobbd.de/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "http://www.nobbd.de/index.php#projekte", "https://twitter.com/disclosedh1" ], 'favIcon': "http://www.nobbd.de/favicon.ico", 'logo': "http://www.nobbd.de/favicon.ico", 'description': "Unofficial Bug Monitoring platform for HackerOne.", } } # Default options opts = { } # Option descriptions optdescs = { } # Be sure to completely clear any class variables in setup() # or you run the risk of data persisting between scan runs. results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() # Clear / reset any other class member variables here # or you risk them persisting between threads. for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["DOMAIN_NAME"] # What events this module produces def producedEvents(self): return ["VULNERABILITY_DISCLOSURE"] # Query h1.nobbd.de def queryOBB(self, qry): ret = list() url = "http://h1.nobbd.de/search.php?q=" + qry res = self.sf.fetchUrl(url, timeout=30, useragent=self.opts['_useragent']) if res['content'] is None: self.debug("No content returned from h1.nobbd.de") return None try: rx = re.compile("" + m[0] + "") except Exception as e: self.error(f"Error processing response from h1.nobbd.de: {e}") return None return ret def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data data = list() self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True obb = self.queryOBB(eventData) if obb: data.extend(obb) for n in data: e = SpiderFootEvent("VULNERABILITY_DISCLOSURE", n, self.__name__, event) self.notifyListeners(e) # End of sfp_h1nobbdde class ================================================ FILE: modules/sfp_hackertarget.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_hackertarget # Purpose: SpiderFoot plug-in to search HackerTarget.com for hosts sharing # the same IP. Optionally, also perform a basic TCP/UDP port scan # for commonly open ports using HackerTarget.com port scan tools. # # Author: Steve Micallef # # Created: 12/04/2014 # Copyright: (c) Steve Micallef 2014 # Licence: MIT # ------------------------------------------------------------------------------- import json import re import urllib.error import urllib.parse import urllib.request from netaddr import IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_hackertarget(SpiderFootPlugin): meta = { 'name': "HackerTarget", 'summary': "Search HackerTarget.com for hosts sharing the same IP.", 'flags': [], 'useCases': ["Footprint", "Investigate"], 'categories': ["Passive DNS"], 'dataSource': { 'website': "https://hackertarget.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://hackertarget.com/research/", "https://hackertarget.com/category/tools/" ], 'favIcon': "https://www.google.com/s2/favicons?domain=https://hackertarget.com/", 'logo': "https://hackertarget.com/wp-content/uploads/2018/03/online-security.png", 'description': "Simplify the security assessment process with hosted vulnerability scanners. " "From attack surface discovery to vulnerability identification, " "actionable network intelligence for IT & security operations. " "Proactively hunt for security weakness. " "Pivot from attack surface discovery to vulnerability identification.", } } opts = { 'cohostsamedomain': False, 'verify': True, 'netblocklookup': True, 'maxnetblock': 24, 'maxcohost': 100, 'http_headers': False, } optdescs = { 'cohostsamedomain': "Treat co-hosted sites on the same target domain as co-hosting?", 'verify': "Verify co-hosts are valid by checking if they still resolve to the shared IP.", 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?", 'maxnetblock': "If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'maxcohost': "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.", 'http_headers': "Retrieve IP HTTP headers using HackerTarget.com", } results = None errorState = False cohostcount = 0 def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.cohostcount = 0 self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "NETBLOCK_OWNER", 'DOMAIN_NAME_PARENT' ] def producedEvents(self): return [ "CO_HOSTED_SITE", "IP_ADDRESS", 'WEBSERVER_HTTPHEADERS', 'RAW_DNS_RECORDS', 'INTERNET_NAME', 'INTERNET_NAME_UNRESOLVED', 'DOMAIN_NAME', 'AFFILIATE_DOMAIN_NAME', 'AFFILIATE_INTERNET_NAME', 'AFFILIATE_INTERNET_NAME_UNRESOLVED' ] def httpHeaders(self, ip): """Retrieve HTTP headers for IP address Args: ip (str): IPv4 address Returns: dict: HTTP headers """ params = urllib.parse.urlencode({ 'q': ip }) res = self.sf.fetchUrl( f"https://api.hackertarget.com/httpheaders/?{params}", useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout'] ) if res['content'] is None: self.error(f"Unable to fetch HTTP headers for {ip} from HackerTarget.com.") return None if res['code'] == '429': self.error("You are being rate-limited by HackerTarget") self.errorState = True return None if not res['content'].startswith('HTTP/'): self.debug(f"Found no HTTP headers for {ip}") return None headers = dict() for header in res['content'].splitlines(): if ': ' not in header: continue k = header.split(': ')[0].lower() v = ': '.join(header.split(': ')[1:]) headers[k] = v return headers def zoneTransfer(self, ip): """Retrieve DNS zone transfer Args: ip (str): IPv4 address Returns: list: DNS zone """ params = urllib.parse.urlencode({ 'q': ip }) res = self.sf.fetchUrl( f"https://api.hackertarget.com/zonetransfer/?{params}", useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout'] ) if res['content'] is None: self.error(f"Unable to fetch DNS zone for {ip} from HackerTarget.com.") return None if res['code'] == '429': self.error("You are being rate-limited by HackerTarget") self.errorState = True return None records = list() for record in res['content'].splitlines(): if record.strip().startswith(';'): continue if record.strip() == '': continue records.append(record.strip()) return records def reverseIpLookup(self, ip): """Reverse lookup hosts on the same IP address Args: ip (str): IPv4 address Returns: list: (co)hosts on provided IP addresses """ params = urllib.parse.urlencode({ 'q': ip }) res = self.sf.fetchUrl( f"https://api.hackertarget.com/reverseiplookup/?{params}", useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout'] ) if res['content'] is None: self.error("Unable to fetch hackertarget.com content.") return None if res['code'] == '429': self.error("You are being rate-limited by HackerTarget") self.errorState = True return None if "No records" in res['content']: return None hosts = res['content'].split('\n') self.debug(f"Found {len(hosts)} on {ip}") return hosts def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.currentEventSrc = event if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if srcModuleName == "sfp_hackertarget" and eventName == "IP_ADDRESS": self.debug(f"Ignoring {eventName}, from self.") return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if eventName == 'NETBLOCK_OWNER': if not self.opts['netblocklookup']: return max_netblock = self.opts['maxnetblock'] net_size = IPNetwork(eventData).prefixlen if net_size < max_netblock: self.debug(f"Network size bigger than permitted: {net_size} > {max_netblock}") return if eventName == 'DOMAIN_NAME_PARENT': records = self.zoneTransfer(eventData) if not records: return evt = SpiderFootEvent('RAW_DNS_RECORDS', "\n".join(records), self.__name__, event) self.notifyListeners(evt) # Try and pull out individual records for row in records: pat = re.compile(r"^(\S+)\.?\s+\d+\s+IN\s+[AC].*", re.IGNORECASE | re.DOTALL) grps = re.findall(pat, row) if len(grps) == 0: continue hosts = list() for strdata in grps: self.debug("Matched: " + strdata) if strdata.endswith("."): hosts.append(strdata[:-1]) else: hosts.append(strdata) for host in set(hosts): if self.getTarget().matches(host, includeChildren=True, includeParents=True): evt_type = 'INTERNET_NAME' else: evt_type = 'AFFILIATE_INTERNET_NAME' if self.opts['verify'] and not self.sf.resolveHost(host) and not self.sf.resolveHost6(host): self.debug(f"Host {host} could not be resolved") evt_type += '_UNRESOLVED' evt = SpiderFootEvent(evt_type, host, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(host, self.opts['_internettlds']): if evt_type.startswith('AFFILIATE'): evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', host, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent('DOMAIN_NAME', host, self.__name__, event) self.notifyListeners(evt) return qrylist = list() if eventName.startswith("NETBLOCK_"): for ipaddr in IPNetwork(eventData): if str(ipaddr) not in self.results: qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: qrylist.append(eventData) self.results[eventData] = True myres = list() for ip in qrylist: if self.checkForStop(): return hosts = self.reverseIpLookup(ip) if not hosts: continue for h in hosts: if " " in h: continue self.info(f"Found something on same IP: {h}") if not self.opts['cohostsamedomain']: if self.getTarget().matches(h, includeParents=True): self.debug(f"Skipping {h} because it is on the same domain.") continue if h not in myres and h != ip: if self.opts['verify'] and not self.sf.validateIP(h, ip): self.debug(f"Host {h} no longer resolves to {ip}") continue if self.cohostcount < self.opts['maxcohost']: # Create an IP Address event stemming from the netblock as the # link to the co-host. if eventName == "NETBLOCK_OWNER": ipe = SpiderFootEvent("IP_ADDRESS", ip, self.__name__, event) self.notifyListeners(ipe) evt = SpiderFootEvent("CO_HOSTED_SITE", h.lower(), self.__name__, ipe) self.notifyListeners(evt) else: evt = SpiderFootEvent("CO_HOSTED_SITE", h.lower(), self.__name__, event) self.notifyListeners(evt) myres.append(h.lower()) self.cohostcount += 1 # For netblocks, we need to create the IP address event so that # the threat intel event is more meaningful. if eventName == 'NETBLOCK_OWNER': pevent = SpiderFootEvent("IP_ADDRESS", ip, self.__name__, event) self.notifyListeners(pevent) else: pevent = event if self.opts.get('http_headers', True): http_headers = self.httpHeaders(ip) if http_headers is not None: e = SpiderFootEvent('WEBSERVER_HTTPHEADERS', json.dumps(http_headers), self.__name__, pevent) e.actualSource = ip self.notifyListeners(e) # End of sfp_hackertarget class ================================================ FILE: modules/sfp_hashes.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_hashes # Purpose: SpiderFoot plug-in for scanning retrieved content by other # modules (such as sfp_spider) and identifying hashes # # Author: Steve Micallef # # Created: 24/01/2020 # Copyright: (c) Steve Micallef 2020 # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_hashes(SpiderFootPlugin): meta = { 'name': "Hash Extractor", 'summary': "Identify MD5 and SHA hashes in web content, files and more.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Content Analysis"] } # Default options opts = { # options specific to this module } # Option descriptions optdescs = { } def setup(self, sfc, userOpts=dict()): self.sf = sfc for opt in userOpts.keys(): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["TARGET_WEB_CONTENT", "BASE64_DATA", "LEAKSITE_CONTENT", "RAW_DNS_RECORDS", "RAW_FILE_META_DATA"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["HASH"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") hashes = SpiderFootHelpers.extractHashesFromText(eventData) for hashtup in hashes: hashalgo, hashval = hashtup evt = SpiderFootEvent("HASH", f"[{hashalgo}] {hashval}", self.__name__, event) if event.moduleDataSource: evt.moduleDataSource = event.moduleDataSource else: evt.moduleDataSource = "Unknown" self.notifyListeners(evt) # End of sfp_hashes class ================================================ FILE: modules/sfp_haveibeenpwned.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_haveibeenpwned # Purpose: Query haveibeenpwned.com to see if an e-mail account has been hacked. # # Author: Steve Micallef # # Created: 19/02/2015 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import re import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_haveibeenpwned(SpiderFootPlugin): meta = { 'name': "HaveIBeenPwned", 'summary': "Check HaveIBeenPwned.com for hacked e-mail addresses identified in breaches.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Leaks, Dumps and Breaches"], 'dataSource': { 'website': "https://haveibeenpwned.com/", 'model': "COMMERCIAL_ONLY", 'references': [ "https://haveibeenpwned.com/API/v3", "https://haveibeenpwned.com/FAQs" ], 'apiKeyInstructions': [ "Visit https://haveibeenpwned.com/API/Key", "Register an account", "Visit https://haveibeenpwned.com/API/Key", ], 'favIcon': "https://haveibeenpwned.com/favicon.ico", 'logo': "https://haveibeenpwned.com/favicon.ico", 'description': "Check if you have an account that has been compromised in a data breach.", } } # Default options opts = { "api_key": "" } # Option descriptions optdescs = { "api_key": "HaveIBeenPwned.com API key." } # Be sure to completely clear any class variables in setup() # or you run the risk of data persisting between scan runs. results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False # Clear / reset any other class member variables here # or you risk them persisting between threads. for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["EMAILADDR", "PHONE_NUMBER"] # What events this module produces def producedEvents(self): return ["EMAILADDR_COMPROMISED", "PHONE_NUMBER_COMPROMISED", "LEAKSITE_CONTENT", "LEAKSITE_URL"] def query(self, qry): if self.opts['api_key']: version = "3" else: version = "2" url = f"https://haveibeenpwned.com/api/v{version}/breachedaccount/{qry}" hdrs = {"Accept": f"application/vnd.haveibeenpwned.v{version}+json"} retry = 0 if self.opts['api_key']: hdrs['hibp-api-key'] = self.opts['api_key'] while retry < 2: # https://haveibeenpwned.com/API/v2#RateLimiting time.sleep(1.5) res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=hdrs) if res['code'] == "200": break if res['code'] == "404": return None if res['code'] == "429": # Back off a little further time.sleep(2) retry += 1 if res['code'] == "401": self.error("Failed to authenticate key with HaveIBeenPwned.com.") self.errorState = True return None try: return json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from HaveIBeenPwned?: {e}") return None def queryPaste(self, qry): url = f"https://haveibeenpwned.com/api/v3/pasteaccount/{qry}" headers = { 'Accept': "application/json", 'hibp-api-key': self.opts['api_key'] } retry = 0 while retry < 2: # https://haveibeenpwned.com/API/v2#RateLimiting time.sleep(1.5) res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot", headers=headers) if res['code'] == "200": break if res['code'] == "404": return None if res['code'] == "429": # Back off a little further time.sleep(2) retry += 1 if res['code'] == "401": self.error("Failed to authenticate key with HaveIBeenPwned.com.") self.errorState = True return None try: return json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from HaveIBeenPwned?: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if self.opts['api_key'] == "": self.error("You enabled sfp_haveibeenpwned but did not set an API key!") self.errorState = True return self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True data = self.query(eventData) if data is not None: for n in data: try: site = n["Name"] except Exception as e: self.debug(f"Unable to parse result from HaveIBeenPwned?: {e}") continue # Notify other modules of what you've found if eventName == 'EMAILADDR': e = SpiderFootEvent("EMAILADDR_COMPROMISED", eventData + " [" + site + "]", self.__name__, event) else: e = SpiderFootEvent("PHONE_NUMBER_COMPROMISED", eventData + " [" + site + "]", self.__name__, event) self.notifyListeners(e) # This API endpoint doesn't support phone numbers if eventName == "PHONE_NUMBER": return pasteData = self.queryPaste(eventData) if pasteData is None: return sites = { "Pastebin": "https://pastebin.com/", "Pastie": "http://pastie.org/p/", "Slexy": "https://slexy.org/view/", "Ghostbin": "https://ghostbin.com/paste/", "JustPaste": "https://justpaste.it/", } links = set() for n in pasteData: try: source = n.get("Source") site = source if source in sites: site = f"{sites[n.get('Source')]}{n.get('Id')}" links.add(site) except Exception as e: self.debug(f"Unable to parse result from HaveIBeenPwned?: {e}") continue for link in links: try: self.debug("Found a link: " + link) if self.checkForStop(): return res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if res['content'] is None: self.debug(f"Ignoring {link} as no data returned") continue if re.search(r"[^a-zA-Z\-\_0-9]" + re.escape(eventData) + r"[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None: continue evt1 = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event) self.notifyListeners(evt1) evt2 = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, evt1) self.notifyListeners(evt2) except Exception as e: self.debug(f"Unable to parse result from HaveIBeenPwned?: {e}") continue # End of sfp_haveibeenpwned class ================================================ FILE: modules/sfp_honeypot.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_honeypot # Purpose: SpiderFoot plug-in for looking up whether IPs appear in the # ProjectHoneyPot.org database. # # Author: Steve Micallef # # Created: 16/04/2014 # Copyright: (c) Steve Micallef 2014 # Licence: MIT # ------------------------------------------------------------------------------- from netaddr import IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_honeypot(SpiderFootPlugin): meta = { 'name': "Project Honey Pot", 'summary': "Query the Project Honey Pot database for IP addresses.", 'flags': ["apikey"], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://www.projecthoneypot.org/", 'model': "FREE_AUTH_UNLIMITED", 'references': [ "https://www.projecthoneypot.org/httpbl_api.php", "https://www.projecthoneypot.org/services_overview.php", "https://www.projecthoneypot.org/faq.php" ], 'apiKeyInstructions': [ "Visit https://www.projecthoneypot.org", "Sign up for a free account", "Navigate to https://www.projecthoneypot.org/httpbl_configure.php'", "Request for an API key", "The API key is listed under 'Your http:BL Access Key'" ], 'favIcon': "https://www.projecthoneypot.org/favicon.ico", 'logo': "https://www.projecthoneypot.org/images/php_logo.gif", 'description': "Project Honey Pot is the first and only distributed system for identifying spammers " "and the spambots they use to scrape addresses from your website. " "Using the Project Honey Pot system you can install addresses " "that are custom-tagged to the time and IP address of a visitor to your site. " "If one of these addresses begins receiving email we not only can tell that the messages are spam, " "but also the exact moment when the address was harvested and the IP address that gathered it.", } } opts = { 'api_key': "", 'searchengine': False, 'threatscore': 0, 'timelimit': 30, 'netblocklookup': True, 'maxnetblock': 24, 'subnetlookup': True, 'maxsubnet': 24 } optdescs = { 'api_key': "ProjectHoneyPot.org API key.", 'searchengine': "Include entries considered search engines?", 'threatscore': "Threat score minimum, 0 being everything and 255 being only the most serious.", 'timelimit': "Maximum days old an entry can be. 255 is the maximum, 0 means you'll get nothing.", 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible hosts on the same target subdomain/domain?", 'maxnetblock': "If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'subnetlookup': "Look up all IPs on subnets which your target is a part of?", 'maxsubnet': "If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)" } results = None errorState = False # Status codes according to: # http://www.projecthoneypot.org/httpbl_api.php statuses = { "0": "Search Engine", "1": "Suspicious", "2": "Harvester", "3": "Suspicious & Harvester", "4": "Comment Spammer", "5": "Suspicious & Comment Spammer", "6": "Harvester & Comment Spammer", "7": "Suspicious & Harvester & Comment Spammer", "8": "Unknown (8)", "9": "Unknown (9)", "10": "Unknown (10)" } def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "AFFILIATE_IPADDR", "NETBLOCK_OWNER", "NETBLOCK_MEMBER", ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "BLACKLISTED_NETBLOCK", "BLACKLISTED_SUBNET", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK", "MALICIOUS_SUBNET", ] # Swap 1.2.3.4 to 4.3.2.1 def reverseAddr(self, ipaddr): return '.'.join(reversed(ipaddr.split('.'))) # Returns text about the IP status returned from DNS def parseDNS(self, addr): bits = addr.split(".") if int(bits[1]) > self.opts['timelimit']: return None if int(bits[2]) < self.opts['threatscore']: return None if int(bits[3]) == 0 and self.opts['searchengine']: return None return f"{self.statuses[bits[3]]}\nLast Activity: {bits[1]} days ago\nThreat Level: {bits[2]}" def queryAddr(self, qaddr, parentEvent): eventName = parentEvent.eventType text = None try: lookup = f"{self.opts['api_key']}.{self.reverseAddr(qaddr)}.dnsbl.httpbl.org" self.debug(f"Checking ProjectHoneyPot: {lookup}") addrs = self.sf.resolveHost(lookup) if not addrs: return self.debug(f"Addresses returned: {addrs}") for addr in addrs: text = self.parseDNS(addr) if text is not None: break except Exception as e: self.debug(f"ProjectHoneyPot did not resolve {qaddr} / {lookup}: {e}") if not text: return if eventName == "AFFILIATE_IPADDR": malicious_type = "MALICIOUS_AFFILIATE_IPADDR" blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" elif eventName == "IP_ADDRESS": malicious_type = "MALICIOUS_IPADDR" blacklist_type = "BLACKLISTED_IPADDR" elif eventName == "NETBLOCK_OWNER": malicious_type = "MALICIOUS_NETBLOCK" blacklist_type = "BLACKLISTED_NETBLOCK" elif eventName == "NETBLOCK_MEMBER": malicious_type = "MALICIOUS_SUBNET" blacklist_type = "BLACKLISTED_SUBNET" else: self.debug(f"Unexpected event type {eventName}, skipping") return url = f"https://www.projecthoneypot.org/ip_{qaddr}" evt = SpiderFootEvent(malicious_type, f"ProjectHoneyPot ({qaddr}): {text}\n{url}", self.__name__, parentEvent) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, f"ProjectHoneyPot ({qaddr}): {text}\n{url}", self.__name__, parentEvent) self.notifyListeners(evt) def handleEvent(self, event): eventName = event.eventType eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {event.module}") if not self.opts['api_key']: self.error(f"You enabled {self.__class__.__name__} but did not set an API key!") self.errorState = True return if eventData in self.results: return self.results[eventData] = True if eventName == 'NETBLOCK_OWNER': if not self.opts['netblocklookup']: return max_netblock = self.opts['maxnetblock'] if IPNetwork(eventData).prefixlen < max_netblock: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_netblock}") return if eventName == 'NETBLOCK_MEMBER': if not self.opts['subnetlookup']: return max_subnet = self.opts['maxsubnet'] if IPNetwork(eventData).prefixlen < max_subnet: self.debug(f"Network size bigger than permitted: {IPNetwork(eventData).prefixlen} > {max_subnet}") return if eventName.startswith("NETBLOCK_"): for addr in IPNetwork(eventData): if self.checkForStop(): return self.queryAddr(str(addr), event) else: self.queryAddr(eventData, event) # End of sfp_honeypot class ================================================ FILE: modules/sfp_hosting.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_hosting # Purpose: SpiderFoot plug-in for looking up whether IPs/Netblocks/Domains # appear in an IP categorization table of hosting providers. # # Author: Steve Micallef # # Created: 16/08/2015 # Copyright: (c) Steve Micallef 2015 # Licence: MIT # ------------------------------------------------------------------------------- from netaddr import IPAddress from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_hosting(SpiderFootPlugin): meta = { 'name': "Hosting Provider Identifier", 'summary': "Find out if any IP addresses identified fall within known 3rd party hosting ranges, e.g. Amazon, Azure, etc.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Content Analysis"] } # Default options opts = { } # Option descriptions optdescs = { } # Target results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.__dataSource__ = "DNS" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['IP_ADDRESS'] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["PROVIDER_HOSTING"] def queryAddr(self, qaddr): data = dict() url = "https://raw.githubusercontent.com/client9/ipcat/master/datacenters.csv" data['content'] = self.sf.cacheGet("sfipcat", 48) if data['content'] is None: data = self.sf.fetchUrl(url, useragent=self.opts['_useragent']) if data['content'] is None: self.error("Unable to fetch " + url) return None self.sf.cachePut("sfipcat", data['content']) for line in data['content'].split('\n'): if "," not in line: continue try: [start, end, title, url] = line.split(",") except Exception: continue try: if IPAddress(qaddr) > IPAddress(start) and IPAddress(qaddr) < IPAddress(end): return [title, url] except Exception as e: self.debug("Encountered an issue processing an IP: " + str(e)) continue return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: return self.results[eventData] = True ret = self.queryAddr(eventData) if ret: evt = SpiderFootEvent("PROVIDER_HOSTING", ret[0] + ": " + ret[1], self.__name__, event) self.notifyListeners(evt) # End of sfp_hosting class ================================================ FILE: modules/sfp_hostio.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_hostio # Purpose: Host.io database query module # # Author: Lev Trubach # # Created: 2020-08-21 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_hostio(SpiderFootPlugin): meta = { "name": "Host.io", "summary": "Obtain information about domain names from host.io.", 'flags': ["apikey"], "useCases": ["Passive"], "categories": ["Passive DNS"], "dataSource": { "website": "https://host.io", "model": "FREE_AUTH_LIMITED", "references": ["https://host.io/docs"], "apiKeyInstructions": [ "Visit https://host.io/signup", "Register a free account", "Visit https://host.io/dashboard and use the authentication token provided", ], "favIcon": "https://host.io/static/images/hostio/favicon.png?v2", "logo": "https://host.io/static/images/hostio/favicon.png?v2", # Seems like they embed it as SVG "description": "We collect data on every known domain name, from every TLD, and update it every month. " "Our data includes DNS records and website data for each of the domains." "We process terabytes of data and summarize it to produce our final results. " "Browse through our site to see backlinks, redirects, server details or IP address " "and hosting provider details courtesy of IPinfo.io.", }, } opts = { "api_key": "", } optdescs = { "api_key": "Host.io API Key.", } errorState = False def setup(self, sfc, userOpts=None): if userOpts is None: userOpts = {} self.sf = sfc self.results = self.tempStorage() self.opts.update(userOpts) def watchedEvents(self): return [ "DOMAIN_NAME", ] # What events this module produces def producedEvents(self): return [ "IP_ADDRESS", "RAW_RIR_DATA", "EMAILADDR", "WEB_ANALYTICS_ID", "WEBSERVER_TECHNOLOGY", "PHYSICAL_COORDINATES", "DESCRIPTION_ABSTRACT", "GEOINFO", ] # When querying third parties, it's best to have a dedicated function # to do so and avoid putting it in handleEvent() def handle_error_response(self, qry, res): try: error_info = json.loads(res["content"]) except Exception: error_info = None if error_info: error_message = error_info.get("error") else: error_message = None if error_message: error_str = f", message {error_message}" else: error_str = "" self.info(f"Failed to get results for {qry}, code {res['code']}{error_str}") def query(self, qry): res = self.sf.fetchUrl( f"https://host.io/api/full/{qry}", headers={"Authorization": f"Bearer {self.opts['api_key']}"}, timeout=self.opts["_fetchtimeout"], useragent="SpiderFoot", ) if res["code"] != "200": self.handle_error_response(qry, res) return None if res["content"] is None: self.info(f"No Host.io info found for {qry}") return None try: return json.loads(res["content"]) except Exception as e: self.error(f"Error processing JSON response from Host.io: {e}") return None def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.error( f"You enabled {self.__class__.__name__} but did not set an API key!" ) self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData} as already mapped.") return self.results[eventData] = True data = self.query(event.data) if not data: self.error(f"No data received for {event.data}") return found = False ipinfo = data.get("ipinfo") if ipinfo and isinstance(ipinfo, dict): for address, ip_data in data["ipinfo"].items(): # Not supporting co-hosted sites yet if not self.sf.validIP(address): continue evt = SpiderFootEvent("IP_ADDRESS", address, self.__name__, event) self.notifyListeners(evt) found = True loc = ip_data.get("loc") if loc and isinstance(loc, str): loc_evt = SpiderFootEvent( "PHYSICAL_COORDINATES", loc, self.__name__, evt ) self.notifyListeners(loc_evt) found = True geo_info = ', '.join(filter(None, (ip_data.get(k) for k in ("city", "region", "country")))) if geo_info: geo_info_evt = SpiderFootEvent( "GEOINFO", geo_info, self.__name__, evt ) self.notifyListeners(geo_info_evt) found = True related = data.get("related") if related and isinstance(related, dict): email_section = related.get("email") if email_section and isinstance(email_section, list): for email_data in email_section: if isinstance(email_data, dict): value = email_data["value"] if value and isinstance(value, str): for email in value.split(','): email = email.strip('.') evt = SpiderFootEvent( "EMAILADDR", email, self.__name__, event ) self.notifyListeners(evt) found = True web = data.get("web") if web and isinstance(web, dict): server = web.get("server") if server and isinstance(server, str): evt = SpiderFootEvent( "WEBSERVER_TECHNOLOGY", server, self.__name__, event ) self.notifyListeners(evt) found = True google_analytics = web.get("googleanalytics") if google_analytics and isinstance(google_analytics, str): evt = SpiderFootEvent( "WEB_ANALYTICS_ID", google_analytics, self.__name__, event ) self.notifyListeners(evt) found = True title = web.get("title") if title and isinstance(title, str): evt = SpiderFootEvent( "DESCRIPTION_ABSTRACT", title, self.__name__, event ) self.notifyListeners(evt) found = True if found: evt = SpiderFootEvent( "RAW_RIR_DATA", json.dumps(data), self.__name__, event ) self.notifyListeners(evt) ================================================ FILE: modules/sfp_hunter.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_hunter # Purpose: Query hunter.io using their API. # # Author: Steve Micallef # # Created: 22/02/2017 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_hunter(SpiderFootPlugin): meta = { 'name': "Hunter.io", 'summary': "Check for e-mail addresses and names on hunter.io.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://hunter.io/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://hunter.io/api" ], 'apiKeyInstructions': [ "Visit https://hunter.io/", "Sign up for a free account", "Click on 'Account Settings'", "Click on 'API'", "The API key is listed under 'Your API Key'" ], 'favIcon': "https://hunter.io/assets/head/favicon-d5796c45076e78aa5cf22dd53c5a4a54155062224bac758a412f3a849f38690b.ico", 'logo': "https://hunter.io/assets/head/touch-icon-iphone-fd9330e31552eeaa12b177489943de997551bfd991c4c44e8c3d572e78aea5f3.png", 'description': "Hunter lets you find email addresses in seconds and connect with the people that matter for your business.\n" "The Domain Search lists all the people working in a company with their name " "and email address found on the web. With 100+ million email addresses indexed, " "effective search filters and scoring, it's the most powerful email-finding tool ever created.", } } # Default options opts = { "api_key": "" } # Option descriptions optdescs = { "api_key": "Hunter.io API key." } # Be sure to completely clear any class variables in setup() # or you run the risk of data persisting between scan runs. results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False # Clear / reset any other class member variables here # or you risk them persisting between threads. for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME", "INTERNET_NAME"] # What events this module produces def producedEvents(self): return ["EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA"] def query(self, qry, offset=0, limit=10): params = { "domain": qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), "api_key": self.opts['api_key'], "offset": str(offset), "limit": str(limit) } url = f"https://api.hunter.io/v2/domain-search?{urllib.parse.urlencode(params)}" res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot") if res['code'] == "404": return None if not res['content']: return None try: return json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from hunter.io: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if self.opts['api_key'] == "": self.error("You enabled sfp_hunter but did not set an API key!") self.errorState = True return data = self.query(eventData, 0, 10) if not data: return if "data" not in data: return # Check if we have more results on further pages if "meta" in data: maxgoal = data['meta'].get('results', 10) else: maxgoal = 10 rescount = len(data['data'].get('emails', list())) while rescount <= maxgoal: for email in data['data'].get('emails', list()): # Notify other modules of what you've found em = email.get('value') if not em: continue if em.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" e = SpiderFootEvent(evttype, em, self.__name__, event) self.notifyListeners(e) if 'first_name' in email and 'last_name' in email: if email['first_name'] is not None and email['last_name'] is not None: n = email['first_name'] + " " + email['last_name'] e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + n, self.__name__, event) self.notifyListeners(e) if rescount >= maxgoal: return data = self.query(eventData, rescount, 10) if data is None: return if "data" not in data: return rescount += len(data['data'].get('emails', list())) # End of sfp_hunter class ================================================ FILE: modules/sfp_hybrid_analysis.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_hybrid_analysis # Purpose: Search Hybrid Analysis for domains and URLs related to the target. # # Authors: # # Created: 2020-08-09 # Copyright: (c) bcoles 2020 # Licence: MIT # ------------------------------------------------------------------------------- import json import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_hybrid_analysis(SpiderFootPlugin): meta = { 'name': "Hybrid Analysis", 'summary': "Search Hybrid Analysis for domains and URLs related to the target.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://www.hybrid-analysis.com", 'model': "FREE_AUTH_UNLIMITED", 'references': [ "https://www.hybrid-analysis.com/knowledge-base", "https://www.hybrid-analysis.com/docs/api/v2" ], 'apiKeyInstructions': [ "Visit https://www.hybrid-analysis.com/signup", "Register a free account", "Navigate to https://www.hybrid-analysis.com/my-account?tab=%23api-key-tab", "Create an API Key", "The API key is listed under 'API Key'" ], 'favIcon': "https://www.hybrid-analysis.com/favicon.ico", 'logo': "https://www.hybrid-analysis.com/img/logo.svg", 'description': "A free malware analysis service for the community. " "Using this service you can submit files for in-depth static and dynamic analysis.", } } # Default options opts = { "api_key": "", "verify": True, "delay": 1 } # Option descriptions optdescs = { "api_key": "Hybrid Analysis API key.", "verify": "Verify identified domains still resolve to the associated specified IP address.", "delay": "Delay between requests, in seconds." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in userOpts.keys(): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["IP_ADDRESS", "DOMAIN_NAME"] def producedEvents(self): return ["RAW_RIR_DATA", "INTERNET_NAME", "DOMAIN_NAME", "LINKED_URL_INTERNAL"] def queryDomain(self, qry): """Query a domain Args: qry (str): domain Returns: str: API response as JSON """ params = { "domain": qry.encode('raw_unicode_escape').decode("ascii", errors='replace') } headers = { "Accept": "application/json", 'api-key': self.opts['api_key'] } res = self.sf.fetchUrl( 'https://www.hybrid-analysis.com/api/v2/search/terms', headers=headers, timeout=15, useragent="Falcon Sandbox", postData=params ) time.sleep(self.opts['delay']) return self.parseApiResponse(res) def queryHost(self, qry): """Query a host Args: qry (str): host Returns: str: API response as JSON """ params = { "host": qry.encode('raw_unicode_escape').decode("ascii", errors='replace') } headers = { "Accept": "application/json", 'api-key': self.opts['api_key'] } res = self.sf.fetchUrl( 'https://www.hybrid-analysis.com/api/v2/search/terms', headers=headers, timeout=15, useragent="Falcon Sandbox", postData=params ) time.sleep(self.opts['delay']) return self.parseApiResponse(res) def queryHash(self, qry): """Query a hash Args: qry (str): hash Returns: str: API response as JSON """ params = { "hash": qry.encode('raw_unicode_escape').decode("ascii", errors='replace') } headers = { "Accept": "application/json", 'api-key': self.opts['api_key'] } res = self.sf.fetchUrl( 'https://www.hybrid-analysis.com/api/v2/search/hash', headers=headers, timeout=15, useragent="Falcon Sandbox", postData=params ) time.sleep(self.opts['delay']) return self.parseApiResponse(res) def parseApiResponse(self, res: dict): """Parse HTTP response from API Args: res (dict): HTTP response from SpiderFoot.fetchUrl() Returns: str: API response as JSON """ if not res: self.error("No response from Hybrid Analysis.") return None if res['code'] == '400': self.error("Failed to retrieve content from Hybrid Analysis: Invalid request") self.debug(f"API response: {res['content']}") return None # Future proofing - Hybrid Analysis does not implement rate limiting if res['code'] == '429': self.error("Failed to retrieve content from Hybrid Analysis: rate limit exceeded") self.errorState = True return None # Catch all non-200 status codes, and presume something went wrong if res['code'] != '200': self.error(f"Failed to retrieve content from Hybrid Analysis: Unexpected response status {res['code']}") self.errorState = True return None if res['content'] is None: return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventName not in ["IP_ADDRESS", "DOMAIN_NAME"]: return if eventName == "IP_ADDRESS": data = self.queryHost(eventData) elif eventName == "DOMAIN_NAME": data = self.queryDomain(eventData) else: return if data is None: self.debug(f"No information found for{eventData}") return results = data.get("result") if not results: return hashes = [] for result in results: file_hash = result.get('sha256') if file_hash: hashes.append(file_hash) if not hashes: return self.info(f"Found {len(hashes)} results for {eventData}") evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) urls = [] domains = [] for file_hash in hashes: results = self.queryHash(file_hash) if not results: self.debug(f"No information found for hash {file_hash}") continue evt = SpiderFootEvent('RAW_RIR_DATA', str(results), self.__name__, event) self.notifyListeners(evt) for result in results: if not result: continue result_domains = result.get('domains') if result_domains: for r in result_domains: domains.append(r) submissions = result.get('submissions') if submissions: for submission in submissions: url = submission.get('url') if url: urls.append(url) for url in set(urls): host = self.sf.urlFQDN(url.lower()) if not self.getTarget().matches(host, includeChildren=True, includeParents=True): continue domains.append(host) evt = SpiderFootEvent('LINKED_URL_INTERNAL', url, self.__name__, event) self.notifyListeners(evt) for domain in set(domains): if self.checkForStop(): return if domain in self.results: continue if not self.getTarget().matches(domain, includeChildren=True, includeParents=True): continue if self.opts['verify'] and not self.sf.resolveHost(domain) and not self.sf.resolveHost6(domain): self.debug(f"Host {domain} could not be resolved") evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", domain, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent("INTERNET_NAME", domain, self.__name__, event) self.notifyListeners(evt) # End of sfp_hybrid_analysis class ================================================ FILE: modules/sfp_iban.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_iban # Purpose: SpiderFoot plug-in for scanning retrieved content by other # modules (such as sfp_spider) and identifying IBANs. # # Author: Krishnasis Mandal # # Created: 26/04/2020 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_iban(SpiderFootPlugin): meta = { 'name': "IBAN Number Extractor", 'summary': "Identify International Bank Account Numbers (IBANs) in any data.", 'flags': ["errorprone"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Content Analysis"] } opts = { } optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() # Override datasource for sfp_iban module self.__dataSource__ = "Target Website" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["TARGET_WEB_CONTENT", "DARKNET_MENTION_CONTENT", "LEAKSITE_CONTENT"] # What events this module produces def producedEvents(self): return ["IBAN_NUMBER"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") ibans = SpiderFootHelpers.extractIbansFromText(eventData) for ibanNumber in set(ibans): self.info(f"Found IBAN number: {ibanNumber}") evt = SpiderFootEvent("IBAN_NUMBER", ibanNumber, self.__name__, event) if event.moduleDataSource: evt.moduleDataSource = event.moduleDataSource else: evt.moduleDataSource = "Unknown" self.notifyListeners(evt) # End of sfp_iban class ================================================ FILE: modules/sfp_iknowwhatyoudownload.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_iknowwhatyoudownload # Purpose: Query iknowwhatyoudownload.com for IP addresses using torrents. # # Author: Steve Micallef # # Created: 03/09/2018 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_iknowwhatyoudownload(SpiderFootPlugin): meta = { 'name': "Iknowwhatyoudownload.com", 'summary': "Check iknowwhatyoudownload.com for IP addresses that have been using torrents.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Secondary Networks"], 'dataSource': { 'website': "https://iknowwhatyoudownload.com/en/peer/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://iknowwhatyoudownload.com/en/api/", "https://iknowwhatyoudownload.com/en/link/", "https://iknowwhatyoudownload.com/en/peer/" ], 'apiKeyInstructions': [ "Visit https://iknowwhatyoudownload.com/en/api/", "Request Demo Key with email id", "The API key will be sent to your email" ], 'favIcon': "https://iknowwhatyoudownload.com/assets/img/utorrent2.png", 'logo': "https://iknowwhatyoudownload.com/assets/img/logo.png", 'description': "Our system collects torrent files in two ways: parsing torrent sites, and listening DHT network. " "The system contains more than 7 million torrents (as of Oct 2021) which were classified and which are using now " "for collecting peer sharing facts (up to 200.000.000 daily).", } } opts = { "daysback": 30, "api_key": "" } optdescs = { "daysback": "How far back (in days) to look for activity.", "api_key": "Iknowwhatyoudownload.com API key." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["IP_ADDRESS", "IPV6_ADDRESS"] def producedEvents(self): return ["MALICIOUS_IPADDR"] def query(self, qry): """Search iknowwhatyoudownload.com for an IPv4/IPv6 address. Args: qry: IPv4/IPv6 address Returns: dict: JSON response containing dowloaded content """ params = urllib.parse.urlencode({ 'ip': qry, 'days': self.opts['daysback'], 'key': self.opts['api_key'], }) res = self.sf.fetchUrl( f"https://api.antitor.com/history/peer/?{params}", timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot" ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from iknowwhatyoudownload.com.") return None if res['content'] is None: self.info(f"No results for {qry} from iknowwhatyoudownload.com") return None try: data = json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from iknowwhatyoudownload.com: {e}") return None error = data.get('error') if error and error == "INVALID_DAYS": self.errorState = True self.error(f"The number of days you have configured ({self.opts['daysback']}) was not accepted. If you have the demo key, try 30 days or less.") return None contents = data.get('contents') if not contents: return None return contents def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.errorState: return if self.opts['api_key'] == "": self.error("You enabled sfp_iknowwhatyoudownload but did not set an API key!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True data = self.query(eventData) if not data: return retdata = [f"https://iknowwhatyoudownload.com/en/peer/?ip={eventData}"] for d in data: torrent = d.get('torrent') if not torrent: continue download_name = torrent.get('name') download_date = d.get("endDate", "Date unknown") retdata.append(f"{download_name} ({download_date})") e = SpiderFootEvent("MALICIOUS_IPADDR", "\n".join(retdata), self.__name__, event) self.notifyListeners(e) # End of sfp_iknowwhatyoudownload class ================================================ FILE: modules/sfp_intelx.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_intelx # Purpose: Query IntelligenceX (intelx.io) for identified IP addresses, # domains, e-mail addresses and phone numbers. # # Author: Steve Micallef # # Created: 28/04/2019 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import datetime import json import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_intelx(SpiderFootPlugin): meta = { 'name': "IntelligenceX", 'summary': "Obtain information from IntelligenceX about identified IP addresses, domains, e-mail addresses and phone numbers.", 'flags': ["apikey"], 'useCases': ["Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://intelx.io/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://ginseg.com/wp-content/uploads/sites/2/2019/07/Manual-Intelligence-X-API.pdf", "https://blog.intelx.io/2019/01/25/new-developer-tab/", "https://github.com/IntelligenceX/SDK" ], 'apiKeyInstructions': [ "Visit https://intelx.io/", "Register a free account", "Navigate to https://intelx.io/account?tab=developer", "The API key is listed under 'Your API details'" ], 'favIcon': "https://intelx.io/favicon/favicon-32x32.png", 'logo': "https://intelx.io/assets/img/IntelligenceX.svg", 'description': "Intelligence X is an independent European technology company founded in 2018 by Peter Kleissner. " "Its mission is to develop and maintain the search engine and data archive.\n" "The search works with selectors, i.e. specific search terms such as " "email addresses, domains, URLs, IPs, CIDRs, Bitcoin addresses, IPFS hashes, etc.\n" "It searches in places such as the darknet, document sharing platforms, whois data, public data leaks and others.\n" "It keeps a historical data archive of results, " "similar to how the Wayback Machine from archive.org stores historical copies of websites.", } } # Default options opts = { "api_key": "", "base_url": "2.intelx.io", "checkcohosts": False, "checkaffiliates": False, 'netblocklookup': False, 'maxnetblock': 24, 'subnetlookup': False, 'maxsubnet': 24, 'maxage': 90 } # Option descriptions optdescs = { "api_key": "IntelligenceX API key.", "base_url": "API URL, as provided in your IntelligenceX account settings.", "checkcohosts": "Check co-hosted sites?", "checkaffiliates": "Check affiliates?", 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible hosts on the same target subdomain/domain?", 'maxnetblock': "If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'subnetlookup': "Look up all IPs on subnets which your target is a part of?", 'maxsubnet': "If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'maxage': "Maximum age (in days) of results to be considered valid. 0 = unlimited." } # Be sure to completely clear any class variables in setup() # or you run the risk of data persisting between scan runs. results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False # Clear / reset any other class member variables here # or you risk them persisting between threads. for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["IP_ADDRESS", "AFFILIATE_IPADDR", "INTERNET_NAME", "EMAILADDR", "CO_HOSTED_SITE", "PHONE_NUMBER", "BITCOIN_ADDRESS"] # What events this module produces def producedEvents(self): return ["LEAKSITE_URL", "DARKNET_MENTION_URL", "INTERNET_NAME", "DOMAIN_NAME", "EMAILADDR", "EMAILADDR_GENERIC"] def query(self, qry, qtype): retdata = list() headers = { "User-Agent": "SpiderFoot", "x-key": self.opts['api_key'], } payload = { "term": qry, "buckets": [], "lookuplevel": 0, "maxresults": 100, "timeout": 0, "datefrom": "", "dateto": "", "sort": 4, "media": 0, "terminate": [] } url = 'https://' + self.opts['base_url'] + '/' + qtype + '/search' res = self.sf.fetchUrl(url, postData=json.dumps(payload), headers=headers, timeout=self.opts['_fetchtimeout']) if res['content'] is None: self.info("No IntelligenceX info found for " + qry) return None if res['code'] == "402": self.info("IntelligenceX credits expired.") self.errorState = True return None try: ret = json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from IntelligenceX: {e}") self.errorState = True return None if ret.get('status', -1) == 0: # Craft API URL with the id to return results resulturl = f"{url}/result?id={ret['id']}" limit = 30 count = 0 status = 3 # status 3 = No results yet, keep trying. 0 = Success with results while status in [3, 0] and count < limit: if self.checkForStop(): return None res = self.sf.fetchUrl(resulturl, headers=headers) if res['content'] is None: self.info("No IntelligenceX info found for results from " + qry) return None if res['code'] == "402": self.info("IntelligenceX credits expired.") self.errorState = True return None try: ret = json.loads(res['content']) except Exception as e: self.error("Error processing JSON response from IntelligenceX: " + str(e)) return None status = ret['status'] count += 1 retdata.append(ret) # No more results left if status == 1: # print data in json format to manipulate as desired break time.sleep(1) return retdata # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if self.opts['api_key'] == "" or self.opts['base_url'] == "": self.error("You enabled sfp_intelx but did not set an API key and/or base URL!") self.errorState = True return self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName.startswith("AFFILIATE") and not self.opts['checkaffiliates']: return if eventName == 'CO_HOSTED_SITE' and not self.opts['checkcohosts']: return data = self.query(eventData, "intelligent") if data is None: return self.info("Found IntelligenceX leak data for " + eventData) agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage']) for info in data: for rec in info.get("records", dict()): try: last_seen = int(datetime.datetime.strptime(rec['added'].split(".")[0], '%Y-%m-%dT%H:%M:%S').strftime('%s')) * 1000 if self.opts['maxage'] > 0 and last_seen < agelimit: self.debug("Record found but too old, skipping.") continue val = None evt = None if "pastes" in rec['bucket']: evt = "LEAKSITE_URL" val = rec['keyvalues'][0]['value'] if rec['bucket'].startswith("darknet."): evt = "DARKNET_MENTION_URL" val = rec['name'] if not val or not evt: # Try generically extracting it if "systemid" not in rec: continue evt = "LEAKSITE_URL" val = "https://intelx.io/?did=" + rec['systemid'] except Exception as e: self.error(f"Error processing content from IntelX: {e}") continue # Notify other modules of what you've found e = SpiderFootEvent(evt, val, self.__name__, event) self.notifyListeners(e) if "public.intelx.io" in self.opts['base_url'] or eventName != "INTERNET_NAME": return data = self.query(eventData, "phonebook") if data is None: return self.info(f"Found IntelligenceX host and email data for {eventData}") for info in data: for rec in info.get("selectors", dict()): try: val = rec['selectorvalueh'] evt = None if rec['selectortype'] == 1: # Email evt = "EMAILADDR" if val.split("@")[0] in self.opts['_genericusers'].split(","): evt = "EMAILADDR_GENERIC" if rec['selectortype'] == 2: # Domain evt = "INTERNET_NAME" if val == eventData: continue if rec['selectortype'] == 3: # URL evt = "LINKED_URL_INTERNAL" if not val or not evt: self.debug("Unexpected record, skipping.") continue except Exception as e: self.error(f"Error processing content from IntelX: {e}") continue # Notify other modules of what you've found e = SpiderFootEvent(evt, val, self.__name__, event) self.notifyListeners(e) if evt == "INTERNET_NAME" and self.sf.isDomain(val, self.opts['_internettlds']): e = SpiderFootEvent("DOMAIN_NAME", val, self.__name__, event) self.notifyListeners(e) # End of sfp_intelx class ================================================ FILE: modules/sfp_intfiles.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_intfiles # Purpose: From Spidered pages found, identifies files of potential interest. # # Author: Steve Micallef # # Created: 06/04/2014 # Copyright: (c) Steve Micallef 2014 # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_intfiles(SpiderFootPlugin): meta = { 'name': "Interesting File Finder", 'summary': "Identifies potential files of interest, e.g. office documents, zip files.", 'flags': [], 'useCases': ["Footprint", "Passive"], 'categories': ["Crawling and Scanning"] } # Default options opts = { 'fileexts': ["doc", "docx", "ppt", "pptx", "pdf", 'xls', 'xlsx', 'zip'] } # Option descriptions optdescs = { 'fileexts': "File extensions of files you consider interesting." } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.__dataSource__ = "Target Website" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["LINKED_URL_INTERNAL"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["INTERESTING_FILE"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: return self.results[eventData] = True for fileExt in self.opts['fileexts']: if "." + fileExt.lower() in eventData.lower(): evt = SpiderFootEvent("INTERESTING_FILE", eventData, self.__name__, event) self.notifyListeners(evt) # End of sfp_intfiles class ================================================ FILE: modules/sfp_ipapico.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_ipapico # Purpose: SpiderFoot plug-in to identify the Geo-location of IP addresses # identified by other modules using ipapi.co # # Author: Krishnasis Mandal # # Created: 02/02/2021 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_ipapico(SpiderFootPlugin): meta = { 'name': "ipapi.co", 'summary': "Queries ipapi.co to identify geolocation of IP Addresses using ipapi.co API", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Real World"], 'dataSource': { 'website': "https://ipapi.co/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://ipapi.co/api/" ], 'favIcon': "https://ipapi.co/static/images/favicon.b64f1de785e1.ico", 'logo': "https://ipapi.co/static/images/favicon.34f0ec468301.png", 'description': "Powerful & Simple REST API for IP Address Geolocation." "ipapi.co provides a REST API to find the location of an IP address.", } } # Default options opts = { } # Option descriptions optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return [ "IP_ADDRESS", "IPV6_ADDRESS" ] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return [ "GEOINFO", "RAW_RIR_DATA" ] def query(self, qry): queryString = f"https://ipapi.co/{qry}/json/" res = self.sf.fetchUrl(queryString, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) time.sleep(1.5) if res['content'] is None: self.info(f"No ipapi.co data found for {qry}") return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") # Don't look up stuff twice if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True data = self.query(eventData) if data is None: self.info("No results returned from ipapi.co") return if data.get('country'): location = ', '.join(filter(None, [data.get('city'), data.get('region'), data.get('region_code'), data.get('country_name'), data.get('country')])) evt = SpiderFootEvent('GEOINFO', location, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) # End of sfp_ipapico class ================================================ FILE: modules/sfp_ipapicom.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_ipapicom # Purpose: SpiderFoot plug-in to identify the Geo-location of IP addresses # identified by other modules using ipapi.com # # Author: Krishnasis Mandal # # Created: 29/01/2021 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_ipapicom(SpiderFootPlugin): meta = { 'name': "ipapi.com", 'summary': "Queries ipapi.com to identify geolocation of IP Addresses using ipapi.com API", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Real World"], 'dataSource': { 'website': "https://ipapi.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://ipapi.com/documentation" ], 'apiKeyInstructions': [ "Visit https://ipapi.com/", "Register a free account", "Browse to https://ipapi.com/dashboard", "Your API Key will be listed under Your API Access Key", ], 'favIcon': "https://ipapi.com/site_images/ipapi_shortcut_icon.ico", 'logo': "https://ipapi.com/site_images/ipapi_icon.png", 'description': "ipapi provides an easy-to-use API interface allowing customers " "to look various pieces of information IPv4 and IPv6 addresses are associated with. " "For each IP address processed, the API returns more than 45 unique data points, " "such as location data, connection data, ISP information, time zone, currency and security assessment data.", } } # Default options opts = { 'api_key': '', } # Option descriptions optdescs = { 'api_key': "ipapi.com API Key.", } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return [ "IP_ADDRESS", "IPV6_ADDRESS" ] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return [ "GEOINFO", "RAW_RIR_DATA" ] def query(self, qry): queryString = f"http://api.ipapi.com/api/{qry}?access_key={self.opts['api_key']}" res = self.sf.fetchUrl(queryString, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) time.sleep(1.5) if res['code'] == "429": self.error("You are being rate-limited by IP-API.com.") self.errorState = True return None if res['content'] is None: self.info(f"No ipapi.com data found for {qry}") return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.errorState: return if self.opts['api_key'] == "": self.error("You enabled sfp_ipapicom but did not set an API key!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True data = self.query(eventData) if not data: return if data.get('country_name'): location = ', '.join(filter(None, [data.get('city'), data.get('region_name'), data.get('region_code'), data.get('country_name'), data.get('country_code')])) evt = SpiderFootEvent('GEOINFO', location, self.__name__, event) self.notifyListeners(evt) if data.get('latitude') and data.get('longitude'): evt = SpiderFootEvent("PHYSICAL_COORDINATES", f"{data.get('latitude')}, {data.get('longitude')}", self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) # End of sfp_ipapicom class ================================================ FILE: modules/sfp_ipinfo.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_ipinfo # Purpose: SpiderFoot plug-in to identify the Geo-location of IP addresses # identified by other modules using ipinfo.io. # # Author: Steve Micallef # # Created: 17/06/2017 # Copyright: (c) Steve Micallef 2017 # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_ipinfo(SpiderFootPlugin): meta = { 'name': "IPInfo.io", 'summary': "Identifies the physical location of IP addresses identified using ipinfo.io.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Real World"], 'dataSource': { 'website': "https://ipinfo.io", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://ipinfo.io/developers" ], 'apiKeyInstructions': [ "Visit https://ipinfo.io/", "Sign up for a free account", "Navigate to https://ipinfo.io/account", "The API key is listed above 'is your access token'" ], 'favIcon': "https://ipinfo.io/static/favicon-96x96.png?v3", 'logo': "https://ipinfo.io/static/deviceicons/android-icon-96x96.png", 'description': "The Trusted Source for IP Address Data.\n" "With IPinfo, you can pinpoint your users’ locations, customize their experiences, " "prevent fraud, ensure compliance, and so much more.", } } # Default options opts = { "api_key": "" } optdescs = { "api_key": "Ipinfo.io access token." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['IP_ADDRESS', 'IPV6_ADDRESS'] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["GEOINFO"] # https://ipinfo.io/developers def queryIP(self, ip): headers = { 'Authorization': "Bearer " + self.opts['api_key'] } res = self.sf.fetchUrl("https://ipinfo.io/" + ip + "/json", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], headers=headers) if res['code'] == "429": self.error("You are being rate-limited by ipinfo.io.") self.errorState = True return None if res['content'] is None: self.info("No GeoIP info found for " + ip) return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "": self.error("You enabled sfp_ipinfo but did not set an API key!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True data = self.queryIP(eventData) if data is None: return if 'country' not in data: return location = ', '.join([_f for _f in [data.get('city'), data.get('region'), data.get('country')] if _f]) self.info("Found GeoIP for " + eventData + ": " + location) evt = SpiderFootEvent("GEOINFO", location, self.__name__, event) self.notifyListeners(evt) # End of sfp_ipinfo class ================================================ FILE: modules/sfp_ipqualityscore.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_ipqualityscore # Purpose: Spiderfoot module to check whether a target is malicious # using IPQualityScore API # # Author: Krishnasis Mandal # # Created: 2020-10-07 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_ipqualityscore(SpiderFootPlugin): meta = { "name": "IPQualityScore", "summary": "Determine if target is malicious using IPQualityScore API", 'flags': ["apikey"], "useCases": ["Investigate", "Passive"], "categories": ["Reputation Systems"], "dataSource": { "website": "https://www.ipqualityscore.com/", "model": "FREE_AUTH_LIMITED", "references": [ "https://www.ipqualityscore.com/documentation/overview" ], "apiKeyInstructions": [ "Visit https://www.ipqualityscore.com/", "Click on 'Plans'", "Register a free account", "Visit https://www.ipqualityscore.com/user/settings", "Your API key will be listed under 'API Key'" ], "favIcon": "https://www.ipqualityscore.com/templates/img/icons/fav/favicon-32x32.png", "logo": "https://www.ipqualityscore.com/templates/img/logo.png", "description": "IPQualityScore's suite of fraud prevention tools automate quality control " "to prevent bots, fake accounts, fraudsters, suspicious transactions, " "& malicious users without interrupting the user experience.", }, } opts = { "api_key": "", "abuse_score_threshold": 85, "strictness": 0 } optdescs = { "api_key": "IPQualityScore API Key", "abuse_score_threshold": "Minimum abuse score for target to be considered malicious (0 - 100)", "strictness": "Depth of the reputation checks to be performed on the target (0 - 2)" } errorState = False def setup(self, sfc, userOpts=None): if userOpts is None: userOpts = {} self.sf = sfc self.results = self.tempStorage() self.opts.update(userOpts) def watchedEvents(self): return [ "DOMAIN_NAME", "EMAILADDR", "IP_ADDRESS", "PHONE_NUMBER", ] def producedEvents(self): return [ "EMAILADDR_DISPOSABLE", "EMAILADDR_COMPROMISED", "GEOINFO", "MALICIOUS_PHONE_NUMBER", "MALICIOUS_EMAILADDR", "MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME", "PHONE_NUMBER_TYPE", "RAW_RIR_DATA" ] def handle_error_response(self, qry, res): try: error_info = json.loads(res["content"]) except Exception: error_info = None if error_info: error_message = error_info.get("message") else: error_message = None if error_message: error_str = f", message {error_message}" else: error_str = "" self.error(f"Failed to get results for {qry}, code {res['code']}{error_str}") def query(self, qry, eventName): queryString = "" if eventName == "PHONE_NUMBER": queryString = f"https://ipqualityscore.com/api/json/phone/{self.opts['api_key']}/{qry}?strictness={self.opts['strictness']}" elif eventName == "EMAILADDR": queryString = f"https://ipqualityscore.com/api/json/email/{self.opts['api_key']}/{qry}?strictness={self.opts['strictness']}" elif eventName in ['IP_ADDRESS', 'DOMAIN_NAME']: queryString = f"https://ipqualityscore.com/api/json/ip/{self.opts['api_key']}/{qry}?strictness={self.opts['strictness']}" res = self.sf.fetchUrl( queryString, timeout=self.opts["_fetchtimeout"], useragent="SpiderFoot", ) if not res['content']: self.info(f"No IPQualityScore info found for {qry}") return None try: r = json.loads(res['content']) if res["code"] != "200" or not r.get("success"): self.handle_error_response(qry, res) return None return r except Exception as e: self.error(f"Error processing JSON response from IPQualityScore: {e}") return None def getGeoInfo(self, data): geoInfo = "" city = data.get('city') country = data.get('country') if not country: country = data.get('country_code') zipcode = data.get('zip_code') region = data.get('region') if city: geoInfo += city + ", " if region: geoInfo += region + ", " if country: geoInfo += country + " " if zipcode: geoInfo += zipcode return geoInfo def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.error( f"You enabled {self.__class__.__name__} but did not set an API Key!" ) self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData} as already mapped.") return self.results[eventData] = True data = self.query(eventData, eventName) if not data: return fraudScore = data.get('fraud_score') recentAbuse = data.get('recent_abuse') botStatus = data.get('bot_status') malicious = False maliciousDesc = "" if fraudScore >= self.opts['abuse_score_threshold'] or recentAbuse or botStatus: evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) malicious = True maliciousDesc = f"IPQualityScore [{eventData}]\n" if eventName == "PHONE_NUMBER": if malicious: maliciousDesc += f" - FRAUD SCORE: {fraudScore}\n - ACTIVE: {data.get('active')}\n - RISKY: {data.get('risky')}\n - RECENT ABUSE: {recentAbuse}" evt = SpiderFootEvent("MALICIOUS_PHONE_NUMBER", maliciousDesc, self.__name__, event) self.notifyListeners(evt) phoneNumberType = data.get('line_type') if phoneNumberType: evt = SpiderFootEvent("PHONE_NUMBER_TYPE", phoneNumberType, self.__name__, event) self.notifyListeners(evt) geoInfo = self.getGeoInfo(data) if geoInfo: evt = SpiderFootEvent("GEOINFO", geoInfo, self.__name__, event) self.notifyListeners(evt) elif eventName == "EMAILADDR": if malicious: maliciousDesc += f" - FRAUD SCORE: {fraudScore}\n - HONEYPOT: {data.get('honeypot')}\n - SPAM TRAP SCORE: {data.get('spam_trap_score')}\n - RECENT ABUSE: {recentAbuse}" evt = SpiderFootEvent("MALICIOUS_EMAILADDR", maliciousDesc, self.__name__, event) self.notifyListeners(evt) if data.get('disposable'): evt = SpiderFootEvent("EMAILADDR_DISPOSABLE", eventData, self.__name__, event) self.notifyListeners(evt) if data.get('leaked'): evt = SpiderFootEvent("EMAILADDR_COMPROMISED", f"{eventData} [Unknown]", self.__name__, event) self.notifyListeners(evt) elif eventName in ['IP_ADDRESS', 'DOMAIN_NAME']: if malicious: maliciousDesc += f" - FRAUD SCORE: {fraudScore}\n - BOT STATUS: {botStatus}\n - RECENT ABUSE: {recentAbuse}\n - ABUSE VELOCITY: {data.get('abuse_velocity')}\n - VPN: {data.get('vpn')}\n - ACTIVE VPN: {data.get('active_vpn')}\n - TOR: {data.get('tor')}\n - ACTIVE TOR: {data.get('active_tor')}" if eventName == "IP_ADDRESS": evt = SpiderFootEvent("MALICIOUS_IPADDR", maliciousDesc, self.__name__, event) elif eventName == "DOMAIN_NAME": evt = SpiderFootEvent("MALICIOUS_INTERNET_NAME", maliciousDesc, self.__name__, event) self.notifyListeners(evt) geoInfo = self.getGeoInfo(data) if geoInfo: evt = SpiderFootEvent("GEOINFO", geoInfo, self.__name__, event) self.notifyListeners(evt) # End of sfp_ipqualityscore class ================================================ FILE: modules/sfp_ipregistry.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_ipregistry # Purpose: ipregistry database query module. # # Author: Leo Trubach # # Created: 2020-09-08 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import urllib.parse from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_ipregistry(SpiderFootPlugin): meta = { "name": "ipregistry", "summary": "Query the ipregistry.co database for reputation and geo-location.", 'flags': ["apikey"], "useCases": ["Passive", "Footprint", "Investigate"], "categories": ["Reputation Systems"], "dataSource": { "website": "https://ipregistry.co/", "model": "FREE_AUTH_LIMITED", "references": ["https://ipregistry.co/docs"], "apiKeyInstructions": [ "Visit https://dashboard.ipregistry.co/signup", "Register a free account", "Click on 'API Keys' in left navbar", "Click on 'Click to reveal API key' for existing Default key", ], "favIcon": "https://cdn.ipregistry.co/icons/favicon-32x32.png", "logo": "https://ipregistry.co/assets/ipregistry.svg", "description": "Ipregistry is a trusted and in-depth IP " "Geolocation and Threat detections source of information that can" "benefit publishers, ad networks, retailers, financial services, " "e-commerce stores and more.", }, } opts = { "api_key": "", } optdescs = { "api_key": "Ipregistry API Key.", } results = None errorState = False def setup(self, sfc, userOpts=None): self.sf = sfc self.results = self.tempStorage() if userOpts: self.opts.update(userOpts) def watchedEvents(self): return ["IP_ADDRESS", "IPV6_ADDRESS"] def producedEvents(self): return ["GEOINFO", "MALICIOUS_IPADDR", "PHYSICAL_COORDINATES", "RAW_RIR_DATA"] def query(self, qry): qs = urllib.parse.urlencode({"key": self.opts["api_key"]}) res = self.sf.fetchUrl( f"https://api.ipregistry.co/{qry}?{qs}", timeout=self.opts["_fetchtimeout"], useragent="SpiderFoot", ) if res["content"] is None: self.info(f"No {self.meta['name']} info found for {qry}") return None try: return json.loads(res["content"]) except Exception as e: self.error( f"Error processing JSON response from {self.meta['name']}: {e}" ) return None def emit(self, etype, data, pevent): evt = SpiderFootEvent(etype, data, self.__name__, pevent) self.notifyListeners(evt) return evt def generate_location_events(self, location, pevent): if not isinstance(location, dict): return physical_location = None country = location.get("country") if isinstance(country, dict): country_name = country.get("name") else: country_name = None region = location.get("region") if isinstance(region, dict): region_name = region.get("name") else: region_name = None latitude = location.get("latitude") longitude = location.get("longitude") if latitude and longitude: physical_location = f"{latitude}, {longitude}" geo_info = ", ".join( [ _f for _f in [ location.get("city"), region_name, location.get("postal"), country_name, ] if _f ] ) if geo_info: self.emit("GEOINFO", geo_info, pevent) if physical_location: self.emit("PHYSICAL_COORDINATES", physical_location, pevent) def generate_security_events(self, security, pevent): if not isinstance(security, dict): return malicious = any( security.get(k) for k in ("is_abuser", "is_attacker", "is_threat") ) if malicious: self.emit("MALICIOUS_IPADDR", f"ipregistry [{pevent.data}]", pevent) def generate_events(self, data, pevent): if not isinstance(data, dict): return self.generate_location_events(data.get("location"), pevent) self.generate_security_events(data.get("security"), pevent) def handleEvent(self, event): if self.errorState: return self.debug(f"Received event, {event.eventType}, from {event.module}") if self.opts["api_key"] == "": self.error(f"You enabled {self.__class__.__name__} but did not set an API key!") self.errorState = True return if event.data in self.results: self.debug(f"Skipping {event.data}, already checked.") return self.results[event.data] = True if event.eventType in ("IP_ADDRESS", "IPV6_ADDRESS"): data = self.query(event.data) self.generate_events(data, event) self.emit("RAW_RIR_DATA", json.dumps(data), event) ================================================ FILE: modules/sfp_ipstack.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_ipstack # Purpose: SpiderFoot plug-in to identify the Geo-location of IP addresses # identified by other modules using ipstack.com. # # Author: Steve Micallef # # Created: 20/08/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_ipstack(SpiderFootPlugin): meta = { 'name': "ipstack", 'summary': "Identifies the physical location of IP addresses identified using ipstack.com.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Real World"], 'dataSource': { 'website': "https://ipstack.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://ipstack.com/documentation", "https://ipstack.com/faq" ], 'apiKeyInstructions': [ "Visit https://ipstack.com/product", "Click on 'Get Free API Key'", "Click on 'Dashboard'", "The API key is listed under 'Your API Access Key'" ], 'favIcon': "https://ipstack.com/ipstack_images/ipstack_logo.svg", 'logo': "https://ipstack.com/ipstack_images/ipstack_logo.svg", 'description': "Locate and identify website visitors by IP address.\n" "ipstack offers one of the leading IP to geolocation APIS " "and global IP database services worldwide.", } } # Default options opts = { "api_key": "" } optdescs = { "api_key": "Ipstack.com API key." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['IP_ADDRESS'] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["GEOINFO"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.errorState: return if self.opts['api_key'] == "": self.error("You enabled sfp_ipstack but did not set an API key!") self.errorState = True return # Don't look up stuff twice if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True res = self.sf.fetchUrl("http://api.ipstack.com/" + eventData + "?access_key=" + self.opts['api_key'], timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if res['content'] is None: self.info("No GeoIP info found for " + eventData) try: hostip = json.loads(res['content']) if 'success' in hostip and hostip['success'] is False: self.error("Invalid ipstack.com API key or usage limits exceeded.") self.errorState = True return except Exception as e: self.debug(f"Error processing JSON response: {e}") return geoinfo = hostip.get('country_name') if geoinfo: self.info(f"Found GeoIP for {eventData}: {geoinfo}") evt = SpiderFootEvent("GEOINFO", geoinfo, self.__name__, event) self.notifyListeners(evt) # End of sfp_ipstack class ================================================ FILE: modules/sfp_isc.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_isc # Purpose: Check if an IP address is malicious according to SANS ISC. # # Author: steve@binarypool.com # # Created: 14/12/2013 # Copyright: (c) Steve Micallef, 2013 # Licence: MIT # ------------------------------------------------------------------------------- import re from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_isc(SpiderFootPlugin): meta = { 'name': "Internet Storm Center", 'summary': "Check if an IP address is malicious according to SANS ISC.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://isc.sans.edu", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://isc.sans.edu/api/", "https://isc.sans.edu/howto.html", "https://isc.sans.edu/honeypot.html", "https://isc.sans.edu/glossary.html", "https://isc.sans.edu/fightback.html" ], 'favIcon': "https://isc.sans.edu/iscfavicon.ico", 'logo': "https://isc.sans.edu/images/logos/isc/large.png", 'description': "The ISC provides a free analysis and warning service to thousands of Internet users " "and organizations, and is actively working with Internet Service Providers to " "fight back against the most malicious attackers.\n" "Thousands of sensors that work with most firewalls, intrusion detection systems, " "home broadband devices, and nearly all operating systems are constantly collecting information about " "unwanted traffic arriving from the Internet. " "These devices feed the DShield database where human volunteers as well as machines pour through " "the data looking for abnormal trends and behavior. " "The resulting analysis is posted to the ISC's main web page where it can be automatically retrieved " "by simple scripts or can be viewed in near real time by any Internet user.", } } opts = { 'checkaffiliates': True } optdescs = { 'checkaffiliates': "Apply checks to affiliates?" } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "IP_ADDRESS", "IPV6_ADDRESS", "AFFILIATE_IPADDR", "AFFILIATE_IPV6_ADDRESS", ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", ] def query(self, ip): if not ip: return None res = self.sf.fetchUrl( f"https://isc.sans.edu/api/ip/{ip}", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from ISC.") self.errorState = True return None if res['content'] is None: self.error("Received no content from ISC") self.errorState = True return None return res['content'] def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']: malicious_type = 'MALICIOUS_IPADDR' blacklist_type = 'BLACKLISTED_IPADDR' elif eventName in ['AFFILIATE_IPADDR', 'AFFILIATE_IPV6_ADDRESS']: if not self.opts.get('checkaffiliates', False): return malicious_type = 'MALICIOUS_AFFILIATE_IPADDR' blacklist_type = 'BLACKLISTED_AFFILIATE_IPADDR' else: self.debug(f"Unexpected event type {eventName}, skipping") return data = self.query(eventData) if not data: return attacks = re.findall(r"([0-9]+)", data) if not attacks: return url = f"https://isc.sans.edu/api/ip/{eventData}" text = f"Internet Storm Center [{eventData}]\n{url}" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_isc class ================================================ FILE: modules/sfp_jsonwhoiscom.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_jsonwhoiscom # Purpose: Search JsonWHOIS.com for WHOIS records associated with a domain. # # Authors: # # Created: 2020-06-20 # Copyright: (c) bcoles 2020 # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_jsonwhoiscom(SpiderFootPlugin): meta = { 'name': "JsonWHOIS.com", 'summary': "Search JsonWHOIS.com for WHOIS records associated with a domain.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://jsonwhois.com", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://jsonwhois.com/docs" ], 'apiKeyInstructions': [ "Visit https://jsonwhois.com", "Sign up for a free account", "Navigate to https://jsonwhois.com/dashboard", "The API key is listed under 'Api Key'" ], 'favIcon': "https://jsonwhois.com/assets/fav.ico", 'logo': "https://jsonwhois.com/assets/fav.ico", 'description': "Get access to accurate Whois records for generic and country TLDs. " "Around 1000 gTLDs include .com, .org, .net, .us, .biz, .info, .mobi, .pro, .asia and many other new ones.\n" "Raw and parsed Whois data are both accessible for downloads in the form of " "MYSQL or MYSQL database dumps and Comma Separated Values (.CSV) files.", } } # Default options opts = { "api_key": "", "delay": 1, } # Option descriptions optdescs = { "api_key": "JsonWHOIS.com API key.", "delay": "Delay between requests, in seconds.", } results = None errorState = False # Initialize module and module options def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in userOpts.keys(): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME", "AFFILIATE_DOMAIN_NAME"] # What events this module produces def producedEvents(self): return ["RAW_RIR_DATA", "DOMAIN_REGISTRAR", "DOMAIN_WHOIS", "PROVIDER_DNS", "EMAILADDR", "EMAILADDR_GENERIC", "PHONE_NUMBER", "PHYSICAL_ADDRESS", "AFFILIATE_DOMAIN_UNREGISTERED"] # Query domain # https://jsonwhois.com/docs def queryDomain(self, qry): params = { 'domain': qry.encode('raw_unicode_escape').decode("ascii", errors='replace') } headers = { "Accept": "application/json", "Authorization": "Token token=" + self.opts["api_key"] } res = self.sf.fetchUrl( f"https://jsonwhois.com/api/v1/whois?{urllib.parse.urlencode(params)}", headers=headers, timeout=15, useragent=self.opts['_useragent'] ) time.sleep(self.opts['delay']) return self.parseApiResponse(res) # Parse API response def parseApiResponse(self, res: dict): if not res: self.error("No response from JsonWHOIS.com.") return None if res['code'] == '404': self.debug("No results for query") return None # Sometimes JsonWHOIS.com returns HTTP 500 errors rather than 404 if res['code'] == '500' and res['content'] == '{"error":"Call failed"}': self.debug("No results for query") return None if res['code'] == "401": self.error("Invalid JsonWHOIS.com API key.") self.errorState = True return None if res['code'] == '429': self.error("You are being rate-limited by JsonWHOIS.com") self.errorState = True return None if res['code'] == '503': self.error("JsonWHOIS.com service unavailable") self.errorState = True return None # Catch all other non-200 status codes, and presume something went wrong if res['code'] != '200': self.error("Failed to retrieve content from JsonWHOIS.com") self.errorState = True return None if res['content'] is None: return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if eventData in self.results: return if self.opts['api_key'] == "": self.error("You enabled sfp_jsonwhoiscom but did not set an API key!") self.errorState = True return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") res = self.queryDomain(eventData) if res is None: self.debug(f"No information found for domain {eventData}") return evt = SpiderFootEvent('RAW_RIR_DATA', str(res), self.__name__, event) self.notifyListeners(evt) dns_providers = list() nameservers = res.get('nameservers') if nameservers: for nameserver in nameservers: if nameserver: nameserver_name = nameserver.get('name') if nameserver_name: dns_providers.append(nameserver_name) contacts = list() registrant_contacts = res.get('registrant_contacts') if registrant_contacts: for contact in registrant_contacts: contacts.append(contact) admin_contacts = res.get('admin_contacts') if admin_contacts: for contact in admin_contacts: contacts.append(contact) technical_contacts = res.get('technical_contacts') if technical_contacts: for contact in technical_contacts: contacts.append(contact) emails = list() names = list() phones = list() locations = list() for contact in contacts: email = contact.get('email') if email: if SpiderFootHelpers.validEmail(email): emails.append(email) name = contact.get("name") if name: names.append(name) phone = contact.get('phone') if phone: phone = phone.replace(" ", "").replace("-", "").replace("(", "").replace(")", "").replace(".", "") phones.append(phone) country = SpiderFootHelpers.countryNameFromCountryCode(contact.get('country_code')) location = ', '.join([_f for _f in [contact.get('address'), contact.get('city'), contact.get('state'), contact.get('zip'), country] if _f]) if location: locations.append(location) for email in set(emails): mail_domain = email.lower().split('@')[1] if self.getTarget().matches(mail_domain, includeChildren=True): if email.split("@")[0] in self.opts['_genericusers'].split(","): evttype = "EMAILADDR_GENERIC" else: evttype = "EMAILADDR" evt = SpiderFootEvent(evttype, email, self.__name__, event) self.notifyListeners(evt) else: evt = SpiderFootEvent("AFFILIATE_EMAILADDR", email, self.__name__, event) self.notifyListeners(evt) if eventName in ["DOMAIN_NAME"]: raw = res.get('raw') if raw: evt = SpiderFootEvent("DOMAIN_WHOIS", raw, self.__name__, event) self.notifyListeners(evt) registrar = res.get("registrar") if registrar: registrar_name = registrar.get("name") if registrar_name: evt = SpiderFootEvent("DOMAIN_REGISTRAR", registrar_name, self.__name__, event) self.notifyListeners(evt) for dns_provider in set(dns_providers): evt = SpiderFootEvent("PROVIDER_DNS", dns_provider, self.__name__, event) self.notifyListeners(evt) for name in set(names): evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name {name}", self.__name__, event) self.notifyListeners(evt) for phone in set(phones): evt = SpiderFootEvent("PHONE_NUMBER", phone, self.__name__, event) self.notifyListeners(evt) for location in set(locations): evt = SpiderFootEvent("PHYSICAL_ADDRESS", location, self.__name__, event) self.notifyListeners(evt) if eventName in ["AFFILIATE_DOMAIN_NAME"]: raw = res.get('raw') if raw: evt = SpiderFootEvent("AFFILIATE_DOMAIN_WHOIS", raw, self.__name__, event) self.notifyListeners(evt) available = res.get('available?') if available: evt = SpiderFootEvent("AFFILIATE_DOMAIN_UNREGISTERED", eventData, self.__name__, event) self.notifyListeners(evt) # End of sfp_jsonwhoiscom class ================================================ FILE: modules/sfp_junkfiles.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_junkfiles # Purpose: From Spidering, identifies backup and temporary files. # # Author: Steve Micallef # # Created: 23/08/2014 # Copyright: (c) Steve Micallef 2014 # Licence: MIT # ------------------------------------------------------------------------------- import random from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_junkfiles(SpiderFootPlugin): meta = { 'name': "Junk File Finder", 'summary': "Looks for old/temporary and other similar files.", 'flags': ["slow", "errorprone", "invasive"], 'useCases': ["Footprint"], 'categories': ["Crawling and Scanning"] } # Default options opts = { 'fileexts': ['tmp', 'bak', 'old'], 'urlextstry': ['asp', 'php', 'jsp', ], 'files': ["old", "passwd", ".htaccess", ".htpasswd", "Thumbs.db", "backup"], 'dirs': ['zip', 'tar.gz', 'tgz', 'tar'] } # Option descriptions optdescs = { 'fileexts': "File extensions to try.", 'urlextstry': "Try those extensions against URLs with these extensions.", 'files': "Try to fetch each of these files from the directory of the URL.", 'dirs': "Try to fetch the containing folder with these extensions." } results = None hosts = None skiphosts = None bases = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.hosts = self.tempStorage() self.skiphosts = self.tempStorage() self.bases = self.tempStorage() self.__dataSource__ = "Target Website" for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["LINKED_URL_INTERNAL"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["JUNK_FILE"] # Test how trustworthy a result is def checkValidity(self, junkUrl): # Try and fetch an obviously missing version of the junk file fetch = junkUrl + str(random.SystemRandom().randint(0, 99999999)) res = self.sf.fetchUrl(fetch, headOnly=True, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], verify=False) if res['code'] != "404": host = SpiderFootHelpers.urlBaseUrl(junkUrl) self.skiphosts[host] = True return False return True # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: return self.results[eventData] = True host = SpiderFootHelpers.urlBaseUrl(eventData) if host in self.skiphosts: self.debug("Skipping " + host + " because it doesn't return 404s.") return # http://www/blah/abc.php -> try http://www/blah/abc.php.[fileexts] for ext in self.opts['urlextstry']: if host in self.skiphosts: self.debug("Skipping " + host + " because it doesn't return 404s.") return if "." + ext + "?" in eventData or "." + ext + "#" in eventData or \ eventData.endswith("." + ext): bits = eventData.split("?") for x in self.opts['fileexts']: if self.checkForStop(): return self.debug("Trying " + x + " against " + eventData) fetch = bits[0] + "." + x if fetch in self.results: self.debug("Skipping, already fetched.") continue self.results[fetch] = True res = self.sf.fetchUrl(fetch, headOnly=True, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], sizeLimit=10000000, verify=False) if res['realurl'] != fetch: self.debug("Skipping because " + res['realurl'] + " isn't the fetched URL of " + fetch) continue if res['code'] == "200": if not self.checkValidity(fetch): continue evt = SpiderFootEvent("JUNK_FILE", fetch, self.__name__, event) self.notifyListeners(evt) base = SpiderFootHelpers.urlBaseDir(eventData) if not base or base in self.bases: return self.bases[base] = True # http://www/blah/abc.html -> try http://www/blah/[files] for f in self.opts['files']: if self.checkForStop(): return if host in self.skiphosts: self.debug("Skipping " + host + " because it doesn't return 404s.") return self.debug("Trying " + f + " against " + eventData) fetch = base + f if fetch in self.results: self.debug("Skipping, already fetched.") continue self.results[fetch] = True res = self.sf.fetchUrl(fetch, headOnly=True, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], verify=False) if res['realurl'] != fetch: self.debug("Skipping because " + res['realurl'] + " isn't the fetched URL of " + fetch) continue if res['code'] == "200": if not self.checkValidity(fetch): continue evt = SpiderFootEvent("JUNK_FILE", fetch, self.__name__, event) self.notifyListeners(evt) # don't do anything with the root directory of a site self.debug(f"Base: {base}, event: {eventData}") if base in [eventData, eventData + "/"]: return # http://www/blah/abc.html -> try http://www/blah.[dirs] for dirfile in self.opts['dirs']: if self.checkForStop(): return if host in self.skiphosts: self.debug("Skipping " + host + " because it doesn't return 404s.") return if base.count('/') == 3: self.debug("Skipping base url.") continue self.debug("Trying " + dirfile + " against " + eventData) fetch = base[0:len(base) - 1] + "." + dirfile if fetch in self.results: self.debug("Skipping, already fetched.") continue self.results[fetch] = True res = self.sf.fetchUrl(fetch, headOnly=True, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], verify=False) if res['realurl'] != fetch: self.debug("Skipping because " + res['realurl'] + " isn't the fetched URL of " + fetch) continue if res['code'] == "200": if not self.checkValidity(fetch): continue evt = SpiderFootEvent("JUNK_FILE", fetch, self.__name__, event) self.notifyListeners(evt) # End of sfp_junkfiles class ================================================ FILE: modules/sfp_keybase.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_keybase # Purpose: Spiderfoot plugin to query KeyBase API # to gather additional information about domain names and identified # usernames. # # Author: Krishnasis Mandal # # Created: 22/05/2020 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import re import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_keybase(SpiderFootPlugin): meta = { 'name': "Keybase", 'summary': "Obtain additional information about domain names and identified usernames.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Public Registries"], 'dataSource': { 'website': "https://keybase.io/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://keybase.io/docs/api/1.0/call/user/lookup", ], 'favIcon': "https://keybase.io/images/icons/icon-keybase-logo-48.png", 'logo': "https://keybase.io/images/icons/icon-keybase-logo-48.png", 'description': "Keybase is a key directory that maps social media identities to encryption keys " "in a publicly auditable manner.", } } opts = { } optdescs = { } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["USERNAME", "LINKED_URL_EXTERNAL", "DOMAIN_NAME"] def producedEvents(self): return [ "RAW_RIR_DATA", "SOCIAL_MEDIA", "USERNAME", "GEOINFO", "BITCOIN_ADDRESS", "PGP_KEY" ] def query(self, selector: str, qry: str) -> str: """Search Keybase for a domain name or username. Args: selector (str): query type ("usernames" | "domain") qry (str): username Returns: str: Search results as JSON string """ if not selector: return None if not qry: return None params = { selector: qry.encode('raw_unicode_escape').decode("ascii", errors='replace') } headers = { 'Accept': "application/json" } res = self.sf.fetchUrl( 'https://keybase.io/_/api/1.0/user/lookup.json?' + urllib.parse.urlencode(params), headers=headers, timeout=15, useragent=self.opts['_useragent'] ) # In this case, it will always be 200 if keybase is queried # The actual response codes are stored in status tag of the response if res['code'] != '200': self.error(f"Unexpected reply from Keybase: {res['code']}") return None try: content = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None status = content.get('status') if not status: return None code = status.get('code') if code != 0: self.error(f"Unexpected JSON response code reply from Keybase: {code}") return None them = content.get('them') if not isinstance(them, list): return None return them def handleEvent(self, event) -> None: eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True # Extract username if a Keybase link is received if eventName == "LINKED_URL_EXTERNAL": linkRegex = r"^https?://keybase.io\/[A-Za-z0-9\-_\.]+" links = re.findall(linkRegex, eventData) if len(links) == 0: self.debug(f"Skipping URL {eventData}, as not a keybase link") return userName = links[0].split("/")[3] data = self.query('usernames', userName) elif eventName == "USERNAME": data = self.query('usernames', eventData) elif eventName == "DOMAIN_NAME": data = self.query('domain', eventData) else: return if not data: self.debug(f"No data found for {eventName}: {eventData}") return for user in data: if not user: continue # Basic information about the username basics = user.get('basics') if not basics: continue username = basics.get('username') if not username: continue # Failsafe to prevent reporting any wrongly received data if eventName == "USERNAME": if eventData.lower() != username.lower(): self.error("Username does not match received response, skipping") continue # For newly discovereed usernames, create a username event to be used as a source event if eventName in ['LINKED_URL_EXTERNAL', 'DOMAIN_NAME']: if username in self.results: self.debug(f"Skipping {userName}, already checked.") continue source_event = SpiderFootEvent("USERNAME", username, self.__name__, event) self.notifyListeners(source_event) self.results[username] = True else: source_event = event evt = SpiderFootEvent("RAW_RIR_DATA", str(user), self.__name__, source_event) self.notifyListeners(evt) # Profile information about the username profile = user.get('profile') if profile: # Get and report full name of user fullName = profile.get('full_name') if fullName: evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {fullName}", self.__name__, source_event) self.notifyListeners(evt) # Get and report location of user location = profile.get('location') if location: evt = SpiderFootEvent("GEOINFO", location, self.__name__, source_event) self.notifyListeners(evt) # Extract social media information proofsSummary = user.get('proofs_summary') if proofsSummary: socialMediaData = proofsSummary.get('all') if socialMediaData: for socialMedia in socialMediaData: socialMediaSite = socialMedia.get('proof_type') socialMediaURL = socialMedia.get('service_url') if socialMediaSite and socialMediaURL: socialMedia = socialMediaSite + ": " + "" + socialMediaURL + "" evt = SpiderFootEvent("SOCIAL_MEDIA", socialMedia, self.__name__, source_event) self.notifyListeners(evt) # Get cryptocurrency addresses cryptoAddresses = user.get('cryptocurrency_addresses') # Extract and report bitcoin addresses if any if cryptoAddresses: bitcoinAddresses = cryptoAddresses.get('bitcoin') if bitcoinAddresses: for bitcoinAddress in bitcoinAddresses: btcAddress = bitcoinAddress.get('address') if not btcAddress: continue evt = SpiderFootEvent("BITCOIN_ADDRESS", btcAddress, self.__name__, source_event) self.notifyListeners(evt) # Extract PGP Keys pgpRegex = r"-----BEGIN\s*PGP\s*(?:PUBLIC?)\s*KEY\s*BLOCK-----(.*?)-----END\s*PGP\s*(?:PUBLIC?)\s*KEY\s*BLOCK-----" pgpKeys = re.findall(pgpRegex, str(user)) for pgpKey in pgpKeys: if len(pgpKey) < 300: self.debug(f"PGP key size ({len(pgpKey)} bytes) is likely invalid (smaller than 300 bytes), skipping.") continue # Remove unescaped \n literals pgpKey = pgpKey.replace("\\n", "\n") # Avoid reporting of duplicate keys pgpKeyHash = self.sf.hashstring(pgpKey) if pgpKeyHash in self.results: continue self.results[pgpKeyHash] = True evt = SpiderFootEvent("PGP_KEY", pgpKey, self.__name__, source_event) self.notifyListeners(evt) # End of sfp_keybase class ================================================ FILE: modules/sfp_koodous.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_koodous # Purpose: Query Koodous for mobile apps. # # Author: # # Created: 2020-09-19 # Copyright: (c) bcoles 2019 # Licence: MIT # ------------------------------------------------------------------------------- import json import re import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_koodous(SpiderFootPlugin): meta = { 'name': "Koodous", 'summary': "Search Koodous for mobile apps.", 'flags': ["apikey"], 'useCases': ["Investigate", "Footprint", "Passive"], 'categories': ["Search Engines"], 'dataSource': { 'model': "FREE_AUTH_LIMITED", "apiKeyInstructions": [ "Visit https://koodous.com/apks", "Register a free account", "Visit https://koodous.com/settings/developers and use the authentication token provided", ], 'references': [ "https://docs.koodous.com/api/apks.html", "https://docs.koodous.com/apks.html#apks-search-system" ], 'website': "https://koodous.com/apks/", 'favIcon': "https://koodous.com/favicon.ico", 'logo': "https://koodous.com/assets/img/koodous-logo.png", "description": "The Collaborative Platform for Android Malware Analysts." } } opts = { "api_key": "", 'max_pages': 10, } optdescs = { "api_key": "Koodous API key.", 'max_pages': "Maximum number of pages of results to fetch.", } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.errorState = False self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ 'DOMAIN_NAME' ] def producedEvents(self): return [ 'APPSTORE_ENTRY', 'RAW_RIR_DATA' ] def queryPackageName(self, qry, cursor=''): package_name = qry.encode('raw_unicode_escape').decode("ascii", errors='replace') params = urllib.parse.urlencode({ 'cursor': cursor, 'search': f"package:{package_name}.*" }) res = self.sf.fetchUrl( f"https://developer.koodous.com/apks/?{params}", headers={"Authorization": f"Token {self.opts['api_key']}"}, useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout'] ) # 100 requests per minute time.sleep(1) return self.parseApiResponse(res) def parseApiResponse(self, res: dict): if not res: self.error("No response from Koodous.") return None if res['code'] == '404': self.debug("No results from Koodous.") return None if res['code'] == "401": self.error("Invalid Koodous API key.") self.errorState = True return None if res['code'] == '429': self.error("You are being rate-limited by Koodous.") self.errorState = True return None if res['code'] == '500' or res['code'] == '502' or res['code'] == '503': self.error("Koodous service is unavailable") self.errorState = True return None # Catch all non-200 status codes, and presume something went wrong if res['code'] != '200': self.error(f"Unexpected reply from Koodous: {res['code']}") self.errorState = True return None if res['content'] is None: return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from Koodous: {e}") return None def handleEvent(self, event): if self.errorState: return self.debug(f"Received event, {event.eventType}, from {event.module}") if self.opts["api_key"] == "": self.error( f"You enabled {self.__class__.__name__} but did not set an API key!" ) self.errorState = True return eventData = event.data if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True # Reverse domain name to create potential package name domain_reversed = '.'.join(list(reversed(eventData.lower().split('.')))) max_pages = int(self.opts['max_pages']) page = 1 cursor = '' while page <= max_pages: found = False if self.checkForStop(): return data = self.queryPackageName(domain_reversed, cursor) page += 1 if not data: self.errorState = True return results = data.get('results') for result in results: package_name = result.get('package_name') if not package_name: continue app = result.get('app') # results can have a null app name, but it is probably a duplicate if not app: continue # TODO: compare company name with target # company = result.get('company') version = result.get('version') if version: app_full_name = f"{app} {version} ({package_name})" else: app_full_name = f"{app} ({package_name})" if ( domain_reversed != package_name.lower() and not package_name.lower().startswith(f"{domain_reversed}.") and not package_name.lower().endswith(f".{domain_reversed}") and f".{domain_reversed}." not in package_name.lower() ): self.debug(f"App {app_full_name} does not match {domain_reversed}, skipping") continue sha256 = result.get('sha256') if not sha256: continue app_data = f"{app_full_name}\nhttps://koodous.com/apks/{sha256}" evt = SpiderFootEvent('APPSTORE_ENTRY', app_data, self.__name__, event) self.notifyListeners(evt) found = True if found: evt = SpiderFootEvent('RAW_RIR_DATA', json.dumps(data), self.__name__, event) self.notifyListeners(evt) if not data.get('next'): break next_cursor = re.findall('cursor=(.+?)&', data.get('next')) if not next_cursor: break cursor = urllib.parse.unquote(next_cursor[0]) # End of sfp_koodous class ================================================ FILE: modules/sfp_leakix.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_leakix # Purpose: Search LeakIX for host data leaks, open ports, software and geoip. # # Authors: # # Created: 2020-06-16 # Copyright: (c) bcoles 2020 # Licence: MIT # ------------------------------------------------------------------------------- import json import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_leakix(SpiderFootPlugin): meta = { 'name': "LeakIX", 'summary': "Search LeakIX for host data leaks, open ports, software and geoip.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Leaks, Dumps and Breaches"], 'dataSource': { 'website': "https://leakix.net/", 'model': "FREE_AUTH_UNLIMITED", 'references': [ "https://leakix.net/api-documentation" ], 'apiKeyInstructions': [ "Visit https://leakix.net/", "Register a free account", "Go to your 'Settings'", "Click on 'API key'", "Click on 'Reset key' to generate a new key" ], 'favIcon': "https://leakix.net/public/img/favicon.png", 'logo': "https://leakix.net/public/img/logoleakix-v1.png", 'description': "LeakIX provides insights into devices and servers that are compromised " "and compromised database schemas online.\n" "In this scope we inspect found services for weak credentials.", } } # Default options opts = { 'api_key': "", 'delay': 1, "verify": True, } # Option descriptions optdescs = { 'api_key': "LeakIX API key", 'delay': 'Delay between requests, in seconds.', "verify": "Verify discovered hostnames are valid by checking if they still resolve.", } results = None errorState = False # Initialize module and module options def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in userOpts.keys(): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["IP_ADDRESS", "DOMAIN_NAME"] # What events this module produces def producedEvents(self): return ["RAW_RIR_DATA", "GEOINFO", "TCP_PORT_OPEN", "OPERATING_SYSTEM", "SOFTWARE_USED", "WEBSERVER_BANNER", "LEAKSITE_CONTENT", "INTERNET_NAME"] # Query host # https://leakix.net/api-documentation def queryApi(self, qryType, qry): headers = { "Accept": "application/json", "api-key": self.opts["api_key"] } res = self.sf.fetchUrl( 'https://leakix.net/' + qryType + '/' + qry, headers=headers, timeout=15, useragent=self.opts['_useragent'] ) time.sleep(self.opts['delay']) return self.parseApiResponse(res) # Parse API response def parseApiResponse(self, res: dict): if not res: self.error("No response from LeakIX.") return None if res['code'] == '404': self.debug("Host not found") return None # Future proofing - LeakIX does not implement rate limiting if res['code'] == '429': self.error("You are being rate-limited by LeakIX") self.errorState = True return None # Catch all non-200 status codes, and presume something went wrong if res['code'] != '200': self.error("Failed to retrieve content from LeakIX") self.errorState = True return None if res['content'] is None: return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data ports = list() hosts = list() oses = list() softwares = list() ips = list() banners = list() locs = list() if self.errorState: return if eventData in self.results: return self.results[eventData] = True if self.opts['api_key'] == "": self.debug("You enabled sfp_leakix but did not set an API key, results are limited") self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventName in ["IP_ADDRESS", "DOMAIN_NAME"]: if eventName == "IP_ADDRESS": data = self.queryApi("host", eventData) if eventName == "DOMAIN_NAME": data = self.queryApi("domain", eventData) if data is None: self.debug("No information found for host " + eventData) return evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) services = data.get("Services") if services: for service in services: src = event ipevt = None hostname = service.get('host') if hostname and eventName == "DOMAIN_NAME" and self.getTarget().matches(hostname) and hostname not in hosts: if self.opts["verify"] and not self.sf.resolveHost(hostname) and not self.sf.resolveHost6(hostname): self.debug(f"Host {hostname} could not be resolved") evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", hostname, self.__name__, event) else: evt = SpiderFootEvent("INTERNET_NAME", hostname, self.__name__, event) self.notifyListeners(evt) src = evt hosts.append(hostname) ip = service.get('ip') if ip and eventName != "IP_ADDRESS" and self.sf.validIP(ip) and ip not in ips: evt = SpiderFootEvent("IP_ADDRESS", ip, self.__name__, src) self.notifyListeners(evt) ips.append(ip) ipevt = evt port = service.get('port') if port and ip + ":" + port not in ports: evt = SpiderFootEvent("TCP_PORT_OPEN", ip + ':' + port, self.__name__, src) self.notifyListeners(evt) ports.append(ip + ":" + port) headers = service.get('headers') if headers: servers = headers.get('Server') if servers: for server in servers: if server and server not in banners: evt = SpiderFootEvent('WEBSERVER_BANNER', server, self.__name__, src) self.notifyListeners(evt) banners.append(server) geoip = service.get('geoip') if geoip: location = ', '.join([_f for _f in [geoip.get('city_name'), geoip.get('region_name'), geoip.get('country_name')] if _f]) if location: if ip and self.sf.validIP(ip) and ipevt: # GEOINFO should be linked to an IP_ADDRESS evt = SpiderFootEvent("GEOINFO", location, self.__name__, ipevt) self.notifyListeners(evt) locs.append(location) software = service.get('software') if software: software_version = ' '.join([_f for _f in [software.get('name'), software.get('version')] if _f]) if software_version and software_version not in softwares: evt = SpiderFootEvent("SOFTWARE_USED", software_version, self.__name__, src) self.notifyListeners(evt) softwares.append(software_version) os = software.get('os') if os and os not in oses: evt = SpiderFootEvent('OPERATING_SYSTEM', os, self.__name__, src) self.notifyListeners(evt) oses.append(os) leaks = data.get("Leaks") if leaks: for leak in leaks: leak_protocol = leak.get('type') hostname = leak.get('host') # If protocol is web, our hostname not empty and is not an IP , # and doesn't belong to our target, discard ( happens when sharing Hosting/CDN IPs ) if leak_protocol == "web" and hostname and not self.sf.validIP(hostname) and not self.getTarget().matches(hostname): continue leak_data = leak.get('data') if leak_data: evt = SpiderFootEvent("LEAKSITE_CONTENT", leak_data, self.__name__, event) self.notifyListeners(evt) # End of sfp_leakix class ================================================ FILE: modules/sfp_maltiverse.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_maltiverse # Purpose: Spiderfoot plugin to search Maltiverse API # for any malicious activity by the target # # Author: Krishnasis Mandal # # Created: 20/05/2020 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json from datetime import datetime from netaddr import IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_maltiverse(SpiderFootPlugin): meta = { 'name': "Maltiverse", 'summary': "Obtain information about any malicious activities involving IP addresses", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://maltiverse.com", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://maltiverse.com/faq", "https://app.swaggerhub.com/apis-docs/maltiverse/api/1.0.0-oas3" ], 'favIcon': "https://maltiverse.com/favicon.ico", 'logo': "https://maltiverse.com/assets/images/logo/logo.png", 'description': "The Open IOC Search Engine.\n" "Enhance your SIEM or Firewall and crosscheck your event data with " "top quality Threat Intelligence information to highlight what requires action.", } } opts = { 'checkaffiliates': True, 'subnetlookup': False, 'netblocklookup': True, 'maxnetblock': 24, 'maxsubnet': 24, "age_limit_days": 30 } # Option descriptions. Delete any options not applicable to this module. optdescs = { 'checkaffiliates': "Check affiliates?", 'subnetlookup': "Look up all IPs on subnets which your target is a part of?", 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible blacklisted hosts on the same target subdomain/domain?", 'maxnetblock': "If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", 'maxsubnet': "If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)", "age_limit_days": "Ignore any records older than this many days. 0 = unlimited.", } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input # For a list of all events, check sfdb.py. def watchedEvents(self): return ["IP_ADDRESS", "NETBLOCK_OWNER", "NETBLOCK_MEMBER", "AFFILIATE_IPADDR"] # What events this module produces def producedEvents(self): return ["IP_ADDRESS", "MALICIOUS_IPADDR", "RAW_RIR_DATA", "MALICIOUS_AFFILIATE_IPADDR"] # Check whether the IP Address is malicious using Maltiverse API # https://app.swaggerhub.com/apis-docs/maltiverse/api/1.0.0-oas3#/IPv4/getIP def queryIPAddress(self, qry): headers = { 'Accept': "application/json", } res = self.sf.fetchUrl( 'https://api.maltiverse.com/ip/' + str(qry), headers=headers, timeout=15, useragent=self.opts['_useragent'] ) if res['code'] == "400": self.error("Bad request. " + qry + " is not a valid IP Address") return None if res['code'] == "404": self.error("API endpoint not found") return None if res['code'] != "200": self.debug("No information found from Maltiverse for IP Address") return None try: # Maltiverse returns \\n instead of \n in the response data = str(res['content']).replace("\\n", " ") return json.loads(data) except Exception: self.error("Incorrectly formatted data received as JSON response") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") # Don't look up stuff twice if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True if eventName == 'NETBLOCK_OWNER': if not self.opts['netblocklookup']: return if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']: self.debug("Network size bigger than permitted: " + str(IPNetwork(eventData).prefixlen) + " > " + str(self.opts['maxnetblock'])) return if eventName == 'NETBLOCK_MEMBER': if not self.opts['subnetlookup']: return if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']: self.debug("Network size bigger than permitted: " + str(IPNetwork(eventData).prefixlen) + " > " + str(self.opts['maxsubnet'])) return qrylist = list() if eventName.startswith("NETBLOCK_"): for ipaddr in IPNetwork(eventData): qrylist.append(str(ipaddr)) self.results[str(ipaddr)] = True else: # If user has enabled affiliate checking if eventName == "AFFILIATE_IPADDR" and not self.opts['checkaffiliates']: return qrylist.append(eventData) for addr in qrylist: if self.checkForStop(): return data = self.queryIPAddress(addr) if data is None: break maliciousIP = data.get('ip_addr') if maliciousIP is None: continue if addr != maliciousIP: self.error("Reported address doesn't match requested, skipping") continue blacklistedRecords = data.get('blacklist') if blacklistedRecords is None or len(blacklistedRecords) == 0: self.debug("No blacklist information found for IP") continue # Data is reported about the IP Address if eventName.startswith("NETBLOCK_"): ipEvt = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event) self.notifyListeners(ipEvt) if eventName.startswith("NETBLOCK_"): evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, ipEvt) self.notifyListeners(evt) else: evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) maliciousIPDesc = f"Maltiverse [{maliciousIP}]\n" for blacklistedRecord in blacklistedRecords: lastSeen = blacklistedRecord.get('last_seen') if lastSeen is None: continue try: lastSeenDate = datetime.strptime(str(lastSeen), "%Y-%m-%d %H:%M:%S") except Exception: self.error("Invalid date in JSON response, skipping") continue today = datetime.now() difference = (today - lastSeenDate).days if difference > int(self.opts["age_limit_days"]): self.debug("Record found is older than age limit, skipping") continue maliciousIPDesc += " - DESCRIPTION : " + str(blacklistedRecord.get("description")) + "\n" maliciousIPDescHash = self.sf.hashstring(maliciousIPDesc) if maliciousIPDescHash in self.results: continue self.results[maliciousIPDescHash] = True if eventName.startswith("NETBLOCK_"): evt = SpiderFootEvent("MALICIOUS_IPADDR", maliciousIPDesc, self.__name__, ipEvt) elif eventName.startswith("AFFILIATE_"): evt = SpiderFootEvent("MALICIOUS_AFFILIATE_IPADDR", maliciousIPDesc, self.__name__, event) else: evt = SpiderFootEvent("MALICIOUS_IPADDR", maliciousIPDesc, self.__name__, event) self.notifyListeners(evt) # End of sfp_maltiverse class ================================================ FILE: modules/sfp_malwarepatrol.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_malwarepatrol # Purpose: SpiderFoot plug-in to search MalwarePatrol's daatabase for # potential malicious IPs/hostnames. # # Author: Steve Micallef # # Created: 25/07/2016 # Copyright: (c) Steve Micallef 2016 # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_malwarepatrol(SpiderFootPlugin): meta = { 'name': "MalwarePatrol", 'summary': "Searches malwarepatrol.net's database of malicious URLs/IPs.", 'flags': ["apikey"], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://www.malwarepatrol.net/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://www.malwarepatrol.net/tech-support/", "https://www.malwarepatrol.net/integrations-formats-tip-siem-soar/", "https://www.malwarepatrol.net/community-contribution-suspicious-emails/", "https://www.malwarepatrol.net/non-commercial/#lists" ], 'apiKeyInstructions': [ "Visit https://www.malwarepatrol.net/free-guard-block-list/", "Register a free account", "The Password/Receipt number will be mailed to your email" ], 'favIcon': "https://www.malwarepatrol.net/wp-content/uploads/2020/02/rsz_12logo-150x150.png", 'logo': "https://www.malwarepatrol.net/wp-content/uploads/2016/06/rsz_mp_logo_clear_-_small.png", 'description': "Based in the USA and Brazil, our history is one of community spirit and dedication to " "Internet security that began in 2005 when a group started sharing malicious links using a simple mailing list.\n" "Collecting, analyzing, and sharing data for over a decade has allowed us " "to develop an extensive network of sensors, sharing agreements, and community contributors. " "The result is our vast database of unique and historically rich – “intelligent” – threat data.", } } opts = { "api_key": "", 'checkaffiliates': True, 'checkcohosts': True, } optdescs = { "api_key": "Malwarepatrol.com 'receipt' ID, provided once signing up for their open-source feed. Without this you cannot obtain their feed.", 'checkaffiliates': "Apply checks to affiliates?", 'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?", } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "IP_ADDRESS", "AFFILIATE_IPADDR", "CO_HOSTED_SITE", ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "BLACKLISTED_INTERNET_NAME", "BLACKLISTED_AFFILIATE_INTERNET_NAME", "BLACKLISTED_COHOST", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_COHOST", ] def queryAddr(self, qaddr): data = dict() url = "http://lists.malwarepatrol.net/cgi/getfile?receipt=" + \ self.opts['api_key'] + "&product=8&list=smoothwall" data['content'] = self.sf.cacheGet("sfmalwarepatrol", 72) if data['content'] is None: data = self.sf.fetchUrl(url, useragent=self.opts['_useragent']) if data['content'] is None: self.error("Unable to fetch " + url) return None self.sf.cachePut("sfmalwarepatrol", data['content']) for line in data['content'].split('\n'): if len(line) < 2 or line.startswith('#'): continue if line.startswith(qaddr): return True return False def handleEvent(self, event): eventName = event.eventType eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {event.module}") if not self.opts['api_key']: self.error("You enabled sfp_malwarepatrol but did not provide a receipt ID!") self.errorState = True return if eventData in self.results: self.debug(f"Skipping {eventData} as already searched.") return self.results[eventData] = True if eventName == 'IP_ADDRESS': malicious_type = "MALICIOUS_IPADDR" blacklist_type = "BLACKLISTED_IPADDR" elif eventName == 'AFFILIATE_IPADDR': if not self.opts.get('checkaffiliates', False): return malicious_type = "MALICIOUS_AFFILIATE_IPADDR" blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" elif eventName == "INTERNET_NAME": malicious_type = "MALICIOUS_INTERNET_NAME" blacklist_type = "BLACKLISTED_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": if not self.opts.get('checkaffiliates', False): return malicious_type = "MALICIOUS_AFFILIATE_INTERNET_NAME" blacklist_type = "BLACKLISTED_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": if not self.opts.get('checkcohosts', False): return malicious_type = "MALICIOUS_COHOST" blacklist_type = "BLACKLISTED_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") return if not self.queryAddr(eventData): return text = f"MalwarePatrol [{eventData}]" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_malwarepatrol class ================================================ FILE: modules/sfp_metadefender.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_metadefender # Purpose: Search MetaDefender API for IP address and domain reputation. # # Author: # # Created: 2019-09-21 # Copyright: (c) bcoles 2019 # Licence: MIT # ------------------------------------------------------------------------------- import json import time from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_metadefender(SpiderFootPlugin): meta = { 'name': "MetaDefender", 'summary': "Search MetaDefender API for IP address and domain IP reputation.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://metadefender.opswat.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://onlinehelp.opswat.com/mdcloud/" ], 'apiKeyInstructions': [ "Visit https://metadefender.opswat.com/", "Register a free account", "Navigate to https://metadefender.opswat.com/account", "The API key is listed under 'API key'" ], 'favIcon': "https://mcl-cdn.opswat.com/1.40.3-729f31db/city/icons/icon-48x48.png?v=61be50566cce944a710aaa90ba6bbb8d", 'logo': "https://mcl-cdn.opswat.com/1.40.3-729f31db/city/icons/icon-48x48.png?v=61be50566cce944a710aaa90ba6bbb8d", 'description': "File Analysis - Analyzing binaries with 30+ anti-malware engines.\n" "Heuristic analysis to detect more unknown and targeted attacks.\n" "Binary vulnerability data assessment, IP/Domain reputation, Threat Intelligence Feeds", } } # Default options opts = { 'api_key': '', # 10 requests / minute for free users # 100 requests / minute for paid users 'delay': 6 } # Option descriptions optdescs = { 'api_key': 'MetaDefender API key.', 'delay': 'Delay between requests, in seconds.' } results = None errorState = False # Initialize module and module options def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['IP_ADDRESS', 'INTERNET_NAME'] # What events this module produces def producedEvents(self): return [ 'MALICIOUS_IPADDR', 'MALICIOUS_INTERNET_NAME', 'BLACKLISTED_IPADDR', 'BLACKLISTED_INTERNET_NAME', 'GEOINFO' ] # Query domain REST API # https://onlinehelp.opswat.com/mdcloud/4.5_Domain_Reputation.html def queryDomain(self, qry): headers = { 'Accept': 'application/json', 'apikey': self.opts['api_key'] } res = self.sf.fetchUrl('https://api.metadefender.com/v4/domain/' + qry, headers=headers, timeout=15, useragent=self.opts['_useragent']) time.sleep(self.opts['delay']) return self.parseApiResponse(res) # Query ip REST API # https://onlinehelp.opswat.com/mdcloud/4.1_IP_Reputation.html def queryIp(self, qry): headers = { 'Accept': 'application/json', 'apikey': self.opts['api_key'] } res = self.sf.fetchUrl('https://api.metadefender.com/v4/ip/' + qry, headers=headers, timeout=15, useragent=self.opts['_useragent']) time.sleep(self.opts['delay']) return self.parseApiResponse(res) # Parse API response def parseApiResponse(self, res: dict): if not res: self.error("No response from MetaDefender.") return None if res['code'] == "401": self.error("Invalid MetaDefender API key") self.errorState = True return None # https://onlinehelp.opswat.com/mdcloud/3._Rate_Limiting.html # https://onlinehelp.opswat.com/mdcloud/4._Throttling.html if res['code'] == "429": self.error("You are being rate-limited by MetaDefender") self.errorState = True return None if res['code'] == "404": return None if res['content'] is None: return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType eventData = event.data if self.errorState: return if eventData in self.results: return if self.opts['api_key'] == "": self.error("You enabled sfp_metadefender but did not set an API key!") self.errorState = True return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {event.module}") if eventName == 'IP_ADDRESS': data = self.queryIp(eventData) if data is None: self.debug("No matches found for " + eventData) return geo_info = data.get('geo_info') if geo_info: location = ', '.join([_f for _f in [geo_info.get('city').get('name'), geo_info.get('country').get('name')] if _f]) evt = SpiderFootEvent('GEOINFO', location, self.__name__, event) self.notifyListeners(evt) res = data.get('lookup_results') if not res: self.debug("No matches found for " + eventData) return sources = res.get('sources') if not sources: self.debug("No matches found for " + eventData) return for m in sources: if not m.get('assessment'): continue if m['assessment'] != "trustworthy": continue provider = m.get('provider') evt = SpiderFootEvent('MALICIOUS_IPADDR', provider + ' [' + eventData + ']', self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent('BLACKLISTED_IPADDR', provider + ' [' + eventData + ']', self.__name__, event) self.notifyListeners(evt) if eventName == 'INTERNET_NAME': data = self.queryDomain(eventData) if data is None: self.debug("No matches found for " + eventData) return res = data.get('lookup_results') if not res: self.debug("No matches found for " + eventData) return sources = res.get('sources') if not sources: self.debug("No matches found for " + eventData) return for m in sources: if not m.get('assessment'): continue if m['assessment'] == "trustworthy": continue provider = m.get('provider') evt = SpiderFootEvent('MALICIOUS_INTERNET_NAME', provider + ' [' + eventData + ']', self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent('BLACKLISTED_INTERNET_NAME', provider + ' [' + eventData + ']', self.__name__, event) self.notifyListeners(evt) # End of sfp_metadefender class ================================================ FILE: modules/sfp_mnemonic.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_mnemonic # Purpose: SpiderFoot plug-in for retrieving passive DNS information # from Mnemonic PassiveDNS API. # # Author: # # Created: 2018-10-12 # Copyright: (c) bcoles 2018 # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_mnemonic(SpiderFootPlugin): meta = { 'name': "Mnemonic PassiveDNS", 'summary': "Obtain Passive DNS information from PassiveDNS.mnemonic.no.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Passive DNS"], 'dataSource': { 'website': "https://www.mnemonic.no", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://www.mnemonic.no/resources/whitepapers/", "https://www.mnemonic.no/research-and-development/", "https://docs.mnemonic.no/display/public/API/PassiveDNS+Integration+Guide" ], 'favIcon': "https://www.mnemonic.no/favicon-96x96.png", 'logo': "https://www.mnemonic.no/UI/logo.svg", 'description': "mnemonic helps businesses manage their security risks, " "protect their data and defend against cyber threats.\n" "Our expert team of security consultants, product specialists, " "threat researchers, incident responders and ethical hackers, combined " "with our Argus security platform ensures we stay ahead of " "advanced cyberattacks and protect our customers from evolving threats.", } } opts = { 'per_page': 500, 'max_pages': 2, 'timeout': 30, 'maxage': 180, # 6 months 'verify': True, 'cohostsamedomain': False, 'maxcohost': 100 } optdescs = { 'per_page': "Maximum number of results per page.", 'max_pages': "Maximum number of pages of results to fetch.", 'timeout': "Query timeout, in seconds.", 'maxage': "The maximum age of the data returned, in days, in order to be considered valid.", 'verify': "Verify identified domains still resolve to the associated specified IP address.", 'cohostsamedomain': "Treat co-hosted sites on the same target domain as co-hosting?", 'maxcohost': "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.", } cohostcount = 0 results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.cohostcount = 0 self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ 'IP_ADDRESS', 'IPV6_ADDRESS', 'INTERNET_NAME', 'DOMAIN_NAME' ] def producedEvents(self): return [ 'IP_ADDRESS', 'IPV6_ADDRESS', 'INTERNAL_IP_ADDRESS', 'CO_HOSTED_SITE', 'INTERNET_NAME', 'DOMAIN_NAME' ] def query(self, qry, limit=500, offset=0): """Query the Mnemonic PassiveDNS v3 API. Args: qry (str): domain name or IP address limit (int): Limit the number of returned values. offset (int): Skip the initial number of values in the resultset. Returns: dict: results as JSON """ params = urllib.parse.urlencode({ 'limit': limit, 'offset': offset }) res = self.sf.fetchUrl( f"https://api.mnemonic.no/pdns/v3/{qry}?{params}", timeout=self.opts['timeout'], useragent=self.opts['_useragent'] ) # Unauthenticated users are limited to 100 requests per minute, and 1000 requests per day. time.sleep(0.75) if res['content'] is None: self.info("No results found for " + qry) return None try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response from Mnemonic: {e}") return None response_code = data.get('responseCode') if not response_code: self.debug("Error retrieving search results.") return None if response_code == 402: self.debug("Error retrieving search results: Resource limit exceeded") self.errorState = True return None if response_code != 200: self.debug(f"Error retrieving search results: {response_code}") return None if 'data' not in data: self.info(f"No results found for {qry}") return None size = data.get('size') count = data.get('count') if not count or not size: self.info(f"No results found for {qry}") return None self.info(f"Retrieved {size} of {count} results") return data['data'] def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") position = 0 max_pages = int(self.opts['max_pages']) per_page = int(self.opts['per_page']) agelimit = int(time.time() * 1000) - (86400000 * self.opts['maxage']) self.cohostcount = 0 cohosts = list() while position < (per_page * max_pages): if self.checkForStop(): break if self.errorState: break data = self.query(eventData, limit=per_page, offset=position) if data is None: self.info(f"No passive DNS data found for {eventData}") break position += per_page for r in data: if "*" in r['query'] or "%" in r['query']: continue if r['lastSeenTimestamp'] < agelimit: self.debug(f"Record {r['answer']} found for {r['query']} is too old, skipping.") continue if eventName in ['IP_ADDRESS']: if r['rrtype'] == 'a': if self.sf.validIP(r['query']): cohosts.append(r['query']) continue if eventName in ['INTERNET_NAME', 'DOMAIN_NAME']: # Ignore PTR records if r['rrtype'] == 'ptr': continue if r['rrtype'] == 'cname': if not self.getTarget().matches(r['query'], includeParents=True): continue cohosts.append(r['query']) if self.opts['verify']: continue answer = r.get('answer') if r['rrtype'] == 'a': if not self.sf.validIP(answer): continue if self.sf.isValidLocalOrLoopbackIp(answer): evt = SpiderFootEvent("INTERNAL_IP_ADDRESS", answer, self.__name__, event) else: evt = SpiderFootEvent("IP_ADDRESS", answer, self.__name__, event) self.notifyListeners(evt) if r['rrtype'] == 'aaaa': if not self.sf.validIP6(r['answer']): continue if self.sf.isValidLocalOrLoopbackIp(answer): evt = SpiderFootEvent("INTERNAL_IP_ADDRESS", answer, self.__name__, event) else: evt = SpiderFootEvent("IPV6_ADDRESS", answer, self.__name__, event) self.notifyListeners(evt) for co in set(cohosts): if self.checkForStop(): return if co in self.results: continue if eventName in ["IP_ADDRESS", "IPV6_ADDRESS"]: if self.opts['verify'] and not self.sf.validateIP(co, eventData): self.debug(f"Host {co} no longer resolves to {eventData}") continue if self.opts['cohostsamedomain']: if self.cohostcount < self.opts['maxcohost']: evt = SpiderFootEvent("CO_HOSTED_SITE", co, self.__name__, event) self.notifyListeners(evt) self.cohostcount += 1 continue if self.getTarget().matches(co, includeParents=True): if self.opts['verify'] and not self.sf.resolveHost(co) and not self.sf.resolveHost6(co): self.debug(f"Host {co} could not be resolved") evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", co, self.__name__, event) self.notifyListeners(evt) continue evt = SpiderFootEvent("INTERNET_NAME", co, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(co, self.opts['_internettlds']): evt = SpiderFootEvent("DOMAIN_NAME", co, self.__name__, event) self.notifyListeners(evt) # End of sfp_mnemonic class ================================================ FILE: modules/sfp_multiproxy.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_multiproxy # Purpose: Check if an IP arress is an open proxy according to multiproxy.org # open proxy list. # # Author: steve@binarypool.com # # Created: 14/12/2013 # Copyright: (c) Steve Micallef, 2013 # Licence: MIT # ------------------------------------------------------------------------------- from netaddr import IPAddress, IPNetwork from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_multiproxy(SpiderFootPlugin): meta = { 'name': "multiproxy.org Open Proxies", 'summary': "Check if an IP address is an open proxy according to multiproxy.org open proxy list.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Secondary Networks"], 'dataSource': { 'website': "https://multiproxy.org/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://multiproxy.org/faq.htm", "https://multiproxy.org/env_check.htm", "https://multiproxy.org/anon_proxy.htm", "https://multiproxy.org/help.htm" ], 'favIcon': "https://www.google.com/s2/favicons?domain=https://multiproxy.org/", 'logo': "https://multiproxy.org/images/mproxy_title.png", 'description': "MultiProxy is a multifunctional personal proxy server that protects your privacy " "while on the Internet as well as speeds up your downloads, " "especially if you are trying to get several files form overseas or from otherwise rather slow server. " "It can also completely hide your IP address by dynamically connecting to " "non-transparent anonymizing public proxy servers. " "You can also test a list of proxy servers and sort them by connection speed and level of anonimity.", } } opts = { 'checkaffiliates': True, 'cacheperiod': 18 } optdescs = { 'checkaffiliates': "Apply checks to affiliates?", 'cacheperiod': "Hours to cache list data before re-fetching." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ 'IP_ADDRESS', 'AFFILIATE_IPADDR', 'NETBLOCK_OWNER', 'NETBLOCK_MEMBER', ] def producedEvents(self): return [ "BLACKLISTED_IPADDR", "BLACKLISTED_AFFILIATE_IPADDR", "BLACKLISTED_SUBNET", "BLACKLISTED_NETBLOCK", "MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK", "MALICIOUS_SUBNET", ] def queryProxyList(self, target, targetType): proxy_list = self.retrieveProxyList() if not proxy_list: self.errorState = True return False if targetType == "ip": if target in proxy_list: self.debug(f"IP address {target} found in multiproxy.org open proxy list.") return True elif targetType == "netblock": netblock = IPNetwork(target) for ip in proxy_list: if IPAddress(ip) in netblock: self.debug(f"IP address {ip} found within netblock/subnet {target} in multiproxy.org open proxy list.") return True return False def retrieveProxyList(self): proxy_list = self.sf.cacheGet('multiproxyopenproxies', 24) if proxy_list is not None: return self.parseProxyList(proxy_list) res = self.sf.fetchUrl( "http://multiproxy.org/txt_all/proxy.txt", timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from multiproxy.org.") self.errorState = True return None if res['content'] is None: self.error("Received no content from multiproxy.org") self.errorState = True return None self.sf.cachePut("multiproxyopenproxies", res['content']) return self.parseProxyList(res['content']) def parseProxyList(self, proxy_list): """Parse plaintext open proxy list Args: proxy_list (str): plaintext open proxy list from multiproxy.org Returns: list: list of open proxy IP addresses """ ips = list() if not proxy_list: return ips for ip in proxy_list.split('\n'): ip = ip.strip().split(":")[0] if ip.startswith('#'): continue if not self.sf.validIP(ip): continue ips.append(ip) return ips def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName == 'IP_ADDRESS': targetType = 'ip' malicious_type = "MALICIOUS_IPADDR" blacklist_type = "BLACKLISTED_IPADDR" elif eventName == 'AFFILIATE_IPADDR': if not self.opts.get('checkaffiliates', False): return targetType = 'ip' malicious_type = "MALICIOUS_AFFILIATE_IPADDR" blacklist_type = "BLACKLISTED_AFFILIATE_IPADDR" elif eventName == 'NETBLOCK_OWNER': if not self.opts.get('checknetblocks', False): return targetType = 'netblock' malicious_type = "MALICIOUS_NETBLOCK" blacklist_type = "BLACKLISTED_NETBLOCK" elif eventName == 'NETBLOCK_MEMBER': if not self.opts.get('checksubnets', False): return targetType = 'netblock' malicious_type = "MALICIOUS_SUBNET" blacklist_type = "BLACKLISTED_SUBNET" else: self.debug(f"Unexpected event type {eventName}, skipping") return self.debug(f"Checking maliciousness of {eventData} ({eventName}) with multiproxy.org open proxy list") if not self.queryProxyList(eventData, targetType): return url = "http://multiproxy.org/txt_all/proxy.txt" text = f"multiproxy.org Open Proxies [{eventData}]\n{url}" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_multiproxy class ================================================ FILE: modules/sfp_myspace.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_myspace # Purpose: Query MySpace for username and location information. # # Author: # # Created: 2018-10-07 # Copyright: (c) bcoles 2018 # Licence: MIT # ------------------------------------------------------------------------------- import re from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_myspace(SpiderFootPlugin): meta = { 'name': "MySpace", 'summary': "Gather username and location from MySpace.com profiles.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Social Media"], 'dataSource': { 'website': "https://myspace.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://www.programmableweb.com/api/myspace" ], 'favIcon': "https://x.myspacecdn.com/new/common/images/favicons/favicon.ico", 'logo': "https://x.myspacecdn.com/new/common/images/favicons/114-Retina-iPhone.png", 'description': "Myspace is a place where people come to connect, discover, and share.\n" "Through an open design, compelling editorial features, " "and analytics-based recommendations, Myspace creates a creative community " "of people who connect around mutual affinity and inspiration for the purpose " "of shaping, sharing, and discovering what's next.", } } opts = { } optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.__dataSource__ = "MySpace.com" self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["EMAILADDR", "SOCIAL_MEDIA"] def producedEvents(self): return ["SOCIAL_MEDIA", "GEOINFO"] def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") # Search by email address if eventName == "EMAILADDR": email = eventData res = self.sf.fetchUrl("https://myspace.com/search/people?q=" + email, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if res['content'] is None: self.error(f"Could not fetch MySpace content for {email}") return # Extract HTML containing potential profile matches profiles = re.findall(r'[^<]+', str(res['content'])) if not profiles: self.debug(f"No profiles found for e-mail: {email}") return # The first result is the closest match, but whether it's an exact match is unknown. profile = profiles[0] # Check for email address as name, at the risk of missed results. try: matches = re.findall(r'\.\?\!]+' + email + r'[\&; :\"\#\*\)\"\'\;\,\<\.\?\!]+', profile, re.IGNORECASE) except Exception: self.debug("Malformed e-mail address, skipping.") return if not matches: self.debug("No concrete match for that e-mail.") return name = matches[0] e = SpiderFootEvent( "SOCIAL_MEDIA", f"MySpace: https://myspace.com/{name}", self.__name__, event ) self.notifyListeners(e) # Retrieve location from MySpace profile if eventName == "SOCIAL_MEDIA": try: network = eventData.split(": ")[0] url = eventData.split(": ")[1].replace("", "").replace("", "") except Exception as e: self.debug(f"Unable to parse SOCIAL_MEDIA: {eventData} ({e})") return if network != "MySpace": self.debug(f"Skipping social network profile, {url}, as not a MySpace profile") return res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) if res['content'] is None: return data = re.findall(r'
100: self.debug("Skipping likely invalid location.") return e = SpiderFootEvent("GEOINFO", location, self.__name__, event) self.notifyListeners(e) # End of sfp_myspace class ================================================ FILE: modules/sfp_nameapi.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_nameapi # Purpose: Spiderfoot plugin to check if an email is # disposable using nameapi.org API. # # Author: Krishnasis Mandal # # Created: 2020-10-02 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_nameapi(SpiderFootPlugin): meta = { 'name': "NameAPI", 'summary': "Check whether an email is disposable", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://www.nameapi.org/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://www.nameapi.org/en/developer/manuals/rest-web-services/53/web-services/disposable-email-address-detector/" ], 'apiKeyInstructions': [ "Visit https://nameapi.org", "Click on 'Get API Key'", "Register a free account", "The API key will be sent to your email" ], 'favIcon': "https://www.nameapi.org/fileadmin/favicon.ico", 'logo': "https://www.nameapi.org/fileadmin/templates/nameprofiler/images/name-api-logo.png", 'description': "The NameAPI DEA-Detector checks email addresses " "against a list of known trash domains such as mailinator.com.\n" "It classifies those as disposable which operate as a time-limited, " "web based way of receiving emails, for example, sign up confirmations.", } } opts = { 'api_key': '' } optdescs = { 'api_key': "API Key for NameAPI" } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "EMAILADDR" ] def producedEvents(self): return [ "EMAILADDR_DISPOSABLE", "RAW_RIR_DATA" ] def queryEmailAddr(self, qry): res = self.sf.fetchUrl( f"http://api.nameapi.org/rest/v5.3/email/disposableemailaddressdetector?apiKey={self.opts['api_key']}&emailAddress={qry}", timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot" ) if res['content'] is None: self.info(f"No NameAPI info found for {qry}") return None try: return json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from NameAPI: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.error( f"You enabled {self.__class__.__name__} but did not set an API key!" ) self.errorState = True return self.results[eventData] = True data = self.queryEmailAddr(eventData) if data is None: return isDisposable = data.get('disposable') if isDisposable == "YES": evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent("EMAILADDR_DISPOSABLE", eventData, self.__name__, event) self.notifyListeners(evt) # End of sfp_nameapi class ================================================ FILE: modules/sfp_names.py ================================================ # coding: utf-8 # ------------------------------------------------------------------------------- # Name: sfp_names # Purpose: Identify human names in content fetched. # # Author: Steve Micallef # # Created: 24/03/2014 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import re from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_names(SpiderFootPlugin): meta = { 'name': "Human Name Extractor", 'summary': "Attempt to identify human names in fetched content.", 'flags': ["errorprone"], 'useCases': ["Footprint", "Passive"], 'categories': ["Content Analysis"] } # Default options opts = { 'algolimit': 75, 'emailtoname': True, 'filterjscss': True } # Option descriptions optdescs = { 'algolimit': "A value between 0-100 to tune the sensitivity of the name finder. Less than 40 will give you a lot of junk, over 50 and you'll probably miss things but will have less false positives.", 'emailtoname': "Convert e-mail addresses in the form of firstname.surname@target to names?", 'filterjscss': "Filter out names that originated from CSS/JS content. Enabling this avoids detection of popular Javascript and web framework author names." } results = None d = None n = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.d = SpiderFootHelpers.dictionaryWordsFromWordlists() self.n = SpiderFootHelpers.humanNamesFromWordlists() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["TARGET_WEB_CONTENT", "EMAILADDR", "DOMAIN_WHOIS", "NETBLOCK_WHOIS", "RAW_RIR_DATA", "RAW_FILE_META_DATA"] # What events this module produces def producedEvents(self): return ["HUMAN_NAME"] # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") # If the source event is web content, check if the source URL was javascript # or CSS, in which case optionally ignore it. if eventName == "TARGET_WEB_CONTENT": url = event.actualSource if url is not None: if self.opts['filterjscss'] and (".js" in url or ".css" in url): self.debug("Ignoring web content from CSS/JS.") return # Find names in email addresses in ".@" format if eventName == "EMAILADDR" and self.opts['emailtoname']: potential_name = eventData.split("@")[0] if "." not in potential_name: return name = " ".join(map(str.capitalize, potential_name.split("."))) # Names usually do not contain numbers if re.search("[0-9]", name): return evt = SpiderFootEvent("HUMAN_NAME", name, self.__name__, event) if event.moduleDataSource: evt.moduleDataSource = event.moduleDataSource else: evt.moduleDataSource = "Unknown" self.notifyListeners(evt) return # For RAW_RIR_DATA, there are only specific modules we # expect to see RELEVANT names within. if eventName == "RAW_RIR_DATA": if srcModuleName not in ["sfp_builtwith", "sfp_clearbit", "sfp_emailcrawlr", "sfp_fullcontact", "sfp_github", "sfp_hunter", "sfp_opencorporates", "sfp_slideshare", "sfp_jsonwhoiscom", "sfp_twitter", "sfp_gravatar", "sfp_keybase"]: self.debug("Ignoring RAW_RIR_DATA from untrusted module.") return # Stage 1: Find things that look (very vaguely) like names rx = re.compile(r"([A-Z][a-z�������������]+)\s+.?.?\s?([A-Z][�������������a-zA-Z\'\-]+)") m = re.findall(rx, eventData) for r in m: # Start off each match as 0 points. p = 0 notindict = False # Shouldn't encounter "Firstname's Secondname" first = r[0].lower() if first[len(first) - 2] == "'" or first[len(first) - 1] == "'": continue # Strip off trailing ' or 's secondOrig = r[1].replace("'s", "") secondOrig = secondOrig.rstrip("'") second = r[1].lower().replace("'s", "") second = second.rstrip("'") # If both words are not in the dictionary, add 75 points. if first not in self.d and second not in self.d: self.debug(f"Both first and second names are not in the dictionary, so high chance of name: ({first}:{second}).") p += 75 notindict = True else: self.debug(first + " was found or " + second + " was found in dictionary.") # If the first word is a known popular first name, award 50 points. if first in self.n: p += 50 # If either word is 2 characters, subtract 50 points. if len(first) == 2 or len(second) == 2: p -= 50 # If the first word is in the dictionary but the second isn't, # subtract 40 points. if not notindict: if first in self.d and second not in self.d: p -= 20 # If the second word is in the dictionary but the first isn't, # reduce 20 points. if first not in self.d and second in self.d: p -= 40 name = r[0] + " " + secondOrig self.debug("Name of " + name + " has score: " + str(p)) if p >= self.opts['algolimit']: # Notify other modules of what you've found evt = SpiderFootEvent("HUMAN_NAME", name, self.__name__, event) if event.moduleDataSource: evt.moduleDataSource = event.moduleDataSource else: evt.moduleDataSource = "Unknown" self.notifyListeners(evt) # End of sfp_names class ================================================ FILE: modules/sfp_networksdb.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_networksdb # Purpose: SpiderFoot plug-in to search NetworksDB.io API for IP address and # domain information. # # Author: # # Created: 2019-09-16 # Copyright: (c) bcoles 2019 # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_networksdb(SpiderFootPlugin): meta = { 'name': "NetworksDB", 'summary': "Search NetworksDB.io API for IP address and domain information.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Passive DNS"], 'dataSource': { 'website': "https://networksdb.io/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://networksdb.io/api/docs" ], 'apiKeyInstructions': [ "Visit https://networksdb.io/api/order/free", "Register a free account", "Click on 'Generate a Free API Key'", "The API key is listed under 'API Key: Free plan'" ], 'favIcon': "https://networksdb.io/img/favicon/favicon-96x96.png", 'logo': "https://networksdb.io/img/logo.png", 'description': "Our database contains information about the public IPv4 and IPv6 addresses, " "networks and domains owned by companies and organisations across the world " "along with city-level IP geolocation data and autonomous system information.", } } # Default options opts = { 'api_key': '', 'delay': 1, 'verify': True, 'cohostsamedomain': False, 'maxcohost': 100 } # Option descriptions optdescs = { 'api_key': 'NetworksDB API key.', 'delay': 'Delay between requests, in seconds.', 'verify': "Verify co-hosts are valid by checking if they still resolve to the shared IP.", 'cohostsamedomain': "Treat co-hosted sites on the same target domain as co-hosting?", 'maxcohost': "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting." } cohostcount = 0 results = None errorState = False # Initialize module and module options def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.cohostcount = 0 self.errorState = False for opt in userOpts.keys(): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["IP_ADDRESS", "IPV6_ADDRESS", "INTERNET_NAME", "DOMAIN_NAME"] # What events this module produces def producedEvents(self): return ["INTERNET_NAME", "IP_ADDRESS", "IPV6_ADDRESS", "NETBLOCK_MEMBER", "CO_HOSTED_SITE", "GEOINFO", "RAW_RIR_DATA"] # Query IP Address Info # https://networksdb.io/api/docs#ipinfo def queryIpInfo(self, qry): params = { 'ip': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), } headers = { 'Accept': 'application/json', 'X-Api-Key': self.opts['api_key'], 'Content-Type': 'application/x-www-form-urlencoded', } res = self.sf.fetchUrl('https://networksdb.io/api/ip-info', headers=headers, postData=urllib.parse.urlencode(params), timeout=15, useragent=self.opts['_useragent']) time.sleep(self.opts['delay']) return self.parseApiResponse(res) # Query IP Geolocation # https://networksdb.io/api/docs#geoip def queryIpGeo(self, qry): params = { 'ip': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), } headers = { 'Accept': 'application/json', 'X-Api-Key': self.opts['api_key'], 'Content-Type': 'application/x-www-form-urlencoded', } res = self.sf.fetchUrl('https://networksdb.io/api/ip-geo', headers=headers, postData=urllib.parse.urlencode(params), timeout=15, useragent=self.opts['_useragent']) time.sleep(self.opts['delay']) return self.parseApiResponse(res) # Query Domains on IP (Reverse DNS) # https://networksdb.io/api/docs#revdns def queryReverseDns(self, qry): params = { 'ip': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), } headers = { 'Accept': 'application/json', 'X-Api-Key': self.opts['api_key'], 'Content-Type': 'application/x-www-form-urlencoded', } res = self.sf.fetchUrl('https://networksdb.io/api/reverse-dns', headers=headers, postData=urllib.parse.urlencode(params), timeout=15, useragent=self.opts['_useragent']) time.sleep(self.opts['delay']) return self.parseApiResponse(res) # Query IPs for Domain (Forward DNS) # https://networksdb.io/api/docs#fwddns def queryForwardDns(self, qry): params = { 'domain': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), } headers = { 'Accept': 'application/json', 'X-Api-Key': self.opts['api_key'], 'Content-Type': 'application/x-www-form-urlencoded', } res = self.sf.fetchUrl('https://networksdb.io/api/dns', headers=headers, postData=urllib.parse.urlencode(params), timeout=15, useragent=self.opts['_useragent']) time.sleep(self.opts['delay']) return self.parseApiResponse(res) # Query Autonomous System Info # https://networksdb.io/api/docs#asinfo # Note: currently unused def queryAsnInfo(self, qry): params = { 'asn': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), } headers = { 'Accept': 'application/json', 'X-Api-Key': self.opts['api_key'], 'Content-Type': 'application/x-www-form-urlencoded', } res = self.sf.fetchUrl('https://networksdb.io/api/asn', headers=headers, postData=urllib.parse.urlencode(params), timeout=15, useragent=self.opts['_useragent']) time.sleep(self.opts['delay']) return self.parseApiResponse(res) # Query Autonomous System Networks # https://networksdb.io/api/docs#asnets # Note: currently unused def queryAsnNetworks(self, qry): params = { 'asn': qry, } headers = { 'Accept': 'application/json', 'X-Api-Key': self.opts['api_key'], 'Content-Type': 'application/x-www-form-urlencoded', } res = self.sf.fetchUrl('https://networksdb.io/api/asn-networks', headers=headers, postData=urllib.parse.urlencode(params), timeout=15, useragent=self.opts['_useragent']) time.sleep(self.opts['delay']) return self.parseApiResponse(res) # Parse API response # https://networksdb.io/api/plans def parseApiResponse(self, res: dict): if not res: self.error("No response from NetworksDB.") return None # Future proofing - NetworksDB does not implement rate limiting if res['code'] == '429': self.error("You are being rate-limited by NetworksDB") self.errorState = True return None if res['code'] == '403': self.error("Authentication failed") self.errorState = True return None if res['content'] is None: return None try: data = json.loads(res['content']) except Exception as e: self.error(f"Error processing JSON response from NetworksDB: {e}") return None if data.get('warning'): self.debug("Received warning from NetworksDB: " + data.get('warning')) if data.get('error'): self.error("Received error from NetworksDB: " + data.get('error')) return data # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if eventData in self.results: return if self.opts['api_key'] == '': self.error("You enabled sfp_networksdb but did not set an API key!") self.errorState = True return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventName in ["IP_ADDRESS", "IPV6_ADDRESS"]: data = self.queryIpInfo(eventData) if data is None: self.debug("No IP address information found for " + eventData) else: evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) network = data.get('network') if network: cidr = network.get('cidr') if cidr and cidr != 'N/A' and self.sf.validIpNetwork(cidr): if ":" in cidr: evt = SpiderFootEvent('NETBLOCKV6_MEMBER', cidr, self.__name__, event) else: evt = SpiderFootEvent('NETBLOCK_MEMBER', cidr, self.__name__, event) self.notifyListeners(evt) data = self.queryIpGeo(eventData) if data is None: self.debug("No IP geolocation information found for " + eventData) else: evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) if data.get('country'): location = ', '.join(filter(None, [data.get('city'), data.get('state'), data.get('country')])) evt = SpiderFootEvent('GEOINFO', location, self.__name__, event) self.notifyListeners(evt) data = self.queryReverseDns(eventData) cohosts = list() if data is None: self.debug("No reverse DNS results for " + eventData) else: evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event) self.notifyListeners(evt) results = data.get('results') if results: for domain in results: cohosts.append(domain) for co in set(cohosts): if self.checkForStop(): return if co in self.results: continue if self.opts['verify'] and not self.sf.validateIP(co, eventData): self.debug("Host " + co + " no longer resolves to " + eventData) continue if not self.opts['cohostsamedomain']: if self.getTarget().matches(co, includeParents=True): evt = SpiderFootEvent('INTERNET_NAME', co, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(co, self.opts['_internettlds']): evt = SpiderFootEvent('DOMAIN_NAME', co, self.__name__, event) self.notifyListeners(evt) continue if self.cohostcount < self.opts['maxcohost']: evt = SpiderFootEvent('CO_HOSTED_SITE', co, self.__name__, event) self.notifyListeners(evt) self.cohostcount += 1 if eventName in ["INTERNET_NAME", "DOMAIN_NAME"]: data = self.queryForwardDns(eventData) if data is None: self.debug("No forward DNS results for " + eventData) return res = data.get('results') if not res: self.debug("No forward DNS results for " + eventData) return evt = SpiderFootEvent('RAW_RIR_DATA', str(res), self.__name__, event) self.notifyListeners(evt) for ip in res: if self.sf.validIP(ip): evt = SpiderFootEvent('IP_ADDRESS', ip, self.__name__, event) self.notifyListeners(evt) elif self.sf.validIP6(ip): evt = SpiderFootEvent('IPV6_ADDRESS', ip, self.__name__, event) self.notifyListeners(evt) # End of sfp_networksdb class ================================================ FILE: modules/sfp_neutrinoapi.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_neutrinoapi # Purpose: SpiderFoot plug-in to search NeutrinoAPI for IP address info, # check IP address reputation, and search for phone location. # # Author: # # Created: 2018-11-30 # Copyright: (c) bcoles 2018 # Licence: MIT # ------------------------------------------------------------------------------- import json from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_neutrinoapi(SpiderFootPlugin): meta = { 'name': "NeutrinoAPI", 'summary': "Search NeutrinoAPI for phone location information, IP address information, and host reputation.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://www.neutrinoapi.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://www.neutrinoapi.com/api/api-basics/", "https://www.neutrinoapi.com/api/phone-validate/", "https://www.neutrinoapi.com/api/ip-info/", "https://www.neutrinoapi.com/api/ip-blocklist/", "https://www.neutrinoapi.com/api/host-reputation/", ], 'apiKeyInstructions': [ "Visit https://www.neutrinoapi.com/", "Sign up for a free account", "Click on 'My Account'", "The API key is listed under 'Master Key'" ], 'favIcon': "https://www.neutrinoapi.com/favicon.png", 'logo': "https://www.neutrinoapi.com/favicon.png", 'description': "Neutrino API - The general-purpose API - Build smarter applications." } } # Default options opts = { 'user_id': '', 'api_key': '', 'timeout': 30 } # Option descriptions optdescs = { 'user_id': "NeutrinoAPI user ID.", 'api_key': "NeutrinoAPI API key.", 'timeout': "Query timeout, in seconds." } results = None errorState = False # Initialize module and module options def setup(self, sfc, userOpts=dict()): self.sf = sfc self.__dataSource__ = "NeutrinoAPI" self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['IP_ADDRESS', 'IPV6_ADDRESS', 'PHONE_NUMBER'] # What events this module produces def producedEvents(self): return [ 'RAW_RIR_DATA', 'BLACKLISTED_IPADDR', 'MALICIOUS_IPADDR', 'PROXY_HOST', 'VPN_HOST', 'TOR_EXIT_NODE', 'GEOINFO', ] # Query the phone-validate REST API # https://www.neutrinoapi.com/api/phone-validate/ def queryPhoneValidate(self, qry): res = self.sf.fetchUrl( 'https://neutrinoapi.com/phone-validate', postData={"output-format": "json", "number": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']}, timeout=self.opts['timeout'], useragent=self.opts['_useragent'] ) return self.parseApiResponse(res) # Query the ip-info REST API # https://www.neutrinoapi.com/api/ip-info/ def queryIpInfo(self, qry): res = self.sf.fetchUrl( "https://neutrinoapi.com/ip-info", postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']}, timeout=self.opts['timeout'], useragent=self.opts['_useragent'] ) return self.parseApiResponse(res) # Query the ip-blocklist REST API # https://www.neutrinoapi.com/api/ip-blocklist/ def queryIpBlocklist(self, qry): res = self.sf.fetchUrl( "https://neutrinoapi.com/ip-blocklist", postData={"output-format": "json", "ip": qry, "vpn-lookup": True, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']}, timeout=self.opts['timeout'], useragent=self.opts['_useragent'] ) return self.parseApiResponse(res) # Query the host-reputation REST API # https://www.neutrinoapi.com/api/host-reputation/ def queryHostReputation(self, qry): res = self.sf.fetchUrl( "https://neutrinoapi.com/host-reputation", postData={"output-format": "json", "host": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']}, timeout=self.opts['timeout'], useragent=self.opts['_useragent'] ) return self.parseApiResponse(res) # Parse API response def parseApiResponse(self, res: dict): if not res: self.error("No response from NeutrinoAPI.") return None if res['code'] == "403": self.error("Authentication failed") self.errorState = True return None if res['content'] is None: return None try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None if res['code'] == "400": if data.get('api-error-msg'): self.error("Error: " + data.get('api-error-msg')) if "EXCEED" in data.get('api-error-msg'): self.errorState = True return None else: self.error("Error: HTTP 400") return None return data def handleEvent(self, event): eventName = event.eventType eventData = event.data if self.errorState: return self.debug(f"Received event, {eventName}, from {event.module}") if self.opts['api_key'] == "": self.error("You enabled sfp_neutrinoapi but did not set an API key!") self.errorState = True return if self.opts['user_id'] == "": self.error("You enabled sfp_neutrinoapi but did not set a user ID!") self.errorState = True return if eventData in self.results: return self.results[eventData] = True if eventName == 'PHONE_NUMBER': data = self.queryPhoneValidate(eventData) if data is None: self.debug("No phone info results found for " + eventData) else: if data.get('location') is not None and data.get('country') is not None: if data.get('location') == data.get('country'): location = data.get('location') else: location = data.get('location') + ', ' + data.get('country') evt = SpiderFootEvent("GEOINFO", location, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) if eventName in ['IP_ADDRESS', 'IPV6_ADDRESS']: data = self.queryIpInfo(eventData) if data is None: self.debug("No IP info results found for " + eventData) else: if data.get('city') is not None and data.get('region') is not None and data.get('country-code') is not None: location = data.get('city') + ', ' + data.get('region') + ', ' + data.get('country-code') evt = SpiderFootEvent("GEOINFO", location, self.__name__, event) self.notifyListeners(evt) data = self.queryIpBlocklist(eventData) if data is None: self.debug("No IP blocklist results found for " + eventData) else: if data.get('is-listed'): evt = SpiderFootEvent("MALICIOUS_IPADDR", f"NeutrinoAPI - IP Blocklist [{eventData}]", self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent("BLACKLISTED_IPADDR", f"NeutrinoAPI - IP Blocklist [{eventData}]", self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) if data.get('is-proxy'): evt = SpiderFootEvent("PROXY_HOST", eventData, self.__name__, event) self.notifyListeners(evt) if data.get('is-vpn'): evt = SpiderFootEvent("VPN_HOST", eventData, self.__name__, event) self.notifyListeners(evt) if data.get('is-tor'): evt = SpiderFootEvent("TOR_EXIT_NODE", eventData, self.__name__, event) self.notifyListeners(evt) data = self.queryHostReputation(eventData) if data is None: self.debug("No host reputation results found for " + eventData) else: if data.get('is-listed'): evt = SpiderFootEvent("MALICIOUS_IPADDR", f"NeutrinoAPI - Host Reputation [{eventData}]", self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent("BLACKLISTED_IPADDR", f"NeutrinoAPI - Host Reputation [{eventData}]", self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) # End of sfp_neutrinoapi class ================================================ FILE: modules/sfp_numverify.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_numverify # Purpose: SpiderFoot plug-in to search numverify.com API for a phone number # and retrieve location and carrier information. # # Author: # # Created: 2019-05-25 # Copyright: (c) bcoles 2019 # Licence: MIT # ------------------------------------------------------------------------------- import json import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin class sfp_numverify(SpiderFootPlugin): meta = { 'name': "numverify", 'summary': "Lookup phone number location and carrier information from numverify.com.", 'flags': ["apikey"], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Real World"], 'dataSource': { 'website': "http://numverify.com/", 'model': "FREE_AUTH_LIMITED", 'references': [ "https://numverify.com/documentation", "https://numverify.com/faq" ], 'apiKeyInstructions': [ "Visit https://numverify.com", "Sign up for a free account", "Navigate to https://numverify.com/dashboard", "The API key is listed under 'Your API Access Key'" ], 'favIcon': "https://numverify.com/images/icons/numverify_shortcut_icon.ico", 'logo': "https://numverify.com/images/logos/numverify_header.png", 'description': "Global Phone Number Validation & Lookup JSON API.\n" "NumVerify offers a full-featured yet simple RESTful JSON API for " "national and international phone number validation and information lookup for a total of 232 countries around the world.\n" "Requested numbers are processed in real-time, cross-checked with the latest international numbering plan databases " "and returned in handy JSON format enriched with useful carrier, geographical location and line type data.", } } # Default options opts = { 'api_key': '' } # Option descriptions optdescs = { 'api_key': 'numverify API key.' } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['PHONE_NUMBER'] # What events this module produces def producedEvents(self): return ['RAW_RIR_DATA', 'GEOINFO', 'PROVIDER_TELCO'] # Query numverify API for the specified phone number # https://numverify.com/documentation def query(self, qry): number = qry.strip('+').strip('(').strip(')') params = { 'number': number.encode('raw_unicode_escape').decode("ascii", errors='replace'), 'country_code': '', 'format': '0', # set to "1" for prettified debug output 'access_key': self.opts['api_key'] } # Free API does not support HTTPS for no adequately explained reason res = self.sf.fetchUrl("http://apilayer.net/api/validate?" + urllib.parse.urlencode(params), timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent']) time.sleep(1) if res['content'] is None: self.debug('No response from apilayer.net') return None if res['code'] == '101': self.error('API error: invalid API key') self.errorState = True return None if res['code'] == '102': self.error('API error: user account deactivated') self.errorState = True return None if res['code'] == '104': self.error('API error: usage limit exceeded') self.errorState = True return None try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None if data.get('error') is not None: self.error('API error: ' + str(data.get('error'))) return None return data # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if self.opts['api_key'] == "": self.error("You enabled sfp_numverify but did not set an API key!") self.errorState = True return if eventData in self.results: return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") data = self.query(eventData) if data is None: self.debug("No phone information found for " + eventData) return evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event) self.notifyListeners(evt) if data.get('country_code'): country = SpiderFootHelpers.countryNameFromCountryCode(data.get('country_code')) location = ', '.join([_f for _f in [data.get('location'), country] if _f]) evt = SpiderFootEvent("GEOINFO", location, self.__name__, event) self.notifyListeners(evt) else: self.debug("No location information found for " + eventData) if data.get('carrier'): evt = SpiderFootEvent("PROVIDER_TELCO", data.get('carrier'), self.__name__, event) self.notifyListeners(evt) else: self.debug("No carrier information found for " + eventData) # End of sfp_numverify class ================================================ FILE: modules/sfp_onioncity.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_onioncity # Purpose: Searches the Tor search engine 'Onion City' using Google Custom # Search for content related to the domain in question. # # Author: Steve Micallef # # Created: 15/07/2015 # Copyright: (c) Steve Micallef 2015 # Licence: MIT # ------------------------------------------------------------------------------- import re from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_onioncity(SpiderFootPlugin): meta = { 'name': "Onion.link", 'summary': "Search Tor 'Onion City' search engine for mentions of the target domain using Google Custom Search.", 'flags': ["apikey", "tor"], 'useCases': ["Footprint", "Investigate"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://onion.link/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://developers.google.com/custom-search/v1", "https://developers.google.com/custom-search/docs/overview", "https://cse.google.com/cse" ], 'apiKeyInstructions': [ "Visit https://developers.google.com/custom-search/v1/introduction", "Register a free Google account", "Click on 'Get A Key'", "Connect a Project", "The API Key will be listed under 'YOUR API KEY'" ], 'favIcon': "https://www.google.com/s2/favicons?domain=https://onion.link", 'logo': "https://onion.link/images/OC.png", 'description': "Enabling search and global access to Tor's onionsites.", } } # Default options opts = { "api_key": "", "cse_id": "013611106330597893267:tfgl3wxdtbp", 'fetchlinks': True, 'fullnames': True } # Option descriptions optdescs = { "api_key": "Google API Key for Onion.link search.", "cse_id": "Google Custom Search Engine ID.", 'fetchlinks': "Fetch the darknet pages (via TOR, if enabled) to verify they mention your target.", 'fullnames': "Search for human names?" } # Target results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["INTERNET_NAME", "DOMAIN_NAME"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["DARKNET_MENTION_URL", "DARKNET_MENTION_CONTENT", "RAW_RIR_DATA"] def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if self.errorState: return if not self.opts['fullnames'] and eventName == 'HUMAN_NAME': return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == "": self.error("You enabled sfp_onioncity but did not set a Google API key!") self.errorState = True return if eventData in self.results: self.debug(f"Already did a search for {eventData}, skipping.") return self.results[eventData] = True # Sites hosted on the domain res = self.sf.googleIterate( searchString="+site:onion.link " + eventData, opts={ "timeout": self.opts["_fetchtimeout"], "useragent": self.opts["_useragent"], "api_key": self.opts["api_key"], "cse_id": self.opts["cse_id"], }, ) if res is None: # Failed to talk to the bing API or no results returned return urls = res["urls"] new_links = list(set(urls) - set(self.results.keys())) # Add new links to results for link in new_links: self.results[link] = True # Submit the Google results for analysis googlesearch_url = res["webSearchUrl"] response = self.sf.fetchUrl( googlesearch_url, timeout=self.opts["_fetchtimeout"], useragent=self.opts["_useragent"], ) if response['code'] in ["200", "201", "202"]: evt = SpiderFootEvent( "RAW_RIR_DATA", response["content"], self.__name__, event ) self.notifyListeners(evt) else: self.error("Failed to fetch Google web search URL") # Check if we've been asked to stop if self.checkForStop(): return darknet_links = [ link for link in new_links if self.sf.urlFQDN(link).endswith(".onion.link") ] for link in darknet_links: self.debug("Found a darknet mention: " + link) torlink = link.replace(".onion.link", ".onion") if self.opts['fetchlinks']: res = self.sf.fetchUrl(torlink, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], verify=False) if res['content'] is None: self.debug("Ignoring " + link + " as no data returned") continue # Sometimes onion city search results false positives if re.search(r"[^a-zA-Z\-\_0-9]" + re.escape(eventData) + r"[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None: self.debug("Ignoring " + link + " as no mention of " + eventData) continue evt = SpiderFootEvent("DARKNET_MENTION_URL", torlink, self.__name__, event) self.notifyListeners(evt) try: startIndex = res['content'].index(eventData) - 120 endIndex = startIndex + len(eventData) + 240 except Exception: self.debug("String not found in content.") continue data = res['content'][startIndex:endIndex] evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...", self.__name__, evt) self.notifyListeners(evt) else: evt = SpiderFootEvent("DARKNET_MENTION_URL", torlink, self.__name__, event) self.notifyListeners(evt) # End of sfp_onioncity class ================================================ FILE: modules/sfp_onionsearchengine.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_onionsearchengine # Purpose: Searches the Tor search engine onionsearchengine.com for content # related to the domain in question. # # Author: Steve Micallef # # Created: 27/10/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import re import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_onionsearchengine(SpiderFootPlugin): meta = { 'name': "Onionsearchengine.com", 'summary': "Search Tor onionsearchengine.com for mentions of the target domain.", 'flags': ["tor"], 'useCases': ["Footprint", "Investigate"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://as.onionsearchengine.com", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://helpdesk.onionsearchengine.com/?v=knowledgebase", "https://onionsearchengine.com/add_url.php" ], 'favIcon': "https://as.onionsearchengine.com/images/onionsearchengine.jpg", 'logo': "https://as.onionsearchengine.com/images/onionsearchengine.jpg", 'description': "No cookies, no javascript, no trace. We protect your privacy.\n" "Onion search engine is search engine with ability to find content on tor network / deepweb / darkweb.", } } # Default options opts = { 'timeout': 10, 'max_pages': 20, 'fetchlinks': True, 'blacklist': ['.*://relate.*'], 'fullnames': True } # Option descriptions optdescs = { 'timeout': "Query timeout, in seconds.", 'max_pages': "Maximum number of pages of results to fetch.", 'fetchlinks': "Fetch the darknet pages (via TOR, if enabled) to verify they mention your target.", 'blacklist': "Exclude results from sites matching these patterns.", 'fullnames': "Search for human names?" } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["DOMAIN_NAME", "HUMAN_NAME", "EMAILADDR"] # What events this module produces # This is to support the end user in selecting modules based on events # produced. def producedEvents(self): return ["DARKNET_MENTION_URL", "DARKNET_MENTION_CONTENT"] def handleEvent(self, event): eventName = event.eventType eventData = event.data if not self.opts['fullnames'] and eventName == 'HUMAN_NAME': return if eventData in self.results: self.debug("Already did a search for " + eventData + ", skipping.") return self.results[eventData] = True keepGoing = True page = 1 while keepGoing and page <= int(self.opts['max_pages']): # Check if we've been asked to stop if self.checkForStop(): return params = { 'search': '"' + eventData.encode('raw_unicode_escape').decode("ascii", errors='replace') + '"', 'submit': 'Search', 'page': str(page) } # Sites hosted on the domain data = self.sf.fetchUrl('https://onionsearchengine.com/search.php?' + urllib.parse.urlencode(params), useragent=self.opts['_useragent'], timeout=self.opts['timeout']) if data is None or not data.get('content'): self.info("No results returned from onionsearchengine.com.") return page += 1 if "url.php?u=" not in data['content']: # Work around some kind of bug in the site if "you didn't submit a keyword" in data['content']: continue return if "forward >" not in data['content']: keepGoing = False links = re.findall(r"url\.php\?u=(.[^\"\']+)[\"\']", data['content'], re.IGNORECASE | re.DOTALL) for link in links: if self.checkForStop(): return if link in self.results: continue self.results[link] = True blacklist = False for r in self.opts['blacklist']: if re.match(r, link, re.IGNORECASE): self.debug("Skipping " + link + " as it matches blacklist " + r) blacklist = True if blacklist: continue self.debug("Found a darknet mention: " + link) if not self.sf.urlFQDN(link).endswith(".onion"): continue if not self.opts['fetchlinks']: evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event) self.notifyListeners(evt) continue res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'], verify=False) if res['content'] is None: self.debug("Ignoring " + link + " as no data returned") continue if eventData not in res['content']: self.debug("Ignoring " + link + " as no mention of " + eventData) continue evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event) self.notifyListeners(evt) try: startIndex = res['content'].index(eventData) - 120 endIndex = startIndex + len(eventData) + 240 except Exception: self.debug('String "' + eventData + '" not found in content.') continue data = res['content'][startIndex:endIndex] evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...", self.__name__, evt) self.notifyListeners(evt) # End of sfp_onionsearchengine class ================================================ FILE: modules/sfp_onyphe.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_onyphe # Purpose: SpiderFoot plug-in to check if the IP is included on Onyphe # data (threat list, geo-location, pastries, vulnerabilities) # # Author: Filip Aleksić # # Created: 2020-08-21 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import json import time from datetime import datetime from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_onyphe(SpiderFootPlugin): meta = { "name": "Onyphe", "summary": "Check Onyphe data (threat list, geo-location, pastries, vulnerabilities) about a given IP.", 'flags': ["apikey"], "useCases": ["Footprint", "Passive", "Investigate"], "categories": ["Reputation Systems"], "dataSource": { "website": "https://www.onyphe.io", "model": "FREE_AUTH_LIMITED", "references": ["https://www.onyphe.io/documentation/api"], "apiKeyInstructions": [ "Visit https://www.onyphe.io/login/#register", "Register a free account", "You should receive your API key on your email, or you can get it by yourself following next steps", "Go to your account settings https://www.onyphe.io/auth/account", "The API key is listed under 'API Key'", ], "favIcon": "https://www.onyphe.io/favicon.ico", "logo": "https://www.onyphe.io/img/logo-solo.png", "description": "ONYPHE is a search engine for open-source " "and cyber threat intelligence data collected by crawling " "various sources available on the Internet or by listening " "to Internet background noise. They make this data available " "through API that we use. We check their data to see following " "information about the IP: geo-location, does it have some " "vulnerabilities, is it on some pastries (PasteBin) and " "is it on their threat list", }, } opts = { "api_key": "", "paid_plan": False, "max_page": 10, "verify": True, "age_limit_days": 30, "cohostsamedomain": False, "maxcohost": 100, } optdescs = { "api_key": "Onyphe access token.", "paid_plan": "Are you using paid plan? Paid plan has pagination enabled", "max_page": "Maximum number of pages to iterate through. Onyphe has a maximum of 1000 pages (10,000 results). Only matters for paid plans", "verify": "Verify identified domains still resolve to the associated specified IP address.", "age_limit_days": "Ignore any records older than this many days. 0 = unlimited.", "maxcohost": "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.", "cohostsamedomain": "Treat co-hosted sites on the same target domain as co-hosting?", } results = None errorState = False cohostcount = 0 def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["IP_ADDRESS", "IPV6_ADDRESS"] # What events this module produces def producedEvents(self): return [ "GEOINFO", "MALICIOUS_IPADDR", "LEAKSITE_CONTENT", "VULNERABILITY_CVE_CRITICAL", "VULNERABILITY_CVE_HIGH", "VULNERABILITY_CVE_MEDIUM", "VULNERABILITY_CVE_LOW", "VULNERABILITY_GENERAL", "RAW_RIR_DATA", "INTERNET_NAME", "INTERNET_NAME_UNRESOLVED", "PHYSICAL_COORDINATES", ] def query(self, endpoint, ip, page=1): retarr = list() headers = { "Content-Type": "application/json", "Authorization": f"apikey {self.opts['api_key']}", } res = self.sf.fetchUrl( f"https://www.onyphe.io/api/v2/simple/{endpoint}/{ip}?page={page}", timeout=self.opts["_fetchtimeout"], useragent=self.opts["_useragent"], headers=headers, ) if res["code"] == "429": self.error("Reaching rate limit on Onyphe API") self.errorState = True return None if res["code"] == 400: self.error("Invalid request or API key on Onyphe") self.errorState = True return None try: info = json.loads(res["content"]) if "status" in info and info["status"] == "nok": self.error( f"Unexpected error happened while requesting data from Onyphe. Error message: {info.get('text', '')}" ) self.errorState = True return None if "results" not in info or info["results"] == []: self.info(f"No Onyphe {endpoint} data found for {ip}") return None except Exception as e: self.debug(f"{e.__class__} {res['code']} {res['content']}") self.error("Error processing JSON response from Onyphe.") return None # Go through other pages if user has paid plan try: current_page = int(info["page"]) if ( self.opts["paid_plan"] and info.get("page") and int(info.get("max_page")) > current_page ): page = current_page + 1 if page > self.opts["max_page"]: self.error( "Maximum number of pages from options for Onyphe reached." ) return [info] retarr.append(info) response = self.query(endpoint, ip, page) if response: retarr.extend(response) else: retarr.append(info) except ValueError: self.error( f"Unexpected value for page in response from Onyphe, url: https://www.onyphe.io/api/v2/simple/{endpoint}/{ip}?page={page}" ) self.errorState = True return None return retarr def emitLocationEvent(self, location, eventData, event): if location is None: return self.info(f"Found location for {eventData}: {location}") evt = SpiderFootEvent("PHYSICAL_COORDINATES", location, self.__name__, event) self.notifyListeners(evt) def emitDomainData(self, response, eventData, event): domains = set() if response.get("domain") is not None and isinstance( response['domain'], list ): for dom in response['domain']: domains.add(dom) if response.get("subdomains") is not None and isinstance( response["subdomains"], list ): for subDomain in response["subdomains"]: domains.add(subDomain) for domain in domains: if self.getTarget().matches(domain): if self.opts['verify']: if self.sf.resolveHost(domain) or self.sf.resolveHost6(domain): evt = SpiderFootEvent('INTERNET_NAME', domain, self.__name__, event) else: evt = SpiderFootEvent('INTERNET_NAME_UNRESOLVED', domain, self.__name__, event) self.notifyListeners(evt) if self.sf.isDomain(domain, self.opts['_internettlds']): evt = SpiderFootEvent('DOMAIN_NAME', domain, self.__name__, event) self.notifyListeners(evt) continue if self.cohostcount < self.opts['maxcohost']: if self.opts["verify"] and not self.sf.validateIP(domain, eventData): self.debug("Host no longer resolves to our IP.") continue if not self.opts["cohostsamedomain"] and self.getTarget().matches(domain, includeParents=True): self.debug(f"Skipping {domain} because it is on the same domain.") continue evt = SpiderFootEvent("CO_HOSTED_SITE", domain, self.__name__, event) self.notifyListeners(evt) self.cohostcount += 1 def isFreshEnough(self, result): limit = self.opts["age_limit_days"] if limit <= 0: return True timestamp = result.get("@timestamp") if timestamp is None: self.debug("Record doesn't have timestamp defined") return False last_dt = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ") last_ts = int(time.mktime(last_dt.timetuple())) age_limit_ts = int(time.time()) - (86400 * limit) if last_ts < age_limit_ts: self.debug("Record found but too old, skipping.") return False return True # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data sentData = set() if self.errorState: return self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts["api_key"] == "": self.error("You enabled sfp_onyphe, but did not set an API key!") self.errorState = True return if eventData in self.results: self.debug("Skipping " + eventData + " as already mapped.") return self.results[eventData] = True geoLocDataArr = self.query("geoloc", eventData) if geoLocDataArr is not None: evt = SpiderFootEvent( "RAW_RIR_DATA", str(geoLocDataArr), self.__name__, event ) self.notifyListeners(evt) for geoLocData in geoLocDataArr: if self.checkForStop(): return for result in geoLocData["results"]: if not self.isFreshEnough(result): continue location = ", ".join( [ _f for _f in [ result.get("city"), result.get("country"), ] if _f ] ) self.info("Found GeoIP for " + eventData + ": " + location) if location in sentData: self.debug(f"Skipping {location}, already sent") continue sentData.add(location) evt = SpiderFootEvent("GEOINFO", location, self.__name__, event) self.notifyListeners(evt) coordinates = result.get("location") if coordinates is None: continue if coordinates in sentData: self.debug(f"Skipping {coordinates}, already sent") continue sentData.add(coordinates) self.emitLocationEvent(coordinates, eventData, event) self.emitDomainData(result, eventData, event) pastriesDataArr = self.query("pastries", eventData) if pastriesDataArr is not None: evt = SpiderFootEvent( "RAW_RIR_DATA", str(pastriesDataArr), self.__name__, event ) self.notifyListeners(evt) for pastriesData in pastriesDataArr: if self.checkForStop(): return for result in pastriesData["results"]: pastry = result.get("content") if pastry is None: continue if pastry in sentData: self.debug(f"Skipping {pastry}, already sent") continue sentData.add(pastry) if not self.isFreshEnough(result): continue evt = SpiderFootEvent( "LEAKSITE_CONTENT", pastry, self.__name__, event ) self.notifyListeners(evt) threatListDataArr = self.query("threatlist", eventData) if threatListDataArr is not None: evt = SpiderFootEvent( "RAW_RIR_DATA", str(threatListDataArr), self.__name__, event ) self.notifyListeners(evt) for threatListData in threatListDataArr: if self.checkForStop(): return for result in threatListData["results"]: threatList = result.get("threatlist") if threatList is None: continue if threatList in sentData: self.debug(f"Skipping {threatList}, already sent") continue sentData.add(threatList) if not self.isFreshEnough(result): continue evt = SpiderFootEvent( "MALICIOUS_IPADDR", result.get("threatlist"), self.__name__, event, ) self.notifyListeners(evt) vulnerabilityDataArr = self.query("vulnscan", eventData) if vulnerabilityDataArr is not None: evt = SpiderFootEvent( "RAW_RIR_DATA", str(vulnerabilityDataArr), self.__name__, event ) self.notifyListeners(evt) for vulnerabilityData in vulnerabilityDataArr: if self.checkForStop(): return for result in vulnerabilityData["results"]: if not self.isFreshEnough(result): continue cves = result.get("cve") if cves is None: continue for cve in cves: if not cve: continue if cve in sentData: continue sentData.add(cve) etype, cvetext = self.sf.cveInfo(cve) evt = SpiderFootEvent( etype, cvetext, self.__name__, event, ) self.notifyListeners(evt) # End of sfp_onyphe class ================================================ FILE: modules/sfp_openbugbounty.py ================================================ # ------------------------------------------------------------------------------- # Name: sfp_openbugbounty # Purpose: Query the Open Bug Bounty database to see if our target appears. # # Author: Steve Micallef # # Created: 04/10/2015 # Copyright: (c) Steve Micallef # Licence: MIT # ------------------------------------------------------------------------------- import re from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_openbugbounty(SpiderFootPlugin): meta = { 'name': "Open Bug Bounty", 'summary': "Check external vulnerability scanning/reporting service openbugbounty.org to see if the target is listed.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Leaks, Dumps and Breaches"], 'dataSource': { 'website': "https://www.openbugbounty.org/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://www.openbugbounty.org/cert/" ], 'favIcon': "https://www.openbugbounty.org/favicon.ico", 'logo': "https://www.openbugbounty.org/images/design/logo-obbnew.svg", 'description': "Open Bug Bounty is an open, disintermediated, cost-free, and community-driven bug bounty platform " "for coordinated, responsible and ISO 29147 compatible vulnerability disclosure.\n" "The role of Open Bug Bounty is limited to independent verification of the " "submitted vulnerabilities and proper notification of website owners by all available means. " "Once notified, the website owner and the researcher are in direct contact to " "remediate the vulnerability and coordinate its disclosure. " "At this and at any later stages, we never act as an intermediary between " "website owners and security researchers.", } } # Default options opts = { } # Option descriptions optdescs = { } # Be sure to completely clear any class variables in setup() # or you run the risk of data persisting between scan runs. results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() # Clear / reset any other class member variables here # or you risk them persisting between threads. for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ["INTERNET_NAME"] # What events this module produces def producedEvents(self): return ["VULNERABILITY_DISCLOSURE"] # Query XSSposed.org def queryOBB(self, qry): ret = list() base = "https://www.openbugbounty.org" url = "https://www.openbugbounty.org/search/?search=" + qry res = self.sf.fetchUrl(url, timeout=30, useragent=self.opts['_useragent']) if res['content'] is None: self.debug("No content returned from openbugbounty.org") return None try: rx = re.compile(".*
(.*" + qry + ").*?
.*", re.IGNORECASE) for m in rx.findall(str(res['content'])): # Report it if m[1] == qry or m[1].endswith("." + qry): ret.append("From openbugbounty.org: " + base + m[0] + "") except Exception as e: self.error("Error processing response from openbugbounty.org: " + str(e)) return None return ret # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data data = list() self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True obb = self.queryOBB(eventData) if obb: data.extend(obb) for n in data: # Notify other modules of what you've found e = SpiderFootEvent("VULNERABILITY_DISCLOSURE", n, self.__name__, event) self.notifyListeners(e) # End of sfp_openbugbounty class ================================================ FILE: modules/sfp_opencorporates.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_opencorporates # Purpose: SpiderFoot plug-in for retrieving company information from # OpenCorporates. # # Author: # # Created: 2018-10-21 # Copyright: (c) bcoles 2018 # Licence: MIT # ------------------------------------------------------------------------------- import json import urllib from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_opencorporates(SpiderFootPlugin): meta = { 'name': "OpenCorporates", 'summary': "Look up company information from OpenCorporates.", 'flags': ["apikey"], 'useCases': ["Passive", "Footprint", "Investigate"], 'categories': ["Search Engines"], 'dataSource': { 'website': "https://opencorporates.com", 'model': "FREE_NOAUTH_LIMITED", 'references': [ "https://api.opencorporates.com/documentation/API-Reference" ], 'apiKeyInstructions': [ "Visit https://opencorporates.com/api_accounts/new" "Register a new account with an email", "Navigate to https://opencorporates.com/users/account and select 'Get Account'", "Select the plan required", "Navigate to https://opencorporates.com/users/account", "The API key is listed under 'API Account'", ], 'favIcon': "https://opencorporates.com/assets/favicons/favicon.png", 'logo': "https://opencorporates.com/contents/ui/theme/img/oc-logo.svg", 'description': "The largest open database of companies in the world.\n" "As the largest, open database of companies in the world, " "our business is making high-quality, official company data openly available. " "Data that can be trusted, accessed, analysed and interrogated when and how it’s needed.", } } opts = { 'confidence': 100, 'api_key': '' } optdescs = { 'confidence': "Confidence that the search result objects are correct (numeric value between 0 and 100).", 'api_key': 'OpenCorporates.com API key. Without this you will be limited to 50 look-ups per day.' } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return ["COMPANY_NAME"] def producedEvents(self): return ["COMPANY_NAME", "PHYSICAL_ADDRESS", "RAW_RIR_DATA"] def searchCompany(self, qry): """Search for company name Args: qry (str): company name Returns: str """ version = '0.4' apiparam = "" if self.opts['api_key']: apiparam = "&api_token=" + self.opts['api_key'] params = urllib.parse.urlencode({ 'q': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), 'format': 'json', 'order': 'score', 'confidence': self.opts['confidence'] }) res = self.sf.fetchUrl( f"https://api.opencorporates.com/v{version}/companies/search?{params}{apiparam}", timeout=60, # High timeouts as they can sometimes take a while useragent=self.opts['_useragent'] ) if res['code'] == "401": self.error("Invalid OpenCorporates API key.") return None if res['code'] == "403": self.error("You are being rate-limited by OpenCorporates.") return None try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None if 'results' not in data: return None return data['results'] def retrieveCompanyDetails(self, jurisdiction_code, company_number): url = f"https://api.opencorporates.com/companies/{jurisdiction_code}/{company_number}" if self.opts['api_key']: url += "?api_token=" + self.opts['api_key'] res = self.sf.fetchUrl( url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'] ) if res['code'] == "401": self.error("Invalid OpenCorporates API key.") return None if res['code'] == "403": self.error("You are being rate-limited by OpenCorporates.") return None try: data = json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None if 'results' not in data: return None return data['results'] # Extract company address, previous names, and officer names def extractCompanyDetails(self, company, sevt): # Extract registered address location = company.get('registered_address_in_full') if location: if len(location) < 3 or len(location) > 100: self.debug("Skipping likely invalid location.") else: if company.get('registered_address'): country = company.get('registered_address').get('country') if country: if not location.endswith(country): location += ", " + country location = location.replace("\n", ',') self.info("Found company address: " + location) e = SpiderFootEvent("PHYSICAL_ADDRESS", location, self.__name__, sevt) self.notifyListeners(e) # Extract previous company names previous_names = company.get('previous_names') if previous_names: for previous_name in previous_names: p = previous_name.get('company_name') if p: self.info("Found previous company name: " + p) e = SpiderFootEvent("COMPANY_NAME", p, self.__name__, sevt) self.notifyListeners(e) # Extract officer names officers = company.get('officers') if officers: for officer in officers: n = officer.get('name') if n: self.info("Found company officer: " + n) e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + n, self.__name__, sevt) self.notifyListeners(e) def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True self.debug(f"Received event, {eventName}, from {srcModuleName}") if self.opts['api_key'] == '': self.error(f"Warning: You enabled {self.__class__.__name__} but did not set an API key! Queries will be limited to 50 per day and 200 per month.") res = self.searchCompany(f"{eventData}*") if res is None: self.debug("Found no results for " + eventData) return companies = res.get('companies') if not companies: self.debug("Found no results for " + eventData) return for c in companies: company = c.get('company') if not company: continue # Check for match if eventData.lower() != company.get('name').lower(): continue # Extract company details from search results self.extractCompanyDetails(company, event) # Retrieve further details jurisdiction_code = company.get('jurisdiction_code') company_number = company.get('company_number') if not company_number or not jurisdiction_code: continue res = self.retrieveCompanyDetails(jurisdiction_code, company_number) if not res: continue c = res.get('company') if not c: continue self.extractCompanyDetails(c, event) # End of sfp_opencorporates class ================================================ FILE: modules/sfp_opendns.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_opendns # Purpose: SpiderFoot plug-in for looking up whether hosts are blocked by # OpenDNS. # # Author: Steve Micallef # # Created: 30/05/2018 # Copyright: (c) Steve Micallef 2018 # Licence: MIT # ------------------------------------------------------------------------------- import dns.resolver from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_opendns(SpiderFootPlugin): meta = { 'name': "OpenDNS", 'summary': "Check if a host would be blocked by OpenDNS.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://www.opendns.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://www.opendns.com/setupguide/?url=familyshield", "https://support.opendns.com/hc/en-us/categories/204012807-OpenDNS-Knowledge-Base", "https://support.opendns.com/hc/en-us/categories/204012907-OpenDNS-Device-Configuration" ], 'favIcon': "https://www.google.com/s2/favicons?domain=https://www.opendns.com/", 'logo': "https://d15ni2z53ptwz9.cloudfront.net/opendns-www/img/logo-opendns.png", 'description': "Cisco Umbrella provides protection against threats on the internet such as " "malware, phishing, and ransomware.\n" "OpenDNS is a suite of consumer products aimed at making your internet faster, safer, and more reliable. " "FamilyShield is the single easiest way to protect your kids online, block adult websites, " "and protect your family from phishing and malware.", } } opts = { } optdescs = { } results = None checks = { "146.112.61.105": "OpenDNS - Botnet", "146.112.61.106": "OpenDNS - Adult", "146.112.61.107": "OpenDNS - Malware", "146.112.61.108": "OpenDNS - Phishing", "146.112.61.109": "OpenDNS - Blocked", "146.112.61.110": "OpenDNS - Malware", } def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "CO_HOSTED_SITE" ] def producedEvents(self): return [ "BLACKLISTED_INTERNET_NAME", "BLACKLISTED_AFFILIATE_INTERNET_NAME", "BLACKLISTED_COHOST", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_COHOST", ] def queryAddr(self, qaddr): if not qaddr: return None res = dns.resolver.Resolver() res.nameservers = ["208.67.222.123", "208.67.220.123"] try: return res.resolve(qaddr) except Exception: self.debug(f"Unable to resolve {qaddr}") return None def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: return self.results[eventData] = True if eventName == "INTERNET_NAME": malicious_type = "MALICIOUS_INTERNET_NAME" blacklist_type = "BLACKLISTED_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": malicious_type = "MALICIOUS_AFFILIATE_INTERNET_NAME" blacklist_type = "BLACKLISTED_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": malicious_type = "MALICIOUS_COHOST" blacklist_type = "BLACKLISTED_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") return res = self.queryAddr(eventData) if not res: return self.debug(f"{eventData} found in OpenDNS Blocklist: {res}") for result in res: k = str(result) if k not in self.checks: continue evt = SpiderFootEvent(blacklist_type, f"{self.checks[k]} [{eventData}]", self.__name__, event) self.notifyListeners(evt) if k in ['146.112.61.105', '146.112.61.107', '146.112.61.108', '146.112.61.110']: evt = SpiderFootEvent(malicious_type, f"{self.checks[k]} [{eventData}]", self.__name__, event) self.notifyListeners(evt) # End of sfp_opendns class ================================================ FILE: modules/sfp_opennic.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_opnenic # Purpose: SpiderFoot plug-in for resolving host names on the OpenNIC # alternative DNS system. # # Author: # # Created: 2021-10-16 # Copyright: (c) bcoles 2021 # Licence: MIT # ------------------------------------------------------------------------------- import dns.resolver from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_opennic(SpiderFootPlugin): meta = { 'name': "OpenNIC DNS", 'summary': "Resolves host names in the OpenNIC alternative DNS system.", 'flags': [], 'useCases': ["Investigate", "Footprint", "Passive"], 'categories': ["DNS"], 'dataSource': { 'website': "https://www.opennic.org/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://wiki.opennic.org/", "https://servers.opennic.org", ], 'description': "An organization of hobbyists who run an alternative DNS network, " "also provides access to domains not administered by ICANN." } } opts = { 'checkaffiliates': True, } optdescs = { 'checkaffiliates': "Apply checks to affiliates?", } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "INTERNET_NAME", "INTERNET_NAME_UNRESOLVED", "AFFILIATE_INTERNET_NAME", "AFFILIATE_INTERNET_NAME_UNRESOLVED", ] def producedEvents(self): return [ "IP_ADDRESS", "IPV6_ADDRESS", "AFFILIATE_IPADDR", "AFFILIATE_IPV6_ADDRESS", ] def queryOpenNIC(self, qaddr): res = dns.resolver.Resolver() # https://servers.opennicproject.org/ res.nameservers = [ "192.3.165.37", "35.211.96.150", "51.89.88.77", "94.247.43.254", "138.197.140.189" ] try: return res.resolve(qaddr) except Exception: self.debug(f"Unable to resolve {qaddr}") return None def tlds(self): """Valid OpenNIC top-level domains. Returns: list: OpenNIC TLDs (and peer TLDs). """ return [ 'bbs', 'chan', 'cyb', 'dyn', 'epic', 'free', 'geek', 'glue', 'gopher', 'indy', 'libre', 'neo', 'null', 'o', 'oss', 'oz', 'parody', 'pirate', # Peers 'bazar', 'bit', 'coin', 'emc', 'fur', 'ku', 'lib', 'te', 'ti', 'uu', ] def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: return self.results[eventData] = True if eventData.split('.')[-1] not in self.tlds(): return affiliate = False if "AFFILIATE" in eventName: if not self.opts.get('checkaffiliates', False): return affiliate = True addrs = self.sf.normalizeDNS(self.queryOpenNIC(eventData)) if not addrs: return self.debug(f"OpenNIC resolved {eventData} to addresses: {addrs}") for addr in set(addrs): if self.sf.validIP(addr): if affiliate and not self.getTarget().matches(addr, includeParents=True): evt = SpiderFootEvent("AFFILIATE_IPADDR", addr, self.__name__, event) else: evt = SpiderFootEvent("IP_ADDRESS", addr, self.__name__, event) self.notifyListeners(evt) elif self.sf.validIP6(addr): if affiliate and not self.getTarget().matches(addr, includeParents=True): evt = SpiderFootEvent("AFFILIATE_IPV6_ADDRESS", addr, self.__name__, event) else: evt = SpiderFootEvent("IPV6_ADDRESS", addr, self.__name__, event) self.notifyListeners(evt) else: continue # End of sfp_opennic class ================================================ FILE: modules/sfp_openphish.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_openphish # Purpose: Check if a host/domain is malicious according to OpenPhish.com. # # Author: steve@binarypool.com # # Created: 28/06/2018 # Copyright: (c) Steve Micallef, 2018 # Licence: MIT # ------------------------------------------------------------------------------- from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_openphish(SpiderFootPlugin): meta = { 'name': "OpenPhish", 'summary': "Check if a host/domain is malicious according to OpenPhish.com.", 'flags': [], 'useCases': ["Investigate", "Passive"], 'categories': ["Reputation Systems"], 'dataSource': { 'website': "https://openphish.com/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://openphish.com/faq.html", "https://openphish.com/feed.txt" ], 'favIcon': "", 'logo': "https://openphish.com/static/openphish_logo2.png", 'description': "Timely. Accurate. Relevant Threat Intelligence.\n" "OpenPhish is a fully automated self-contained platform for phishing intelligence. " "It identifies phishing sites and performs intelligence analysis ""in real time " "without human intervention and without using any external resources, such as blacklists.", } } opts = { 'checkaffiliates': True, 'checkcohosts': True, 'cacheperiod': 18 } optdescs = { 'checkaffiliates': "Apply checks to affiliates?", 'checkcohosts': "Apply checks to sites found to be co-hosted on the target's IP?", 'cacheperiod': "Hours to cache list data before re-fetching." } results = None errorState = False def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() self.errorState = False for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] def watchedEvents(self): return [ "INTERNET_NAME", "AFFILIATE_INTERNET_NAME", "CO_HOSTED_SITE", ] def producedEvents(self): return [ "BLACKLISTED_INTERNET_NAME", "BLACKLISTED_AFFILIATE_INTERNET_NAME", "BLACKLISTED_COHOST", "MALICIOUS_INTERNET_NAME", "MALICIOUS_AFFILIATE_INTERNET_NAME", "MALICIOUS_COHOST", ] def queryBlacklist(self, target): blacklist = self.retrieveBlacklist() if not blacklist: return False if target.lower() in blacklist: self.debug(f"Host name {target} found in OpenPhish blacklist.") return True return False def retrieveBlacklist(self): blacklist = self.sf.cacheGet('openphish', 24) if blacklist is not None: return self.parseBlacklist(blacklist) res = self.sf.fetchUrl( "https://www.openphish.com/feed.txt", timeout=10, useragent=self.opts['_useragent'], ) if res['code'] != "200": self.error(f"Unexpected HTTP response code {res['code']} from OpenPhish.") self.errorState = True return None if res['content'] is None: self.error("Received no content from OpenPhish") self.errorState = True return None self.sf.cachePut("openphish", res['content']) return self.parseBlacklist(res['content']) def parseBlacklist(self, blacklist): """Parse plaintext blacklist Args: blacklist (str): plaintext blacklist from OpenPhish Returns: list: list of blacklisted host names """ hosts = list() if not blacklist: return hosts for line in blacklist.split('\n'): if not line: continue if not line.startswith('http'): continue # Note: URL parsing and validation with sf.validHost() is too slow to use here url = line.strip().lower() if len(url.split("/")) < 3: continue host = url.split("/")[2] if not host: continue if "." not in host: continue hosts.append(host) return hosts def handleEvent(self, event): eventName = event.eventType eventData = event.data self.debug(f"Received event, {eventName}, from {event.module}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return if self.errorState: return self.results[eventData] = True if eventName == "INTERNET_NAME": malicious_type = "MALICIOUS_INTERNET_NAME" blacklist_type = "BLACKLISTED_INTERNET_NAME" elif eventName == "AFFILIATE_INTERNET_NAME": if not self.opts.get('checkaffiliates', False): return malicious_type = "MALICIOUS_AFFILIATE_INTERNET_NAME" blacklist_type = "BLACKLISTED_AFFILIATE_INTERNET_NAME" elif eventName == "CO_HOSTED_SITE": if not self.opts.get('checkcohosts', False): return malicious_type = "MALICIOUS_COHOST" blacklist_type = "BLACKLISTED_COHOST" else: self.debug(f"Unexpected event type {eventName}, skipping") return self.debug(f"Checking maliciousness of {eventData} ({eventName}) with OpenPhish") if not self.queryBlacklist(eventData): return url = "https://www.openphish.com/feed.txt" text = f"OpenPhish [{eventData}]\n{url}" evt = SpiderFootEvent(malicious_type, text, self.__name__, event) self.notifyListeners(evt) evt = SpiderFootEvent(blacklist_type, text, self.__name__, event) self.notifyListeners(evt) # End of sfp_openphish class ================================================ FILE: modules/sfp_openstreetmap.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_openstreetmap # Purpose: SpiderFoot plug-in to retrieve latitude/longitude coordinates # for physical addresses from OpenStreetMap API. # # Author: # # Created: 2018-10-27 # Copyright: (c) bcoles 2018 # Licence: MIT # ------------------------------------------------------------------------------- import json import re import time import urllib.error import urllib.parse import urllib.request from spiderfoot import SpiderFootEvent, SpiderFootPlugin class sfp_openstreetmap(SpiderFootPlugin): meta = { 'name': "OpenStreetMap", 'summary': "Retrieves latitude/longitude coordinates for physical addresses from OpenStreetMap API.", 'flags': [], 'useCases': ["Footprint", "Investigate", "Passive"], 'categories': ["Real World"], 'dataSource': { 'website': "https://www.openstreetmap.org/", 'model': "FREE_NOAUTH_UNLIMITED", 'references': [ "https://wiki.openstreetmap.org/wiki/API", "https://wiki.openstreetmap.org/wiki/API_v0.6" ], 'favIcon': "https://www.openstreetmap.org/assets/osm_logo-b7061f13a03615f787a7e0e56a0db5252eb2a217ab063183e78526a8cc10989b.svg", 'logo': "https://www.openstreetmap.org/assets/osm_logo-b7061f13a03615f787a7e0e56a0db5252eb2a217ab063183e78526a8cc10989b.svg", 'description': "OpenStreetMap powers map data on thousands of web sites, mobile apps, and hardware devices.", } } opts = { } optdescs = { } results = None def setup(self, sfc, userOpts=dict()): self.sf = sfc self.results = self.tempStorage() for opt in list(userOpts.keys()): self.opts[opt] = userOpts[opt] # What events is this module interested in for input def watchedEvents(self): return ['PHYSICAL_ADDRESS'] # What events this module produces def producedEvents(self): return ['PHYSICAL_COORDINATES'] # Search for address # https://operations.osmfoundation.org/policies/nominatim/ def query(self, qry): params = { 'q': qry.encode('raw_unicode_escape').decode("ascii", errors='replace'), 'format': 'json', 'polygon': '0', 'addressdetails': '0' } res = self.sf.fetchUrl("https://nominatim.openstreetmap.org/search?" + urllib.parse.urlencode(params), timeout=self.opts['_fetchtimeout'], useragent='SpiderFoot') if res['content'] is None: self.info("No location info found for " + qry) return None try: return json.loads(res['content']) except Exception as e: self.debug(f"Error processing JSON response: {e}") return None # Handle events sent to this module def handleEvent(self, event): eventName = event.eventType srcModuleName = event.module eventData = event.data self.debug(f"Received event, {eventName}, from {srcModuleName}") if eventData in self.results: self.debug(f"Skipping {eventData}, already checked.") return self.results[eventData] = True address = eventData # Skip post office boxes if address.lower().startswith('po box'): self.debug("Skipping PO BOX address") return rx1 = re.compile(r'^(c/o|care of|attn:|attention:)\s+[0-9a-z\s\.]', flags=re.IGNORECASE) # Remove address prefixes for delivery instructions address = re.sub(rx1, r'', address) rx2 = re.compile(r'^(Level|Floor|Suite|Room)\s+[0-9a-z]+,', flags=re.IGNORECASE) # Remove address prefixes known to return no results (floor, level, suite, etc). address = re.sub(rx2, r'', address) # Search for address data = self.query(eventData) # Usage Policy mandates no more than 1 request per second time.sleep(1) if data is None: self.debug("Found no results for " + eventData) return self.info("Found " + str(len(data)) + " matches for " + eventData) for location in data: try: lat = location.get('lat') lon = location.get('lon') except Exception as e: self.debug("Failed to get lat/lon: " + str(e)) continue if not lat or not lon: continue coords = str(lat) + "," + str(lon) self.debug("Found coordinates: " + coords) evt = SpiderFootEvent("PHYSICAL_COORDINATES", coords, self.__name__, event) self.notifyListeners(evt) # End of sfp_openstreetmap class ================================================ FILE: modules/sfp_pageinfo.py ================================================ # -*- coding: utf-8 -*- # ------------------------------------------------------------------------------- # Name: sfp_pageinfo # Purpose: SpiderFoot plug-in for scanning retrieved content by other # modules (such as sfp_spider) and building up information about # the page, such as whether it uses Javascript, has forms, and more. # # Author: Steve Micallef # # Created: 02/05/2012 # Copyright: (c) Steve Micallef 2012 # Licence: MIT # ------------------------------------------------------------------------------- import re from spiderfoot import SpiderFootEvent, SpiderFootPlugin # Indentify pages that use Javascript libs, handle passwords, have forms, # permit file uploads and more to come. regexps = dict({ 'URL_JAVASCRIPT': list(['text/javascript', '
================================================ FILE: spiderfoot/templates/error.tmpl ================================================ <%include file="HEADER.tmpl"/>

${message}



Please go back and try again. If you're pretty sure this is a bug, please contact the support mailing list at support@spiderfoot.net.
<%include file="FOOTER.tmpl"/> ================================================ FILE: spiderfoot/templates/newscan.tmpl ================================================ <%include file="HEADER.tmpl"/>

New Scan

  Your scan target may be one of the following. SpiderFoot will automatically detect the target type based on the format of your input:
Domain Name: e.g. example.com
IPv4 Address: e.g. 1.2.3.4
IPv6 Address: e.g. 2606:4700:4700::1111
Hostname/Sub-domain: e.g. abc.example.com
Subnet: e.g. 1.2.3.0/24
Bitcoin Address: e.g. 1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R
E-mail address: e.g. bob@example.com
Phone Number: e.g. +12345678901 (E.164 format)
Human Name: e.g. "John Smith" (must be in quotes)
Username: e.g. "jsmith2000" (must be in quotes)
Network ASN: e.g. 1234
AllGet anything and everything about the target.

All SpiderFoot modules will be enabled (slow) but every possible piece of information about the target will be obtained and analysed.

FootprintUnderstand what information this target exposes to the Internet.

Gain an understanding about the target's network perimeter, associated identities and other information that is obtained through a lot of web crawling and search engine use.

InvestigateBest for when you suspect the target to be malicious but need more information.

Some basic footprinting will be performed in addition to querying of blacklists and other sources that may have information about your target's maliciousness.

PassiveWhen you don't want the target to even suspect they are being investigated.

As much information will be gathered without touching the target or their affiliates, therefore only modules that do not touch the target will be enabled.
<% modlist = dict() for item in modules: modlist[modules[item]['name']] = item %> % for it in sorted(modlist, key=lambda v: v.upper()): <% item = modlist[it] %> <% keylist = dict((k, v) for k, v in modules[item]['opts'].items() if not k.startswith('_')) %> <% keyicon = "" %> % if len(keylist) > 0: <% apikeylist = dict((k, v) for k, v in modules[item]['opts'].items() if k.find("api_key") >= 0) %> <% if len(apikeylist) > 0: keyicon = "  " %> % endif % if item != "sfp__stor_db" and item != "sfp__stor_stdout": % endif % endfor <% count = 0 %> % for item in sorted(types): % if count % 2 == 0: % endif % if not count % 2 == 0: % endif <% count = count + 1 %> % endfor
<%include file="FOOTER.tmpl"/> ================================================ FILE: spiderfoot/templates/opts.tmpl ================================================ <%include file="HEADER.tmpl"/>

Settings

% if updated:

Success!

Settings updated. These will take effect the next time you run a scan.
% endif


  • Global
    • % for mod in sorted(opts['__modules__'].keys()): <% keylist = dict((k, v) for k, v in opts['__modules__'][mod]['opts'].items() if not k.startswith('_')) %> % if len(keylist) > 0: <% keyicon = "" %> <% apikeylist = dict((k, v) for k, v in opts['__modules__'][mod]['opts'].items() if k.find("api_key") >= 0) %> <% if len(apikeylist) > 0: keyicon = "  " %>
  • ${opts['__modules__'][mod]['name']}${keyicon}
    • % endif % endfor

Global Settings

% for opt in sorted(opts.keys()): % if not opt.startswith("__"): % endif % endfor
Option Value
${opts['__globaloptdescs__'].get(opt) or "No description available"} % if type(opts[opt]) is int:
% endif % if type(opts[opt]) is str:
% endif % if type(opts[opt]) is bool: <% if opts[opt] == True: seltrue = "selected" selfalse = "" else: selfalse = "selected" seltrue = "" %> % endif % if type(opts[opt]) is list and type(opts[opt][0]) is str: <% expandedopt = ','.join(opts[opt]) %>
% endif % if type(opts[opt]) is list and type(opts[opt][0]) is int: <% expandedopt = ','.join(str(x) for x in opts[opt]) %>
% endif
% for mod in sorted(opts['__modules__'].keys()): <% summary = opts['__modules__'][mod].get('descr') or 'No summary available.' categories = opts['__modules__'][mod].get('cats') or [] labels = opts['__modules__'][mod].get('labels') or [] meta = opts['__modules__'][mod].get('meta') modopts = opts['__modules__'][mod].get('opts') optdescs = opts['__modules__'][mod].get('optdescs') data_source = meta.get('dataSource') or {} website = data_source.get('website') or '' description = data_source.get('description') or '' apiKeyInstructions = data_source.get('apiKeyInstructions') or '' %> % endfor


<%include file="FOOTER.tmpl"/> ================================================ FILE: spiderfoot/templates/scaninfo.tmpl ================================================ ## -*- coding: utf-8 -*- <%include file="HEADER.tmpl"/> <% if status == "FINISHED": statusy = "alert-success" elif status.startswith("ABORT"): statusy = "alert-warning" elif status == "CREATED" or status == "RUNNING" or status == "STARTED" or status == "STARTING" or status == "INITIALIZING": statusy = "alert-info" elif status == "ERROR-FAILED": statusy = "alert-danger" else: statusy = "alert-info" end %>

${name} ${status} 

<%include file="FOOTER.tmpl"/> ================================================ FILE: spiderfoot/templates/scanlist.tmpl ================================================ <%include file="HEADER.tmpl"/>

Scans  

% if newscan:

Success!

Scan '${newscan}' has successfully been initiated. Reload this page for up-to-date status on the scan.
% endif % if rerunscans:

Success!

Scans have successfully been re-initiated as separate instances. Reload this page for up-to-date status on the scans.
% endif % if stoppedscan and len(errors) == 0:

Scan aborted.

Please allow a minute or two for the scan to cleanly shut itself down.
% endif % if stoppedscan and len(errors) > 0:

Error

Some or all of the scans could not be aborted:
    % for err in errors:
  • ${err}
    • % endfor
% endif
<%include file="FOOTER.tmpl"/> ================================================ FILE: spiderfoot/threadpool.py ================================================ import queue import logging import threading from time import sleep from contextlib import suppress class SpiderFootThreadPool: """ Each thread in the pool is spawned only once, and reused for best performance. Example 1: using map() with SpiderFootThreadPool(self.opts["_maxthreads"]) as pool: # callback("a", "arg1"), callback("b", "arg1"), ... for result in pool.map( callback, ["a", "b", "c", "d"], "arg1", taskName="sfp_testmodule" saveResult=True ): yield result Example 2: using submit() with SpiderFootThreadPool(self.opts["_maxthreads"]) as pool: pool.start() # callback("arg1"), callback("arg2") pool.submit(callback, "arg1", taskName="sfp_testmodule", saveResult=True) pool.submit(callback, "arg2", taskName="sfp_testmodule", saveResult=True) for result in pool.shutdown()["sfp_testmodule"]: yield result """ def __init__(self, threads: int = 100, qsize: int = 10, name: str = '') -> None: """Initialize the SpiderFootThreadPool class. Args: threads (int): Max number of threads qsize (int): Queue size name (str): Name """ self.log = logging.getLogger(f"spiderfoot.{__name__}") self.threads = int(threads) self.qsize = int(qsize) self.pool = [None] * self.threads self.name = str(name) self.inputThread = None self.inputQueues = dict() self.outputQueues = dict() self._stop = False self._lock = threading.Lock() def start(self) -> None: self.log.debug(f'Starting thread pool "{self.name}" with {self.threads:,} threads') for i in range(self.threads): t = ThreadPoolWorker(pool=self, name=f"{self.name}_worker_{i + 1}") t.start() self.pool[i] = t @property def stop(self) -> bool: return self._stop @stop.setter def stop(self, val: bool): assert val in (True, False), "stop must be either True or False" for t in self.pool: with suppress(Exception): t.stop = val self._stop = val def shutdown(self, wait: bool = True) -> dict: """Shut down the pool. Args: wait (bool): Whether to wait for the pool to finish executing Returns: results (dict): (unordered) results in the format: {"taskName": [returnvalue1, returnvalue2, ...]} """ results = dict() self.log.debug(f'Shutting down thread pool "{self.name}" with wait={wait}') if wait: while not self.finished and not self.stop: with self._lock: outputQueues = list(self.outputQueues) for taskName in outputQueues: moduleResults = list(self.results(taskName)) try: results[taskName] += moduleResults except KeyError: results[taskName] = moduleResults sleep(.1) self.stop = True # make sure input queues are empty with self._lock: inputQueues = list(self.inputQueues.values()) for q in inputQueues: with suppress(Exception): while 1: q.get_nowait() with suppress(Exception): q.close() # make sure output queues are empty with self._lock: outputQueues = list(self.outputQueues.items()) for taskName, q in outputQueues: moduleResults = list(self.results(taskName)) try: results[taskName] += moduleResults except KeyError: results[taskName] = moduleResults with suppress(Exception): q.close() return results def submit(self, callback, *args, **kwargs) -> None: """Submit a function call to the pool. The "taskName" and "maxThreads" arguments are optional. Args: callback (function): callback function *args: Passed through to callback **kwargs: Passed through to callback, except for taskName and maxThreads """ taskName = kwargs.get('taskName', 'default') maxThreads = kwargs.pop('maxThreads', 100) # block if this module's thread limit has been reached while self.countQueuedTasks(taskName) >= maxThreads: sleep(.01) continue self.log.debug(f"Submitting function \"{callback.__name__}\" from module \"{taskName}\" to thread pool \"{self.name}\"") self.inputQueue(taskName).put((callback, args, kwargs)) def countQueuedTasks(self, taskName: str) -> int: """For the specified task, returns the number of queued function calls plus the number of functions which are currently executing Args: taskName (str): Name of task Returns: int: the number of queued function calls plus the number of functions which are currently executing """ queuedTasks = 0 with suppress(Exception): queuedTasks += self.inputQueues[taskName].qsize() runningTasks = 0 for t in self.pool: with suppress(Exception): if t.taskName == taskName: runningTasks += 1 return queuedTasks + runningTasks def inputQueue(self, taskName: str = "default") -> str: try: return self.inputQueues[taskName] except KeyError: self.inputQueues[taskName] = queue.Queue(self.qsize) return self.inputQueues[taskName] def outputQueue(self, taskName: str = "default") -> str: try: return self.outputQueues[taskName] except KeyError: self.outputQueues[taskName] = queue.Queue(self.qsize) return self.outputQueues[taskName] def map(self, callback, iterable, *args, **kwargs) -> None: # noqa: A003 """map. Args: callback: the function to thread iterable: each entry will be passed as the first argument to the function args: additional arguments to pass to callback function kwargs: keyword arguments to pass to callback function Yields: return values from completed callback function """ taskName = kwargs.get("taskName", "default") self.inputThread = threading.Thread(target=self.feedQueue, args=(callback, iterable, args, kwargs)) self.inputThread.start() self.start() sleep(.1) yield from self.results(taskName, wait=True) def results(self, taskName: str = "default", wait: bool = False) -> None: while 1: result = False with suppress(Exception): while 1: yield self.outputQueue(taskName).get_nowait() result = True if self.countQueuedTasks(taskName) == 0 or not wait: break if not result: # sleep briefly to save CPU sleep(.1) def feedQueue(self, callback, iterable, args, kwargs) -> None: for i in iterable: if self.stop: break self.submit(callback, i, *args, **kwargs) @property def finished(self): if self.stop: return True finishedThreads = [not t.busy for t in self.pool if t is not None] try: inputThreadAlive = self.inputThread.is_alive() except AttributeError: inputThreadAlive = False inputQueuesEmpty = [q.empty() for q in self.inputQueues.values()] return not inputThreadAlive and all(inputQueuesEmpty) and all(finishedThreads) def __enter__(self): return self def __exit__(self, exception_type, exception_value, traceback): self.shutdown() class ThreadPoolWorker(threading.Thread): def __init__(self, pool, name: str = None) -> None: self.log = logging.getLogger(f"spiderfoot.{__name__}") self.pool = pool self.taskName = "" # which module submitted the callback self.busy = False self.stop = False super().__init__(name=name) def run(self) -> None: # Round-robin through each module's input queue while not self.stop: ran = False with self.pool._lock: inputQueues = list(self.pool.inputQueues.values()) for q in inputQueues: if self.stop: break try: self.busy = True callback, args, kwargs = q.get_nowait() self.taskName = kwargs.pop("taskName", "default") saveResult = kwargs.pop("saveResult", False) try: result = callback(*args, **kwargs) ran = True except Exception: # noqa: B902 import traceback self.log.error(f'Error in thread worker {self.name}: {traceback.format_exc()}') break if saveResult: self.pool.outputQueue(self.taskName).put(result) except queue.Empty: self.busy = False finally: self.busy = False self.taskName = "" # sleep briefly to save CPU if not ran: sleep(.05) ================================================ FILE: test/README.md ================================================ # Tests SpiderFoot includes various test suites. ## Unit and Integration Tests Unit and integration tests require test dependencies to be installed: ``` pip3 install -r test/requirements.txt ``` To run the tests locally, run `./test/run` from the SpiderFoot root directory. These tests are run on all pull requests automatically. Module integration tests are excluded. To run all unit and integration tests, including module integration tests, run: ``` python3 -m pytest -n auto --flake8 --dist loadfile --durations=5 --cov-report html --cov=. . ``` ## Module Integration Tests The module integration tests check module integration with remote third-party data sources. To run the tests: ``` python3 -m pytest -n auto --flake8 --dist loadfile --durations=5 --cov-report html --cov=. test/integration/modules/ ``` ## Acceptance Tests The acceptance tests check that the web intereface is working as intended and that SpiderFooot is operating correctly as a whole. These tests use a headless browser (Firefox by default), and must be run with `./test/acceptance` as current working directory. Requires SpiderFoot web server to be running on default port (`5001`). Requires test dependencies to be installed: ``` pip3 install -r test/acceptance/requirements.txt ``` To run the tests, start the SpiderFoot web interface on the default port: ``` python3 ./sf.py -l 127.0.0.1:5001 ``` Then run robot (override the `BROWSER` variable if necessary): ``` cd test/acceptance robot --variable BROWSER:Firefox --outputdir results scan.robot ``` ================================================ FILE: test/__init__.py ================================================ ================================================ FILE: test/acceptance/requirements.txt ================================================ # To run the tests, start sf web interface: # python3 ./sf.py -l 127.0.0.1:5001 # then: # robot --variable BROWSER:Firefox --outputdir results example.robot -r ../requirements.txt robotframework robotframework-seleniumlibrary ================================================ FILE: test/acceptance/run ================================================ #!/bin/bash # Run acceptance tests # # Must be run with ./test/acceptance as current working directory. # # Requires SpiderFoot web server to be running on default port. # # Requires robotframework: # cd test/acceptance; pip3 install -r requirements.txt time robot --outputdir results scan.robot ================================================ FILE: test/acceptance/scan.robot ================================================ # To run the tests, start sf web interface: # python3 ./sf.py -l 127.0.0.1:5001 # then run robot (override the BROWSER variable if necessary): # robot --variable BROWSER:Firefox --outputdir results scan.robot *** Settings *** Library SeleniumLibrary Test Teardown Close All Browsers *** Variables *** ${BROWSER} Firefox ${HOST} 127.0.0.1 ${PORT} 5001 ${URL} http://${HOST}:${PORT} *** Keywords *** Create a module scan [Arguments] ${scan_name} ${scan_target} ${module_name} Open browser ${URL}/newscan ${BROWSER} Press Keys name:scanname ${scan_name} Press Keys name:scantarget ${scan_target} Click Element id:moduletab Click Element id:btn-deselect-all Scroll To Element id:module_${module_name} Set Focus To Element id:module_${module_name} Click Element id:module_${module_name} Scroll To Element id:btn-run-scan Click Element id:btn-run-scan Create a use case scan [Arguments] ${scan_name} ${scan_target} ${use_case} Open browser ${URL}/newscan ${BROWSER} Press Keys name:scanname ${scan_name} Press Keys name:scantarget ${scan_target} Click Element id:usecase_${use_case} Scroll To Element id:btn-run-scan Click Element id:btn-run-scan Scan info page should render tabs Element Should Be Visible id:btn-status Element Should Be Visible id:btn-browse Element Should Be Visible id:btn-correlations Element Should Be Visible id:btn-graph Element Should Be Visible id:btn-info Element Should Be Visible id:btn-log Scan info Summary tab should render Scan info page should render tabs Element Should Be Visible id:vbarsummary Scan info Browse tab should render Scan info page should render tabs Element Should Be Visible id:btn-refresh Element Should Be Visible id:btn-export Element Should Be Visible id:searchvalue Element Should Be Visible id:searchbutton Scan info Correlations tab should render Scan info page should render tabs Element Should Be Visible id:scansummary-content Scan info Graph tab should render Scan info page should render tabs Element Should Be Visible id:graph-container Scan info Settings tab should render Scan info page should render tabs Page Should Contain Meta Information Page Should Contain Global Settings Scan info Log tab should render Scan info page should render tabs Element Should Be Visible id:btn-refresh Element Should Be Visible id:btn-download-logs Scan list page should render Element Should Be Visible id:scanlist Element Should Be Visible id:btn-rerun Element Should Be Visible id:btn-stop Element Should Be Visible id:btn-refresh Element Should Be Visible id:btn-export Element Should Be Visible id:btn-delete Settings page should render Element Should Be Visible id:savesettingsform Element Should Be Visible id:btn-save-changes Element Should Be Visible id:btn-import-config Element Should Be Visible id:btn-opt-export Element Should Be Visible id:btn-reset-settings New scan page should render Element Should Be Visible id:scanname Element Should Be Visible id:scantarget Element Should Be Visible id:usetab Element Should Be Visible id:typetab Element Should Be Visible id:moduletab Scroll To Element [Arguments] ${locator} ${x}= Get Horizontal Position ${locator} ${y}= Get Vertical Position ${locator} Execute Javascript window.scrollTo(${x} - 100, ${y} - 100) Wait Until Element is visible ${locator} timeout=5s *** Test Cases *** Main navigation pages should render correctly Open browser ${URL} ${BROWSER} Click Element id:nav-link-newscan Wait Until Page Contains New Scan timeout=5s New scan page should render Click Element id:nav-link-scans Wait Until Page Contains Started timeout=5s Scan list page should render Click Element id:nav-link-settings Wait Until Page Contains Settings timeout=5s Settings page should render Scan info page should render correctly Create a module scan test scan info spiderfoot.net sfp_countryname Wait Until Page Contains Browse timeout=5s Wait Until Element Contains scanstatusbadge FINISHED timeout=10s Click Element id:btn-status Scan info Summary tab should render Click Element id:btn-browse Scan info Browse tab should render Click Element id:btn-graph Scan info Graph tab should render Click Element id:btn-info Scan info Settings tab should render Click Element id:btn-log Scan info Log tab should render Scan list page should list scans Create a module scan test scan list spiderfoot.net sfp_countryname Click Element id:nav-link-scans Wait Until Page Contains test scan list timeout=5s A sfp_dnsresolve scan should resolve INTERNET_NAME to IP_ADDRESS Create a module scan dns resolve spiderfoot.net sfp_dnsresolve Wait Until Page Contains Browse timeout=5s Wait Until Element Contains scanstatusbadge FINISHED timeout=10s Click Element id:btn-browse Scan info Browse tab should render Page Should Contain Domain Name Page Should Contain Internet Name Page Should Contain IP Address A sfp_dnsresolve scan should reverse resolve IP_ADDRESS to INTERNET_NAME Create a module scan reverse resolve 1.1.1.1 sfp_dnsresolve Wait Until Page Contains Browse timeout=5s Wait Until Element Contains scanstatusbadge FINISHED timeout=10s Click Element id:btn-browse Scan info Browse tab should render Page Should Contain Domain Name Page Should Contain Internet Name Page Should Contain IP Address #A passive scan with unresolvable target internet name should fail # Create a use case scan shouldnotresolve.doesnotexist.local passive # Wait Until Page Contains Browse timeout=5s # Wait Until Element Contains scanstatusbadge RUNNING timeout=10s # Click Element id:btn-browse # Page Should Contain Domain Name # Page Should Contain Internet Name # Page Should Contain IP Address ================================================ FILE: test/bandit ================================================ #!/bin/sh # Run bandit static analysis checks # # Must be run from SpiderFoot root directory; ie: # ./test/bandit python3 -m bandit *.py spiderfoot/* modules/*.py ================================================ FILE: test/conftest.py ================================================ import pytest from spiderfoot import SpiderFootHelpers @pytest.fixture(autouse=True) def default_options(request): request.cls.default_options = { '_debug': False, '__logging': True, # Logging in general '__outputfilter': None, # Event types to filter from modules' output '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests '_dnsserver': '', # Override the default resolver '_fetchtimeout': 5, # number of seconds before giving up on a fetch '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat', '_internettlds_cache': 72, '_genericusers': ",".join(SpiderFootHelpers.usernamesFromWordlists(['generic-usernames'])), '__database': f"{SpiderFootHelpers.dataPath()}/spiderfoot.test.db", # note: test database file '__modules__': None, # List of modules. Will be set after start-up. '__correlationrules__': None, # List of correlation rules. Will be set after start-up. '_socks1type': '', '_socks2addr': '', '_socks3port': '', '_socks4user': '', '_socks5pwd': '', '__logstdout': False } request.cls.web_default_options = { 'root': '/' } request.cls.cli_default_options = { "cli.debug": False, "cli.silent": False, "cli.color": True, "cli.output": "pretty", "cli.history": True, "cli.history_file": "", "cli.spool": False, "cli.spool_file": "", "cli.ssl_verify": True, "cli.username": "", "cli.password": "", "cli.server_baseurl": "http://127.0.0.1:5001" } ================================================ FILE: test/integration/__init__.py ================================================ ================================================ FILE: test/integration/modules/__init__.py ================================================ ================================================ FILE: test/integration/modules/test_sfp__stor_db.py ================================================ import pytest import unittest from modules.sfp__stor_db import sfp__stor_db from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegration_stor_db(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp__stor_db() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp__stor_stdout.py ================================================ import pytest import unittest from modules.sfp__stor_stdout import sfp__stor_stdout from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegration_stor_stdout(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp__stor_stdout() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_abstractapi.py ================================================ import pytest import unittest from modules.sfp_abstractapi import sfp_abstractapi from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationAbstractapi(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_abstractapi() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_abusech.py ================================================ import pytest import unittest from modules.sfp_abusech import sfp_abusech from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationAbusech(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_abusech() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_abuseipdb.py ================================================ import pytest import unittest from modules.sfp_abuseipdb import sfp_abuseipdb from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationabuseipdb(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_abuseipdb() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_abusix.py ================================================ import pytest import unittest from modules.sfp_abusix import sfp_abusix from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationAbusix(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_abusix() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_accounts.py ================================================ import pytest import unittest from modules.sfp_accounts import sfp_accounts from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationaccounts(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_accounts() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_adblock.py ================================================ import pytest import unittest from modules.sfp_adblock import sfp_adblock from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationAdblock(unittest.TestCase): def test_handleEvent_event_data_provider_javascript_url_matching_ad_filter_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_adblock() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'URL_ADBLOCKED_EXTERNAL' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'https://example.local/lib/ad.js' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_adblock) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'PROVIDER_JAVASCRIPT' event_data = 'https://example.local/lib/ad.js' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_external_url_matching_ad_filter_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_adblock() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'URL_ADBLOCKED_EXTERNAL' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'https://example.local/lib/ad.js' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_adblock) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'LINKED_URL_EXTERNAL' event_data = 'https://example.local/lib/ad.js' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_external_url_not_matching_ad_filter_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_adblock() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_adblock) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'LINKED_URL_EXTERNAL' event_data = 'https://example.local/lib/example.js' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_adguard_dns.py ================================================ import pytest import unittest from modules.sfp_adguard_dns import sfp_adguard_dns from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationAdGuardDns(unittest.TestCase): def test_handleEvent_event_data_adult_internet_name_blocked_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_adguard_dns() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'BLACKLISTED_INTERNET_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'AdGuard - Family Filter [pornhub.com]' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_adguard_dns) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'pornhub.com' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_adguard_dns() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_adguard_dns) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'spiderfoot.net' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_ahmia.py ================================================ import pytest import unittest from modules.sfp_ahmia import sfp_ahmia from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationAhmia(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_ahmia() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_alienvault.py ================================================ import pytest import unittest from modules.sfp_alienvault import sfp_alienvault from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationalienvault(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_alienvault() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_alienvaultiprep.py ================================================ import pytest import unittest from modules.sfp_alienvaultiprep import sfp_alienvaultiprep from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationAlienvaultiprep(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_alienvaultiprep() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_apple_itunes.py ================================================ import pytest import unittest from modules.sfp_apple_itunes import sfp_apple_itunes from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationAppleItunes(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_apple_itunes() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_archiveorg.py ================================================ import pytest import unittest from modules.sfp_archiveorg import sfp_archiveorg from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationarchiveorg(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_archiveorg() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_arin.py ================================================ import pytest import unittest from modules.sfp_arin import sfp_arin from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationarin(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_arin() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_azureblobstorage.py ================================================ import pytest import unittest from modules.sfp_azureblobstorage import sfp_azureblobstorage from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationazureblobstorage(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_azureblobstorage() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_bgpview.py ================================================ import pytest import unittest from modules.sfp_bgpview import sfp_bgpview from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationBgpview(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_bgpview() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_binaryedge.py ================================================ import pytest import unittest from modules.sfp_binaryedge import sfp_binaryedge from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationBinaryedge(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_binaryedge() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_bingsearch.py ================================================ import pytest import unittest from modules.sfp_bingsearch import sfp_bingsearch from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationbingsearch(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_bingsearch() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_bingsharedip.py ================================================ import pytest import unittest from modules.sfp_bingsharedip import sfp_bingsharedip from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationbingsharedip(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_bingsharedip() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_bitcoinabuse.py ================================================ import pytest import unittest from modules.sfp_bitcoinabuse import sfp_bitcoinabuse from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationBitcoinAbuse(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_bitcoinabuse() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_bitcoinwhoswho.py ================================================ import pytest import unittest from modules.sfp_bitcoinwhoswho import sfp_bitcoinwhoswho from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationBitcoinwhoswho(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_bitcoinwhoswho() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_blockchain.py ================================================ import pytest import unittest from modules.sfp_blockchain import sfp_blockchain from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationBlockchain(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_blockchain() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_blocklistde.py ================================================ import pytest import unittest from modules.sfp_blocklistde import sfp_blocklistde from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationBlocklistde(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_blocklistde() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_botscout.py ================================================ import pytest import unittest from modules.sfp_botscout import sfp_botscout from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationBotscout(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_botscout() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_botvrij.py ================================================ import pytest import unittest from modules.sfp_botvrij import sfp_botvrij from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationBotvrij(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_botvrij() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_builtwith.py ================================================ import pytest import unittest from modules.sfp_builtwith import sfp_builtwith from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationBuiltwith(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_builtwith() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_c99.py ================================================ import pytest import unittest from modules.sfp_c99 import sfp_c99 from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationC99(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_c99() module.setup(sf, dict()) target_value = "example target value" target_type = "IP_ADDRESS" target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = "ROOT" event_data = "example data" event_module = "" source_event = "" evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_callername.py ================================================ import pytest import unittest from modules.sfp_callername import sfp_callername from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationcallername(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_callername() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_censys.py ================================================ import pytest import unittest from modules.sfp_censys import sfp_censys from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCensys(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_censys() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_certspotter.py ================================================ import pytest import unittest from modules.sfp_certspotter import sfp_certspotter from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCertspotter(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_certspotter() module.setup(sf, dict()) target_value = 'example target value' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_cinsscore.py ================================================ import pytest import unittest from modules.sfp_cinsscore import sfp_cinsscore from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCinsscore(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_cinsscore() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_circllu.py ================================================ import pytest import unittest from modules.sfp_circllu import sfp_circllu from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCircllu(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_circllu() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_citadel.py ================================================ import pytest import unittest from modules.sfp_citadel import sfp_citadel from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationcitadel(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_citadel() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_cleanbrowsing.py ================================================ import pytest import unittest from modules.sfp_cleanbrowsing import sfp_cleanbrowsing from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationcleanbrowsing(unittest.TestCase): def test_handleEvent_event_data_adult_internet_name_blocked_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_cleanbrowsing() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'BLACKLISTED_INTERNET_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'CleanBrowsing DNS - Adult [pornhub.com]' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_cleanbrowsing) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'pornhub.com' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_cleanbrowsing() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_cleanbrowsing) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'spiderfoot.net' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_cleantalk.py ================================================ import pytest import unittest from modules.sfp_cleantalk import sfp_cleantalk from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationcleantalk(unittest.TestCase): def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_cleantalk() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) module.opts['_fetchtimeout'] = 15 module.optdescs['_fetchtimeout'] = '' module.opts['_useragent'] = '' module.optdescs['_useragent'] = '' def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_cleantalk) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'IP_ADDRESS' event_data = '1.0.0.1' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_clearbit.py ================================================ import pytest import unittest from modules.sfp_clearbit import sfp_clearbit from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationclearbit(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_clearbit() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_cloudflaredns.py ================================================ import pytest import unittest from modules.sfp_cloudflaredns import sfp_cloudflaredns from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCloudflaredns(unittest.TestCase): def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_cloudflaredns() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_cloudflaredns) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'cloudflare.com' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) def test_handleEvent_event_data_adult_internet_name_blocked_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_cloudflaredns() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'BLACKLISTED_INTERNET_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'CloudFlare - Family [pornhub.com]' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_cloudflaredns) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'pornhub.com' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) ================================================ FILE: test/integration/modules/test_sfp_coinblocker.py ================================================ import pytest import unittest from modules.sfp_coinblocker import sfp_coinblocker from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCoinblocker(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_coinblocker() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_commoncrawl.py ================================================ import pytest import unittest from modules.sfp_commoncrawl import sfp_commoncrawl from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationcommoncrawl(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_commoncrawl() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_comodo.py ================================================ import pytest import unittest from modules.sfp_comodo import sfp_comodo from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationcomodo(unittest.TestCase): def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_comodo() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_comodo) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'comodo.com' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_crobat_api.py ================================================ import pytest import unittest from modules.sfp_crobat_api import sfp_crobat_api from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCrobat_api(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_crobat_api() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_crossref.py ================================================ import pytest import unittest from modules.sfp_crossref import sfp_crossref from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCrossref(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_crossref() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_crt.py ================================================ import pytest import unittest from modules.sfp_crt import sfp_crt from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCrt(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_crt() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_crxcavator.py ================================================ import pytest import unittest from modules.sfp_crxcavator import sfp_crxcavator from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCrxcavator(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_crxcavator() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_customfeed.py ================================================ import pytest import unittest from modules.sfp_customfeed import sfp_customfeed from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCustomfeed(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_customfeed() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_cybercrimetracker.py ================================================ import pytest import unittest from modules.sfp_cybercrimetracker import sfp_cybercrimetracker from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCybercrimetracker(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_cybercrimetracker() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_debounce.py ================================================ import pytest import unittest from modules.sfp_debounce import sfp_debounce from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDebounce(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_debounce() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_dehashed.py ================================================ import pytest import unittest from modules.sfp_dehashed import sfp_dehashed from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDehashed(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_dehashed() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_digitaloceanspace.py ================================================ import pytest import unittest from modules.sfp_digitaloceanspace import sfp_digitaloceanspace from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDigitaloceanspace(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_digitaloceanspace() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_dns_for_family.py ================================================ import pytest import unittest from modules.sfp_dns_for_family import sfp_dns_for_family from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDnsForFamily(unittest.TestCase): def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_dns_for_family() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_dns_for_family) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'dns_for_family.com' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) def test_handleEvent_event_data_adult_internet_name_blocked_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_dns_for_family() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'BLACKLISTED_INTERNET_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'DNS for Family [pornhub.com]' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_dns_for_family) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'pornhub.com' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) ================================================ FILE: test/integration/modules/test_sfp_dnsbrute.py ================================================ import pytest import unittest from modules.sfp_dnsbrute import sfp_dnsbrute from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDnsBrute(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_dnsbrute() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_dnscommonsrv.py ================================================ import pytest import unittest from modules.sfp_dnscommonsrv import sfp_dnscommonsrv from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDnsCommonsrv(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_dnscommonsrv() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_dnsdb.py ================================================ import pytest import unittest from modules.sfp_dnsdb import sfp_dnsdb from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationdnsdb(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_dnsdb() module.setup(sf, dict()) target_value = "example target value" target_type = "IP_ADDRESS" target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = "ROOT" event_data = "example data" event_module = "" source_event = "" evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_dnsdumpster.py ================================================ import pytest import unittest from modules.sfp_dnsdumpster import sfp_dnsdumpster from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDnsDumpster(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_dnsdumpster() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_dnsgrep.py ================================================ import pytest import unittest from modules.sfp_dnsgrep import sfp_dnsgrep from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDnsgrep(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_dnsgrep() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_dnsneighbor.py ================================================ import pytest import unittest from modules.sfp_dnsneighbor import sfp_dnsneighbor from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDnsneighbor(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_dnsneighbor() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_dnsraw.py ================================================ import pytest import unittest from modules.sfp_dnsraw import sfp_dnsraw from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationdnsraw(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_dnsraw() module.setup(sf, dict()) target_value = '1.1.1.1' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = '1.1.1.1' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_dnsresolve.py ================================================ import pytest import unittest from modules.sfp_dnsresolve import sfp_dnsresolve from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDnsResolve(unittest.TestCase): def test_enrichTarget_should_return_SpiderFootTarget(self): sf = SpiderFoot(self.default_options) module = sfp_dnsresolve() module.setup(sf, dict()) target_value = '127.0.0.1' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) result = module.enrichTarget(target) self.assertIsInstance(result, SpiderFootTarget) self.assertEqual(result.targetType, target_type) self.assertEqual(result.targetValue, target_value) def test_resolveTargets_should_return_list(self): sf = SpiderFoot(self.default_options) module = sfp_dnsresolve() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) invalid_types = [None, "", list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): resolve_targets = module.resolveTargets(invalid_type, False) self.assertIsInstance(resolve_targets, list) target = SpiderFootTarget("spiderfoot.net", "INTERNET_NAME") resolve_targets = module.resolveTargets(target, False) self.assertIsInstance(resolve_targets, list) self.assertIn('spiderfoot.net', resolve_targets) target = SpiderFootTarget("127.0.0.1", "IP_ADDRESS") resolve_targets = module.resolveTargets(target, False) self.assertIsInstance(resolve_targets, list) self.assertIn('127.0.0.1', resolve_targets) target = SpiderFootTarget("::1", "IPV6_ADDRESS") resolve_targets = module.resolveTargets(target, False) self.assertIsInstance(resolve_targets, list) self.assertIn('::1', resolve_targets) target = SpiderFootTarget("127.0.0.1/32", "NETBLOCK_OWNER") resolve_targets = module.resolveTargets(target, False) self.assertIsInstance(resolve_targets, list) self.assertIn('127.0.0.1', resolve_targets) # note: test fails on MacOSX on CI def test_handleEvent_event_data_ip_address_should_return_internet_name_event(self): sf = SpiderFoot(self.default_options) module = sfp_dnsresolve() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'INTERNET_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "one.one.one.one" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_dnsresolve) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'IP_ADDRESS' event_data = '1.1.1.1' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) # note: test fails on MacOSX on CI def test_handleEvent_event_data_ipv6_address_should_return_internet_name_event(self): sf = SpiderFoot(self.default_options) module = sfp_dnsresolve() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'INTERNET_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "one.one.one.one" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_dnsresolve) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'IPV6_ADDRESS' event_data = '2606:4700:4700::1111' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) # note: test fails on MacOSX on CI def test_handleEvent_event_data_affiliate_ip_address_should_return_affiliate_internet_name_event(self): sf = SpiderFoot(self.default_options) module = sfp_dnsresolve() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'AFFILIATE_INTERNET_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "one.one.one.one" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_dnsresolve) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'AFFILIATE_IPADDR' event_data = '1.1.1.1' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_raw_rir_data_containing_subdomain_should_return_internet_name_event(self): """ Test handleEvent(self, event) """ sf = SpiderFoot(self.default_options) module = sfp_dnsresolve() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'INTERNET_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "www.spiderfoot.net" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_dnsresolve) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'RAW_RIR_DATA' event_data = 'example data www.spiderfoot.net example data' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) ================================================ FILE: test/integration/modules/test_sfp_dnszonexfer.py ================================================ import pytest import unittest from modules.sfp_dnszonexfer import sfp_dnszonexfer from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDnsZoneXfer(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_dnszonexfer() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_dronebl.py ================================================ import pytest import unittest from modules.sfp_dronebl import sfp_dronebl from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDronebl(unittest.TestCase): def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_dronebl() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_dronebl) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'IP_ADDRESS' event_data = '1.0.0.1' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_duckduckgo.py ================================================ import pytest import unittest from modules.sfp_duckduckgo import sfp_duckduckgo from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationDuckduckgo(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_duckduckgo() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_email.py ================================================ import pytest import unittest from modules.sfp_email import sfp_email from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationEmail(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_email() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_emailcrawlr.py ================================================ import pytest import unittest from modules.sfp_emailcrawlr import sfp_emailcrawlr from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationemailcrawlr(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_emailcrawlr() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_emailformat.py ================================================ import pytest import unittest from modules.sfp_emailformat import sfp_emailformat from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationEmailFormat(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_emailformat() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_emailrep.py ================================================ import pytest import unittest from modules.sfp_emailrep import sfp_emailrep from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationEmailrep(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_emailrep() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_emergingthreats.py ================================================ import pytest import unittest from modules.sfp_emergingthreats import sfp_emergingthreats from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationEmergingthreats(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_emergingthreats() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_etherscan.py ================================================ import pytest import unittest from modules.sfp_etherscan import sfp_etherscan from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationEtherscan(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_etherscan() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_filemeta.py ================================================ import pytest import unittest from modules.sfp_filemeta import sfp_filemeta from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationFilemeta(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_filemeta() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_flickr.py ================================================ import pytest import unittest from modules.sfp_flickr import sfp_flickr from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationFlickr(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_flickr() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_focsec.py ================================================ import pytest import unittest from modules.sfp_focsec import sfp_focsec from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationFocsec(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_focsec() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_fortinet.py ================================================ import pytest import unittest from modules.sfp_fortinet import sfp_fortinet from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationFortinet(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_fortinet() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_fraudguard.py ================================================ import pytest import unittest from modules.sfp_fraudguard import sfp_fraudguard from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationFraudguard(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_fraudguard() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_fsecure_riddler.py ================================================ import pytest import unittest from modules.sfp_fsecure_riddler import sfp_fsecure_riddler from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationFsecureRiddler(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_fsecure_riddler() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_fullcontact.py ================================================ import pytest import unittest from modules.sfp_fullcontact import sfp_fullcontact from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationFullContact(unittest.TestCase): def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_fullcontact() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_fullhunt.py ================================================ import pytest import unittest from modules.sfp_fullhunt import sfp_fullhunt from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationFullhunt(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_fullhunt() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_github.py ================================================ import pytest import unittest from modules.sfp_github import sfp_github from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationGithub(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_github() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_gleif.py ================================================ import pytest import unittest from modules.sfp_gleif import sfp_gleif from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationGleif(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_gleif() module.setup(sf, dict()) target_value = '7ZW8QJWVPR4P1J1KQY45' target_type = 'LEI' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_google_tag_manager.py ================================================ import pytest import unittest from modules.sfp_google_tag_manager import sfp_google_tag_manager from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationGoogleTagManager(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_google_tag_manager() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_googlemaps.py ================================================ import pytest import unittest from modules.sfp_googlemaps import sfp_googlemaps from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationGoogleMaps(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_googlemaps() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_googleobjectstorage.py ================================================ import pytest import unittest from modules.sfp_googleobjectstorage import sfp_googleobjectstorage from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationGoogleObjectStorage(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_googleobjectstorage() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_googlesafebrowsing.py ================================================ import pytest import unittest from modules.sfp_googlesafebrowsing import sfp_googlesafebrowsing from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationGoogleSafebrowsing(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_googlesafebrowsing() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_googlesearch.py ================================================ import pytest import unittest from modules.sfp_googlesearch import sfp_googlesearch from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationGoogleSearch(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_googlesearch() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_gravatar.py ================================================ import pytest import unittest from modules.sfp_gravatar import sfp_gravatar from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationGravatar(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_gravatar() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_grayhatwarfare.py ================================================ import pytest import unittest from modules.sfp_grayhatwarfare import sfp_grayhatwarfare from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationGrayhatWarfare(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_grayhatwarfare() module.setup(sf, dict()) target_value = 'example target value' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_greensnow.py ================================================ import pytest import unittest from modules.sfp_greensnow import sfp_greensnow from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationgreensnow(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_greensnow() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_grep_app.py ================================================ import pytest import unittest from modules.sfp_grep_app import sfp_grep_app from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationGrepApp(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_grep_app() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_greynoise.py ================================================ import pytest import unittest from modules.sfp_greynoise import sfp_greynoise from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationGreynoise(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_greynoise() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_h1nobbdde.py ================================================ import pytest import unittest from modules.sfp_h1nobbdde import sfp_h1nobbdde from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationH1nobbdde(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_h1nobbdde() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_hackertarget.py ================================================ import pytest import unittest from modules.sfp_hackertarget import sfp_hackertarget from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationHackertarget(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_hackertarget() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_haveibeenpwned.py ================================================ import pytest import unittest from modules.sfp_haveibeenpwned import sfp_haveibeenpwned from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationHaveibeenpwned(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_haveibeenpwned() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_honeypot.py ================================================ import pytest import unittest from modules.sfp_honeypot import sfp_honeypot from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationHoneypot(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_honeypot() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_hosting.py ================================================ import pytest import unittest from modules.sfp_hosting import sfp_hosting from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationHosting(unittest.TestCase): @unittest.skip("todo") def test_handleEvent_event_data_ip_address_hosted_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_hosting() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'PROVIDER_HOSTING' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "Amazon AWS: http://www.amazon.com/aws/" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_hosting) event_type = 'ROOT' event_data = '3.1.1.1' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) @unittest.skip("todo") def test_handleEvent_event_data_ip_address_not_hosted_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_hosting() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_hosting) event_type = 'ROOT' event_data = '127.0.0.1' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_hostio.py ================================================ import pytest import unittest from modules.sfp_hostio import sfp_hostio from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationHostio(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_hostio() module.setup(sf, dict()) target_value = 'example target value' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_hunter.py ================================================ import pytest import unittest from modules.sfp_hunter import sfp_hunter from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationHunter(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_hunter() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_hybrid_analysis.py ================================================ import pytest import unittest from modules.sfp_hybrid_analysis import sfp_hybrid_analysis from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationHybridAnalysis(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_hybrid_analysis() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_iknowwhatyoudownload.py ================================================ import pytest import unittest from modules.sfp_iknowwhatyoudownload import sfp_iknowwhatyoudownload from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationIknowwhatyoudownload(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_iknowwhatyoudownload() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_intelx.py ================================================ import pytest import unittest from modules.sfp_intelx import sfp_intelx from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationIntelx(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_intelx() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_ipapico.py ================================================ import pytest import unittest from modules.sfp_ipapico import sfp_ipapico from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationIpapico(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_ipapico() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_ipapicom.py ================================================ import pytest import unittest from modules.sfp_ipapicom import sfp_ipapicom from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationIpapicom(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_ipapicom() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_ipinfo.py ================================================ import pytest import unittest from modules.sfp_ipinfo import sfp_ipinfo from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationIpinfo(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_ipinfo() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_ipqualityscore.py ================================================ import pytest import unittest from modules.sfp_ipqualityscore import sfp_ipqualityscore from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationIpqualityscore(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_ipqualityscore() module.setup(sf, dict()) target_value = 'example target value' target_type = 'PHONE_NUMBER' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_ipregistry.py ================================================ import pytest import unittest from modules.sfp_ipregistry import sfp_ipregistry from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationIpregistry(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_ipregistry() module.setup(sf, dict()) target_value = "example target value" target_type = "IP_ADDRESS" target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = "ROOT" event_data = "example data" event_module = "" source_event = "" evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_ipstack.py ================================================ import pytest import unittest from modules.sfp_ipstack import sfp_ipstack from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationIpstack(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_ipstack() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_isc.py ================================================ import pytest import unittest from modules.sfp_isc import sfp_isc from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationIsc(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_isc() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_jsonwhoiscom.py ================================================ import pytest import unittest from modules.sfp_jsonwhoiscom import sfp_jsonwhoiscom from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationJsonwhoiscom(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_jsonwhoiscom() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_junkfiles.py ================================================ import pytest import unittest from modules.sfp_junkfiles import sfp_junkfiles from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationJunkfiles(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_junkfiles() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_keybase.py ================================================ import pytest import unittest from modules.sfp_keybase import sfp_keybase from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationKeybase(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_keybase() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_koodous.py ================================================ import pytest import unittest from modules.sfp_koodous import sfp_koodous from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationKoodous(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_koodous() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_leakix.py ================================================ import pytest import unittest from modules.sfp_leakix import sfp_leakix from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationleakix(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_leakix() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_maltiverse.py ================================================ import pytest import unittest from modules.sfp_maltiverse import sfp_maltiverse from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationMaltiverse(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_maltiverse() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_malwarepatrol.py ================================================ import pytest import unittest from modules.sfp_malwarepatrol import sfp_malwarepatrol from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationMalwarepatrol(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_malwarepatrol() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_metadefender.py ================================================ import pytest import unittest from modules.sfp_metadefender import sfp_metadefender from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationMetadefender(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_metadefender() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_mnemonic.py ================================================ import pytest import unittest from modules.sfp_mnemonic import sfp_mnemonic from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationMnemonic(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_mnemonic() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_multiproxy.py ================================================ import pytest import unittest from modules.sfp_multiproxy import sfp_multiproxy from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationMultiproxy(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_multiproxy() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_myspace.py ================================================ import pytest import unittest from modules.sfp_myspace import sfp_myspace from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationMyspace(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_myspace() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_nameapi.py ================================================ import pytest import unittest from modules.sfp_nameapi import sfp_nameapi from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationNameapi(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_nameapi() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_networksdb.py ================================================ import pytest import unittest from modules.sfp_networksdb import sfp_networksdb from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationNetworksdb(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_networksdb() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_neutrinoapi.py ================================================ import pytest import unittest from modules.sfp_neutrinoapi import sfp_neutrinoapi from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationNeutrinoapi(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_neutrinoapi() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_numverify.py ================================================ import pytest import unittest from modules.sfp_numverify import sfp_numverify from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationNumverify(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_numverify() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_onioncity.py ================================================ import pytest import unittest from modules.sfp_onioncity import sfp_onioncity from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationOnioncity(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_onioncity() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_onionsearchengine.py ================================================ import pytest import unittest from modules.sfp_onionsearchengine import sfp_onionsearchengine from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationOnionsearchengine(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_onionsearchengine() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_onyphe.py ================================================ import pytest import unittest from modules.sfp_onyphe import sfp_onyphe from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationOnyphe(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_onyphe() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_openbugbounty.py ================================================ import pytest import unittest from modules.sfp_openbugbounty import sfp_openbugbounty from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationOpenbugbounty(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_openbugbounty() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_opencorporates.py ================================================ import pytest import unittest from modules.sfp_opencorporates import sfp_opencorporates from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationOpencorporates(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_opencorporates() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_opendns.py ================================================ import pytest import unittest from modules.sfp_opendns import sfp_opendns from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationOpendns(unittest.TestCase): def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_opendns() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_opendns) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'opendns.com' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) def test_handleEvent_event_data_adult_internet_name_blocked_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_opendns() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'BLACKLISTED_INTERNET_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'OpenDNS - Adult [pornhub.com]' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_opendns) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'pornhub.com' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) ================================================ FILE: test/integration/modules/test_sfp_opennic.py ================================================ import pytest import unittest from modules.sfp_opennic import sfp_opennic from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationOpenNic(unittest.TestCase): def test_handleEvent_event_data_internet_name_with_opennic_tld_should_return_ip_address_event(self): sf = SpiderFoot(self.default_options) module = sfp_opennic() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'IP_ADDRESS' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_opennic) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'opennic.glue' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) ================================================ FILE: test/integration/modules/test_sfp_openphish.py ================================================ import pytest import unittest from modules.sfp_openphish import sfp_openphish from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationOpenphish(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_openphish() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_openstreetmap.py ================================================ import pytest import unittest from modules.sfp_openstreetmap import sfp_openstreetmap from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationOpenstreetmap(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_openstreetmap() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_pageinfo.py ================================================ import pytest import unittest from modules.sfp_pageinfo import sfp_pageinfo from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationPageInfo(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_pageinfo() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_pastebin.py ================================================ import pytest import unittest from modules.sfp_pastebin import sfp_pastebin from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationPastebin(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_pastebin() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_pgp.py ================================================ import pytest import unittest from modules.sfp_pgp import sfp_pgp from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationPgp(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_pgp() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_phishstats.py ================================================ import pytest import unittest from modules.sfp_phishstats import sfp_phishstats from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationPhishstats(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_phishstats() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_phishtank.py ================================================ import pytest import unittest from modules.sfp_phishtank import sfp_phishtank from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationPhishtank(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_phishtank() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_portscan_tcp.py ================================================ import pytest import unittest from modules.sfp_portscan_tcp import sfp_portscan_tcp from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationPortscanTcp(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_portscan_tcp() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_projectdiscovery.py ================================================ import pytest import unittest from modules.sfp_projectdiscovery import sfp_projectdiscovery from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationProjectdiscovery(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_projectdiscovery() module.setup(sf, dict()) target_value = "example target value" target_type = "IP_ADDRESS" target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = "ROOT" event_data = "example data" event_module = "" source_event = "" evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_psbdmp.py ================================================ import pytest import unittest from modules.sfp_psbdmp import sfp_psbdmp from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationPsbdmp(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_psbdmp() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_pulsedive.py ================================================ import pytest import unittest from modules.sfp_pulsedive import sfp_pulsedive from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationPulsedive(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_pulsedive() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_punkspider.py ================================================ import pytest import unittest from modules.sfp_punkspider import sfp_punkspider from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationPunkspider(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_punkspider() module.setup(sf, dict()) target_value = 'example target value' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_quad9.py ================================================ import pytest import unittest from modules.sfp_quad9 import sfp_quad9 from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationQuad9(unittest.TestCase): def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_quad9() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_quad9) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'quad9.net' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_reversewhois.py ================================================ import pytest import unittest from modules.sfp_reversewhois import sfp_reversewhois from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationReversewhois(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_reversewhois() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'DOMAIN_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_ripe.py ================================================ import pytest import unittest from modules.sfp_ripe import sfp_ripe from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationRipe(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_ripe() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_riskiq.py ================================================ import pytest import unittest from modules.sfp_riskiq import sfp_riskiq from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationRiskiq(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_riskiq() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_robtex.py ================================================ import pytest import unittest from modules.sfp_robtex import sfp_robtex from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationRobtex(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_robtex() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_s3bucket.py ================================================ import pytest import unittest from modules.sfp_s3bucket import sfp_s3bucket from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationS3bucket(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_s3bucket() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_searchcode.py ================================================ import pytest import unittest from modules.sfp_searchcode import sfp_searchcode from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationCodesearch(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_searchcode() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'DOMAIN_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_securitytrails.py ================================================ import pytest import unittest from modules.sfp_securitytrails import sfp_securitytrails from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSecuritytrails(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_securitytrails() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_seon.py ================================================ import pytest import unittest from modules.sfp_seon import sfp_seon from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationseon(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_seon() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_shodan.py ================================================ import pytest import unittest from modules.sfp_shodan import sfp_shodan from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationShodan(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_shodan() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_similar.py ================================================ import pytest import unittest from modules.sfp_similar import sfp_similar from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSimilar(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_similar() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_skymem.py ================================================ import pytest import unittest from modules.sfp_skymem import sfp_skymem from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSkymem(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_skymem() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_slideshare.py ================================================ import pytest import unittest from modules.sfp_slideshare import sfp_slideshare from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSlideshare(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_slideshare() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_snov.py ================================================ import pytest import unittest from modules.sfp_snov import sfp_snov from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSnov(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_snov() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_sociallinks.py ================================================ import pytest import unittest from modules.sfp_sociallinks import sfp_sociallinks from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationsociallinks(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_sociallinks() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_socialprofiles.py ================================================ import pytest import unittest from modules.sfp_socialprofiles import sfp_socialprofiles from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSocialprofiles(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_socialprofiles() module.setup(sf, dict()) target_value = 'example target value' target_type = 'HUMAN_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_sorbs.py ================================================ import pytest import unittest from modules.sfp_sorbs import sfp_sorbs from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSorbs(unittest.TestCase): def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_sorbs() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_sorbs) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'IP_ADDRESS' event_data = '1.0.0.1' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_spamcop.py ================================================ import pytest import unittest from modules.sfp_spamcop import sfp_spamcop from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSpamcop(unittest.TestCase): def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_spamcop() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_spamcop) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'IP_ADDRESS' event_data = '1.0.0.1' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_spamhaus.py ================================================ import pytest import unittest from modules.sfp_spamhaus import sfp_spamhaus from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSpamhaus(unittest.TestCase): def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_spamhaus() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_spamhaus) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'IP_ADDRESS' event_data = '1.0.0.1' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_spider.py ================================================ import pytest import unittest from modules.sfp_spider import sfp_spider from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSpider(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_spider() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_spur.py ================================================ import pytest import unittest from modules.sfp_spur import sfp_spur from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSpur(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_spur() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_spyonweb.py ================================================ import pytest import unittest from modules.sfp_spyonweb import sfp_spyonweb from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSpyonweb(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_spyonweb() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_sslcert.py ================================================ import pytest import unittest from modules.sfp_sslcert import sfp_sslcert from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSslCert(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_sslcert() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_stackoverflow.py ================================================ import pytest import unittest from modules.sfp_stackoverflow import sfp_stackoverflow from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationStackoverflow(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_stackoverflow() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_stevenblack_hosts.py ================================================ import pytest import unittest from modules.sfp_stevenblack_hosts import sfp_stevenblack_hosts from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationStevenblackHosts(unittest.TestCase): def test_handleEvent_event_data_affiliate_internet_name_matching_ad_server_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_stevenblack_hosts() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) module.opts['_fetchtimeout'] = 15 module.optdescs['_fetchtimeout'] = '' module.opts['_useragent'] = '' module.optdescs['_useragent'] = '' def new_notifyListeners(self, event): expected = 'MALICIOUS_AFFILIATE_INTERNET_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'Steven Black Hosts Blocklist [ads.google.com]\nhttps://raw.githubusercontent.com/StevenBlack/hosts/master/hosts' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_stevenblack_hosts) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'AFFILIATE_INTERNET_NAME' event_data = 'ads.google.com' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_affiliate_internet_name_not_matching_ad_server_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_stevenblack_hosts() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) module.opts['_fetchtimeout'] = 15 module.optdescs['_fetchtimeout'] = '' module.opts['_useragent'] = '' module.optdescs['_useragent'] = '' def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_stevenblack_hosts) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'AFFILIATE_INTERNET_NAME' event_data = 'no.ads.safe.local' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_subdomain_takeover.py ================================================ import pytest import unittest from modules.sfp_subdomain_takeover import sfp_subdomain_takeover from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSubdomainTakeover(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_subdomain_takeover() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_sublist3r.py ================================================ import pytest import unittest from modules.sfp_sublist3r import sfp_sublist3r from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSublist3r(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_sublist3r() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_surbl.py ================================================ import pytest import unittest from modules.sfp_surbl import sfp_surbl from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationSurbl(unittest.TestCase): def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_surbl() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_surbl) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'IP_ADDRESS' event_data = '1.0.0.1' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_talosintel.py ================================================ import pytest import unittest from modules.sfp_talosintel import sfp_talosintel from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationTalosintel(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_talosintel() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_textmagic.py ================================================ import pytest import unittest from modules.sfp_textmagic import sfp_textmagic from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationTextmagic(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_textmagic() module.setup(sf, dict()) target_value = 'example target value' target_type = 'PHONE_NUMBER' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_threatcrowd.py ================================================ import pytest import unittest from modules.sfp_threatcrowd import sfp_threatcrowd from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationThreatcrowd(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_threatcrowd() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_threatfox.py ================================================ import pytest import unittest from modules.sfp_threatfox import sfp_threatfox from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationThreatFox(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_threatfox() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_threatminer.py ================================================ import pytest import unittest from modules.sfp_threatminer import sfp_threatminer from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationThreatminer(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_threatminer() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_tldsearch.py ================================================ import pytest import unittest from modules.sfp_tldsearch import sfp_tldsearch from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationTldsearch(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_tldsearch() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_torch.py ================================================ import pytest import unittest from modules.sfp_torch import sfp_torch from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationTorch(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_torch() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_torexits.py ================================================ import pytest import unittest from modules.sfp_torexits import sfp_torexits from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationTorExits(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_torexits() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_trashpanda.py ================================================ import pytest import unittest from modules.sfp_trashpanda import sfp_trashpanda from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationTrashpanda(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_trashpanda() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_trumail.py ================================================ import pytest import unittest from modules.sfp_trumail import sfp_trumail from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationTrumail(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_trumail() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_twilio.py ================================================ import pytest import unittest from modules.sfp_twilio import sfp_twilio from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationTwilio(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_twilio() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_twitter.py ================================================ import pytest import unittest from modules.sfp_twitter import sfp_twitter from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationTwitter(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_twitter() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_uceprotect.py ================================================ import pytest import unittest from modules.sfp_uceprotect import sfp_uceprotect from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationUceprotect(unittest.TestCase): def test_handleEvent_event_data_safe_ip_address_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_uceprotect() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_uceprotect) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'IP_ADDRESS' event_data = '1.0.0.1' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_urlscan.py ================================================ import pytest import unittest from modules.sfp_urlscan import sfp_urlscan from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationUrlscan(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_urlscan() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_venmo.py ================================================ import pytest import unittest from modules.sfp_venmo import sfp_venmo from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationVenmo(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_venmo() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_viewdns.py ================================================ import pytest import unittest from modules.sfp_viewdns import sfp_viewdns from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationViewdns(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_viewdns() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_virustotal.py ================================================ import pytest import unittest from modules.sfp_virustotal import sfp_virustotal from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationVirustotal(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_virustotal() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_voipbl.py ================================================ import pytest import unittest from modules.sfp_voipbl import sfp_voipbl from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationVoipbl(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_voipbl() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_vxvault.py ================================================ import pytest import unittest from modules.sfp_vxvault import sfp_vxvault from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationVxvault(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_vxvault() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_webserver.py ================================================ import pytest import unittest from modules.sfp_webserver import sfp_webserver from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationWebserver(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_webserver() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_whatcms.py ================================================ import pytest import unittest from modules.sfp_whatcms import sfp_whatcms from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationWhatCMS(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_whatcms() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_whois.py ================================================ import pytest import unittest from modules.sfp_whois import sfp_whois from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationwhois(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_whois() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_whoisology.py ================================================ import pytest import unittest from modules.sfp_whoisology import sfp_whoisology from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationWhoisology(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_whoisology() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_whoxy.py ================================================ import pytest import unittest from modules.sfp_whoxy import sfp_whoxy from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationWhoxy(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_whoxy() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_wigle.py ================================================ import pytest import unittest from modules.sfp_wigle import sfp_wigle from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationWigle(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_wigle() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_wikileaks.py ================================================ import pytest import unittest from modules.sfp_wikileaks import sfp_wikileaks from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationWikileaks(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_wikileaks() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_wikipediaedits.py ================================================ import pytest import unittest from modules.sfp_wikipediaedits import sfp_wikipediaedits from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationWikipediaedits(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_wikipediaedits() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_xforce.py ================================================ import pytest import unittest from modules.sfp_xforce import sfp_xforce from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationXforce(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_xforce() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/integration/modules/test_sfp_yandexdns.py ================================================ import pytest import unittest from modules.sfp_yandexdns import sfp_yandexdns from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationYandexDns(unittest.TestCase): def test_handleEvent_event_data_safe_internet_name_not_blocked_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_yandexdns() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_yandexdns) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'yandex.com' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) def test_handleEvent_event_data_adult_internet_name_blocked_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_yandexdns() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'BLACKLISTED_INTERNET_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'Yandex - Adult [pornhub.com]' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_yandexdns) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'INTERNET_NAME' event_data = 'pornhub.com' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) ================================================ FILE: test/integration/modules/test_sfp_zetalytics.py ================================================ import pytest import unittest from modules.sfp_zetalytics import sfp_zetalytics from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationZetalytics(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_zetalytics() module.setup(sf, dict()) target_value = 'example target value' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/modules/test_sfp_zoneh.py ================================================ import pytest import unittest from modules.sfp_zoneh import sfp_zoneh from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntegrationZoneh(unittest.TestCase): @unittest.skip("todo") def test_handleEvent(self): sf = SpiderFoot(self.default_options) module = sfp_zoneh() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/integration/test_sf.py ================================================ # test_sf.py import subprocess import sys import unittest class TestSf(unittest.TestCase): """ Test TestSf """ default_types = [ "" ] default_modules = [ "sfp_base64", "sfp_bitcoin", "sfp_company", "sfp_cookie", "sfp_countryname", "sfp_creditcard", "sfp_email", "sfp_errors", "sfp_ethereum", "sfp_filemeta", "sfp_hashes", "sfp_iban", "sfp_names", "sfp_pageinfo", "sfp_phone", "sfp_strangeheaders", "sfp_webframework", "sfp_webserver", "sfp_webanalytics", ] def execute(self, command): proc = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) out, err = proc.communicate() return out, err, proc.returncode def test_no_args_should_print_arg_l_required(self): out, err, code = self.execute([sys.executable, "sf.py"]) self.assertIn(b"SpiderFoot requires -l : to start the web server. Try --help for guidance.", out) self.assertEqual(b"", err) self.assertEqual(255, code) def test_help_arg_should_print_help_and_exit(self): out, err, code = self.execute([sys.executable, "sf.py", "-h"]) self.assertIn(b"show this help message and exit", out) self.assertEqual(b"", err) self.assertEqual(0, code) def test_modules_arg_should_print_modules_and_exit(self): out, err, code = self.execute([sys.executable, "sf.py", "-M"]) self.assertIn(b"Modules available:", err) self.assertEqual(0, code) def test_types_arg_should_print_types_and_exit(self): out, err, code = self.execute([sys.executable, "sf.py", "-T"]) self.assertIn(b"Types available:", err) self.assertEqual(0, code) @unittest.skip("todo") def test_l_arg_should_start_web_server(self): listen = "127.0.0.1:5001" out, err, code = self.execute([sys.executable, "sf.py", "-l", listen]) self.assertIn(bytes(f"Starting web server at {listen}", 'utf-8'), err) self.assertEqual(0, code) def test_debug_arg_should_enable_and_print_debug_output(self): out, err, code = self.execute([sys.executable, "sf.py", "-d", "-m", "example module", "-s", "spiderfoot.net"]) self.assertIn(b"[DEBUG]", err) self.assertIn(b"sfp__stor_db : Storing an event: ROOT", err) self.assertEqual(0, code) def test_quiet_arg_should_hide_debug_output(self): out, err, code = self.execute([sys.executable, "sf.py", "-q", "-m", "example module", "-s", "spiderfoot.net"]) self.assertNotIn(b"[INFO]", err) self.assertEqual(0, code) def test_run_scan_invalid_target_should_exit(self): invalid_target = '.' out, err, code = self.execute([sys.executable, "sf.py", "-s", invalid_target]) self.assertIn(bytes(f"Could not determine target type. Invalid target: {invalid_target}", 'utf-8'), err) self.assertEqual(255, code) def test_run_scan_with_modules_no_target_should_exit(self): out, err, code = self.execute([sys.executable, "sf.py", "-m", ",".join(self.default_modules)]) self.assertIn(b"You must specify a target when running in scan mode", err) self.assertEqual(255, code) def test_run_scan_with_types_no_target_should_exit(self): out, err, code = self.execute([sys.executable, "sf.py", "-t", ",".join(self.default_types)]) self.assertIn(b"You must specify a target when running in scan mode", err) self.assertEqual(255, code) def test_run_scan_with_invalid_module_should_run_scan_and_exit(self): module = "invalid module" out, err, code = self.execute([sys.executable, "sf.py", "-m", module, "-s", "spiderfoot.net"]) self.assertIn(bytes(f"Failed to load module: {module}", 'utf-8'), err) self.assertEqual(0, code) def test_run_scan_with_invalid_type_should_exit(self): out, err, code = self.execute([sys.executable, "sf.py", "-t", "invalid type", "-s", "spiderfoot.net"]) self.assertIn(b"Based on your criteria, no modules were enabled", err) self.assertEqual(255, code) def test_run_scan_should_run_scan_and_exit(self): target = "spiderfoot.net" out, err, code = self.execute([sys.executable, "sf.py", "-m", ",".join(self.default_modules), "-s", target]) self.assertIn(b"Scan completed with status FINISHED", err) self.assertEqual(0, code) for module in self.default_modules: with self.subTest(module=module): self.assertIn(module.encode(), err) @unittest.skip("output buffering sometimes causes this test to fail") def test_run_scan_should_print_scan_result_and_exit(self): target = "spiderfoot.net" out, err, code = self.execute([sys.executable, "sf.py", "-m", ",".join(self.default_modules), "-s", target, "-o", "csv"]) self.assertIn(b"Scan completed with status FINISHED", err) self.assertEqual(0, code) for module in self.default_modules: with self.subTest(module=module): self.assertIn(module.encode(), err) expected_output = [ "Source,Type,Data", "SpiderFoot UI,Internet Name,spiderfoot.net,spiderfoot.net\n", "SpiderFoot UI,Domain Name,spiderfoot.net,spiderfoot.net\n", "sfp_countryname,Country Name,spiderfoot.net,United States\n", ] for output in expected_output: self.assertIn(bytes(output, 'utf-8'), out) ================================================ FILE: test/integration/test_sfcli.py ================================================ # test_sfcli.py import subprocess import sys import unittest class TestSfcli(unittest.TestCase): """ Test TestSfcli """ def execute(self, command): proc = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) out, err = proc.communicate() return out, err, proc.returncode def test_help_arg_should_print_help_and_exit(self): out, err, code = self.execute([sys.executable, "sfcli.py", "-h"]) self.assertIn(b"show this help message and exit", out) self.assertEqual(b"", err) self.assertEqual(0, code) ================================================ FILE: test/integration/test_sfwebui.py ================================================ # test_sfwebui.py import os import unittest import cherrypy from cherrypy.test import helper from spiderfoot import SpiderFootHelpers from sfwebui import SpiderFootWebUi class TestSpiderFootWebUiRoutes(helper.CPWebCase): @staticmethod def setup_server(): default_config = { '_debug': False, # Debug '__logging': True, # Logging in general '__outputfilter': None, # Event types to filter from modules' output '_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests '_dnsserver': '', # Override the default resolver '_fetchtimeout': 5, # number of seconds before giving up on a fetch '_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat', '_internettlds_cache': 72, '_genericusers': ",".join(SpiderFootHelpers.usernamesFromWordlists(['generic-usernames'])), '__database': f"{SpiderFootHelpers.dataPath()}/spiderfoot.test.db", # note: test database file '__modules__': None, # List of modules. Will be set after start-up. '__correlationrules__': None, # List of correlation rules. Will be set after start-up. '_socks1type': '', '_socks2addr': '', '_socks3port': '', '_socks4user': '', '_socks5pwd': '', '__logstdout': False } default_web_config = { 'root': '/' } mod_dir = os.path.dirname(os.path.abspath(__file__)) + '/../../modules/' default_config['__modules__'] = SpiderFootHelpers.loadModulesAsDict(mod_dir, ['sfp_template.py']) conf = { '/query': { 'tools.encode.text_only': False, 'tools.encode.add_charset': True, }, '/static': { 'tools.staticdir.on': True, 'tools.staticdir.dir': 'static', 'tools.staticdir.root': f"{os.path.dirname(os.path.abspath(__file__))}/../../spiderfoot", } } cherrypy.tree.mount(SpiderFootWebUi(default_web_config, default_config), script_name=default_web_config.get('root'), config=conf) def test_invalid_page_returns_404(self): self.getPage("/doesnotexist") self.assertStatus('404 Not Found') def test_static_returns_200(self): self.getPage("/static/img/spiderfoot-header.png") self.assertStatus('200 OK') def test_scaneventresultexport_invalid_scan_id_returns_200(self): self.getPage("/scaneventresultexport?id=doesnotexist&type=doesnotexist") self.assertStatus('200 OK') def test_scaneventresultexportmulti(self): self.getPage("/scaneventresultexportmulti?ids=doesnotexist") self.assertStatus('200 OK') def test_scansearchresultexport(self): self.getPage("/scansearchresultexport?id=doesnotexist") self.assertStatus('200 OK') def test_scanexportjsonmulti(self): self.getPage("/scanexportjsonmulti?ids=doesnotexist") self.assertStatus('200 OK') def test_scanviz(self): self.getPage("/scanviz?id=doesnotexist") self.assertStatus('200 OK') def test_scanvizmulti(self): self.getPage("/scanvizmulti?ids=doesnotexist") self.assertStatus('200 OK') def test_scanopts_invalid_scan_returns_200(self): self.getPage("/scanopts?id=doesnotexist") self.assertStatus('200 OK') def test_rerunscan(self): self.getPage("/rerunscan?id=doesnotexist") self.assertStatus('200 OK') self.assertInBody("Invalid scan ID.") def test_rerunscanmulti_invalid_scan_id_returns_200(self): self.getPage("/rerunscanmulti?ids=doesnotexist") self.assertStatus('200 OK') self.assertInBody("Invalid scan ID.") def test_newscan_returns_200(self): self.getPage("/newscan") self.assertStatus('200 OK') self.assertInBody("Scan Name") self.assertInBody("Scan Target") def test_clonescan(self): self.getPage("/clonescan?id=doesnotexist") self.assertStatus('200 OK') self.assertInBody("Invalid scan ID.") def test_index_returns_200(self): self.getPage("/") self.assertStatus('200 OK') def test_scaninfo_invalid_scan_returns_200(self): self.getPage("/scaninfo?id=doesnotexist") self.assertStatus('200 OK') self.assertInBody("Scan ID not found.") @unittest.skip("todo") def test_opts_returns_200(self): self.getPage("/opts") self.assertStatus('200 OK') def test_optsexport(self): self.getPage("/optsexport") self.assertStatus('200 OK') self.getPage("/optsexport?pattern=api_key") self.assertStatus('200 OK') self.assertHeader("Content-Disposition", "attachment; filename=\"SpiderFoot.cfg\"") self.assertInBody(":api_key=") def test_optsraw(self): self.getPage("/optsraw") self.assertStatus('200 OK') def test_scandelete_invalid_scan_id_returns_404(self): self.getPage("/scandelete?id=doesnotexist") self.assertStatus('404 Not Found') self.assertInBody('Scan doesnotexist does not exist') @unittest.skip("todo") def test_savesettings(self): self.getPage("/savesettings") self.assertStatus('200 OK') @unittest.skip("todo") def test_savesettingsraw(self): self.getPage("/savesettingsraw") self.assertStatus('200 OK') def test_resultsetfp(self): self.getPage("/resultsetfp?id=doesnotexist&resultids=doesnotexist&fp=1") self.assertStatus('200 OK') self.assertInBody("No IDs supplied.") def test_eventtypes(self): self.getPage("/eventtypes") self.assertStatus('200 OK') self.assertInBody('"DOMAIN_NAME"') def test_modules(self): self.getPage("/modules") self.assertStatus('200 OK') self.assertInBody('"name":') def test_ping_returns_200(self): self.getPage("/ping") self.assertStatus('200 OK') self.assertInBody('"SUCCESS"') def test_query_returns_200(self): self.getPage("/query?query=SELECT+1") self.assertStatus('200 OK') self.assertInBody('[{"1": 1}]') def test_startscan_invalid_scan_name_returns_error(self): self.getPage("/startscan?scanname=&scantarget=&modulelist=&typelist=&usecase=") self.assertStatus('200 OK') self.assertInBody('Invalid request: scan name was not specified.') def test_startscan_invalid_scan_target_returns_error(self): self.getPage("/startscan?scanname=example-scan&scantarget=&modulelist=&typelist=&usecase=") self.assertStatus('200 OK') self.assertInBody('Invalid request: scan target was not specified.') def test_startscan_unrecognized_scan_target_returns_error(self): self.getPage("/startscan?scanname=example-scan&scantarget=invalid-target&modulelist=doesnotexist&typelist=doesnotexist&usecase=doesnotexist") self.assertStatus('200 OK') self.assertInBody('Invalid target type. Could not recognize it as a target SpiderFoot supports.') def test_startscan_invalid_modules_returns_error(self): self.getPage("/startscan?scanname=example-scan&scantarget=spiderfoot.net&modulelist=&typelist=&usecase=") self.assertStatus('200 OK') self.assertInBody('Invalid request: no modules specified for scan.') def test_startscan_invalid_typelist_returns_error(self): self.getPage("/startscan?scanname=example-scan&scantarget=spiderfoot.net&modulelist=&typelist=doesnotexist&usecase=") self.assertStatus('200 OK') self.assertInBody('Invalid request: no modules specified for scan.') def test_startscan_should_start_a_scan(self): self.getPage("/startscan?scanname=spiderfoot.net&scantarget=spiderfoot.net&modulelist=doesnotexist&typelist=doesnotexist&usecase=doesnotexist") self.assertStatus('303 See Other') def test_stopscan_invalid_scan_id_returns_404(self): self.getPage("/stopscan?id=doesnotexist") self.assertStatus('404 Not Found') self.assertInBody('Scan doesnotexist does not exist') def test_scanlog_invalid_scan_returns_200(self): self.getPage("/scanlog?id=doesnotexist") self.assertStatus('200 OK') def test_scanerrors_invalid_scan_returns_200(self): self.getPage("/scanerrors?id=doesnotexist") self.assertStatus('200 OK') def test_scanlist_returns_200(self): self.getPage("/scanlist") self.assertStatus('200 OK') def test_scanstatus_invalid_scan_returns_200(self): self.getPage("/scanstatus?id=doesnotexist") self.assertStatus('200 OK') def test_scansummary_invalid_scan_returns_200(self): self.getPage("/scansummary?id=doesnotexist&by=anything") self.assertStatus('200 OK') def test_scaneventresults_invalid_scan_returns_200(self): self.getPage("/scaneventresults?id=doesnotexist&eventType=anything") self.assertStatus('200 OK') def test_scaneventresultsunique_invalid_scan_returns_200(self): self.getPage("/scaneventresultsunique?id=doesnotexist&eventType=anything") self.assertStatus('200 OK') def test_search_returns_200(self): self.getPage("/search?id=doesnotexist&eventType=doesnotexist&value=doesnotexist") self.assertStatus('200 OK') def test_scanhistory_invalid_scan_returns_200(self): self.getPage("/scanhistory?id=doesnotexist") self.assertStatus('200 OK') def test_scanelementtypediscovery_invalid_scan_id_returns_200(self): self.getPage("/scanelementtypediscovery?id=doesnotexist&eventType=anything") self.assertStatus('200 OK') ================================================ FILE: test/requirements.txt ================================================ -r ../requirements.txt darglint==1.8.1 dlint==0.14.0 flake8==5.0.4 flake8-annotations==2.9.1 flake8-blind-except==0.2.1 flake8-bugbear==23.2.13 flake8-builtins==2.1.0 flake8-quotes==3.3.2 flake8-return==1.2.0 flake8-sfs==0.0.4 flake8-simplify==0.19.3 pytest==7.2.1 pytest-cov==4.0.0 pytest-mock==3.10.0 pytest-xdist==3.2.0 pycodestyle==2.9.0 responses==0.22.0 ================================================ FILE: test/run ================================================ #!/bin/bash # Run unit and integration tests (excluding module integration tests). # These same tests are run on all pull requests automatically. # # Must be run from SpiderFoot root directory; ie: # ./test/run echo Running flake8 ... time python3 -m flake8 . --count --show-source --statistics || exit echo Running pytest ... time python3 -m pytest -n auto --dist loadfile --ignore=test/integration/modules/ --durations=5 --cov-report html --cov=. . ================================================ FILE: test/unit/__init__.py ================================================ ================================================ FILE: test/unit/modules/__init__.py ================================================ ================================================ FILE: test/unit/modules/test_sfp__stor_db.py ================================================ import pytest import unittest from modules.sfp__stor_db import sfp__stor_db from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleStor_db(unittest.TestCase): @unittest.skip("This module contains an extra private option") def test_opts(self): module = sfp__stor_db() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp__stor_db() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp__stor_db() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp__stor_db() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp__stor_stdout.py ================================================ import pytest import unittest from modules.sfp__stor_stdout import sfp__stor_stdout from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleStor_stdout(unittest.TestCase): @unittest.skip("This module contains an extra private option") def test_opts(self): module = sfp__stor_stdout() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp__stor_stdout() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp__stor_stdout() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp__stor_stdout() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_abstractapi.py ================================================ import pytest import unittest from modules.sfp_abstractapi import sfp_abstractapi from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleAbstractapi(unittest.TestCase): def test_opts(self): module = sfp_abstractapi() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_abstractapi() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_abstractapi() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_abstractapi() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "204", "429"] for code in http_codes: with self.subTest(code=code): module = sfp_abstractapi() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "422", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_abstractapi() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_abstractapi() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_abusech.py ================================================ import pytest import unittest from modules.sfp_abusech import sfp_abusech from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleAbusech(unittest.TestCase): def test_opts(self): module = sfp_abusech() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_abusech() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_abusech() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_abusech() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_abuseipdb.py ================================================ import pytest import unittest from modules.sfp_abuseipdb import sfp_abuseipdb from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleAbuseipdb(unittest.TestCase): def test_opts(self): module = sfp_abuseipdb() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_abuseipdb() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_abuseipdb() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_abuseipdb() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_abuseipdb() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_abusix.py ================================================ import pytest import unittest from modules.sfp_abusix import sfp_abusix from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleAbusix(unittest.TestCase): def test_opts(self): module = sfp_abusix() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_abusix() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_abusix() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_abusix() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_abusix() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_accounts.py ================================================ import pytest import unittest from modules.sfp_accounts import sfp_accounts from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleAccounts(unittest.TestCase): def test_opts(self): module = sfp_accounts() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_accounts() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_accounts() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_accounts() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_adblock.py ================================================ import pytest import unittest from modules.sfp_adblock import sfp_adblock from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleAdblock(unittest.TestCase): def test_opts(self): module = sfp_adblock() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_adblock() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_adblock() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_adblock() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_adguard_dns.py ================================================ import pytest import unittest from modules.sfp_adguard_dns import sfp_adguard_dns from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleAdGuardDns(unittest.TestCase): def test_opts(self): module = sfp_adguard_dns() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_adguard_dns() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_adguard_dns() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_adguard_dns() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_ahmia.py ================================================ import pytest import unittest from modules.sfp_ahmia import sfp_ahmia from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleAhmia(unittest.TestCase): def test_opts(self): module = sfp_ahmia() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_ahmia() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_ahmia() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_ahmia() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_alienvault.py ================================================ import pytest import unittest from modules.sfp_alienvault import sfp_alienvault from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleAlienvault(unittest.TestCase): def test_opts(self): module = sfp_alienvault() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_alienvault() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_alienvault() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_alienvault() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "404"] for code in http_codes: with self.subTest(code=code): module = sfp_alienvault() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) # TODO: http_codes = ["401", "402", "403", "429", "500", "502", "503"] http_codes = ["403", "429"] for code in http_codes: with self.subTest(code=code): module = sfp_alienvault() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_alienvault() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_alienvaultiprep.py ================================================ import pytest import unittest from modules.sfp_alienvaultiprep import sfp_alienvaultiprep from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleAlienvaultiprep(unittest.TestCase): def test_opts(self): module = sfp_alienvaultiprep() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_alienvaultiprep() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_alienvaultiprep() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_alienvaultiprep() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_apple_itunes.py ================================================ import pytest import unittest from modules.sfp_apple_itunes import sfp_apple_itunes from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleAppleItunes(unittest.TestCase): def test_opts(self): module = sfp_apple_itunes() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_apple_itunes() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_apple_itunes() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_apple_itunes() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_archiveorg.py ================================================ import pytest import unittest from modules.sfp_archiveorg import sfp_archiveorg from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleArchiveorg(unittest.TestCase): def test_opts(self): module = sfp_archiveorg() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_archiveorg() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_archiveorg() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_archiveorg() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_arin.py ================================================ import pytest import unittest from modules.sfp_arin import sfp_arin from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleArin(unittest.TestCase): def test_opts(self): module = sfp_arin() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_arin() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_arin() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_arin() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_azureblobstorage.py ================================================ import pytest import unittest from modules.sfp_azureblobstorage import sfp_azureblobstorage from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleAzureblobstorage(unittest.TestCase): def test_opts(self): module = sfp_azureblobstorage() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_azureblobstorage() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_azureblobstorage() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_azureblobstorage() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_base64.py ================================================ import pytest import unittest from modules.sfp_base64 import sfp_base64 from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleBase64(unittest.TestCase): def test_opts(self): module = sfp_base64() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_base64() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_base64() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_base64() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_url_containing_base64_string_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_base64() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'BASE64_DATA' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "U3BpZGVyRm9vdA== (SpiderFoot)" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_base64) event_type = 'ROOT' event_data = 'https://spiderfoot.net/path?param=example%20data%20U3BpZGVyRm9vdA%3d%3d%20example%20data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_not_containing_base64_string_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_base64() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_base64) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_bgpview.py ================================================ import pytest import unittest from modules.sfp_bgpview import sfp_bgpview from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleBgpview(unittest.TestCase): def test_opts(self): module = sfp_bgpview() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_bgpview() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_bgpview() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_bgpview() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_binaryedge.py ================================================ import pytest import unittest from modules.sfp_binaryedge import sfp_binaryedge from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleBinaryedge(unittest.TestCase): def test_opts(self): module = sfp_binaryedge() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_binaryedge() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_binaryedge() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_binaryedge() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_binaryedge() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_bingsearch.py ================================================ import pytest import unittest from modules.sfp_bingsearch import sfp_bingsearch from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleBingsearch(unittest.TestCase): def test_opts(self): module = sfp_bingsearch() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_bingsearch() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_bingsearch() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_bingsearch() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_bingsearch() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_bingsharedip.py ================================================ import pytest import unittest from modules.sfp_bingsharedip import sfp_bingsharedip from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleBingsharedip(unittest.TestCase): def test_opts(self): module = sfp_bingsharedip() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_bingsharedip() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_bingsharedip() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_bingsharedip() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_bingsharedip() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_binstring.py ================================================ import pytest import unittest from modules.sfp_binstring import sfp_binstring from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleBinstring(unittest.TestCase): def test_opts(self): module = sfp_binstring() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_binstring() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_binstring() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_binstring() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_bitcoin.py ================================================ import pytest import unittest from modules.sfp_bitcoin import sfp_bitcoin from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleBitcoin(unittest.TestCase): def test_opts(self): module = sfp_bitcoin() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_bitcoin() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_bitcoin() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_bitcoin() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_containing_bitcoin_string_in_legacy_base58_format_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_bitcoin() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'BITCOIN_ADDRESS' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = '1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_bitcoin) event_type = 'ROOT' event_data = 'example data 1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_containing_bitcoin_string_in_bech32_format_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_bitcoin() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'BITCOIN_ADDRESS' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'bc1q4r8h8vqk02gnvlus758qmpk8jmajpy2ld23xtr73a39ps0r9z82qq0qqye' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_bitcoin) event_type = 'ROOT' event_data = 'example data bc1q4r8h8vqk02gnvlus758qmpk8jmajpy2ld23xtr73a39ps0r9z82qq0qqye example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_not_containing_bitcoin_string_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_bitcoin() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_bitcoin) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_bitcoinabuse.py ================================================ import pytest import unittest from modules.sfp_bitcoinabuse import sfp_bitcoinabuse from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleBitcoinAbuse(unittest.TestCase): def test_opts(self): module = sfp_bitcoinabuse() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_bitcoinabuse() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_bitcoinabuse() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_bitcoinabuse() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "404"] for code in http_codes: with self.subTest(code=code): module = sfp_bitcoinabuse() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_bitcoinabuse() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_bitcoinabuse() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_bitcoinwhoswho.py ================================================ import pytest import unittest from modules.sfp_bitcoinwhoswho import sfp_bitcoinwhoswho from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleBitcoinwhoswho(unittest.TestCase): def test_opts(self): module = sfp_bitcoinwhoswho() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_bitcoinwhoswho() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_bitcoinwhoswho() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_bitcoinwhoswho() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_bitcoinwhoswho() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_blockchain.py ================================================ import pytest import unittest from modules.sfp_blockchain import sfp_blockchain from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleBlockchain(unittest.TestCase): def test_opts(self): module = sfp_blockchain() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_blockchain() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_blockchain() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_blockchain() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_blocklistde.py ================================================ import pytest import unittest from modules.sfp_blocklistde import sfp_blocklistde from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleBlocklistde(unittest.TestCase): def test_opts(self): module = sfp_blocklistde() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_blocklistde() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_blocklistde() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_blocklistde() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_botscout.py ================================================ import pytest import unittest from modules.sfp_botscout import sfp_botscout from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleBotscout(unittest.TestCase): def test_opts(self): module = sfp_botscout() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_botscout() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_botscout() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_botscout() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200"] for code in http_codes: with self.subTest(code=code): module = sfp_botscout() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_botscout() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_botvrij.py ================================================ import pytest import unittest from modules.sfp_botvrij import sfp_botvrij from sflib import SpiderFoot @pytest.mark.usefixtures class TestModulebotvrij(unittest.TestCase): def test_opts(self): module = sfp_botvrij() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_botvrij() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_botvrij() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_botvrij() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_builtwith.py ================================================ import pytest import unittest from modules.sfp_builtwith import sfp_builtwith from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModulebuiltwith(unittest.TestCase): def test_opts(self): module = sfp_builtwith() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_builtwith() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_builtwith() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_builtwith() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_builtwith() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_c99.py ================================================ import pytest import unittest from modules.sfp_c99 import sfp_c99 from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleC99(unittest.TestCase): def test_opts(self): module = sfp_c99() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_c99() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_c99() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_c99() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_c99() module.setup(sf, dict()) target_value = "example target value" target_type = "IP_ADDRESS" target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = "ROOT" event_data = "example data" event_module = "" source_event = "" evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_callername.py ================================================ import pytest import unittest from modules.sfp_callername import sfp_callername from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCallername(unittest.TestCase): def test_opts(self): module = sfp_callername() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_callername() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_callername() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_callername() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_censys.py ================================================ import pytest import unittest from modules.sfp_censys import sfp_censys from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleCensys(unittest.TestCase): def test_opts(self): module = sfp_censys() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_censys() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_censys() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_censys() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "400", "404"] for code in http_codes: with self.subTest(code=code): module = sfp_censys() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_censys() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_censys() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_certspotter.py ================================================ import pytest import unittest from modules.sfp_certspotter import sfp_certspotter from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleiCertspotter(unittest.TestCase): def test_opts(self): module = sfp_certspotter() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_certspotter() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_certspotter() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_certspotter() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_certspotter() module.setup(sf, dict()) target_value = 'example target value' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_cinsscore.py ================================================ import pytest import unittest from modules.sfp_cinsscore import sfp_cinsscore from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCinsscore(unittest.TestCase): def test_opts(self): module = sfp_cinsscore() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_cinsscore() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_cinsscore() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_cinsscore() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_circllu.py ================================================ import pytest import unittest from modules.sfp_circllu import sfp_circllu from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleCircllu(unittest.TestCase): def test_opts(self): module = sfp_circllu() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_circllu() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_circllu() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_circllu() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_circllu() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_citadel.py ================================================ import pytest import unittest from modules.sfp_citadel import sfp_citadel from sflib import SpiderFoot @pytest.mark.usefixtures class TestModulecitadel(unittest.TestCase): def test_opts(self): module = sfp_citadel() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_citadel() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_citadel() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_citadel() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_cleanbrowsing.py ================================================ import pytest import unittest from modules.sfp_cleanbrowsing import sfp_cleanbrowsing from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCleanbrowsing(unittest.TestCase): def test_opts(self): module = sfp_cleanbrowsing() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_cleanbrowsing() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_cleanbrowsing() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_cleanbrowsing() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_cleantalk.py ================================================ import pytest import unittest from modules.sfp_cleantalk import sfp_cleantalk from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCleantalk(unittest.TestCase): def test_opts(self): module = sfp_cleantalk() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_cleantalk() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_cleantalk() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_cleantalk() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_clearbit.py ================================================ import pytest import unittest from modules.sfp_clearbit import sfp_clearbit from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleClearbit(unittest.TestCase): def test_opts(self): module = sfp_clearbit() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_clearbit() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_clearbit() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_clearbit() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "404", "429"] for code in http_codes: with self.subTest(code=code): module = sfp_clearbit() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_clearbit() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_clearbit() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_cloudflaredns.py ================================================ import pytest import unittest from modules.sfp_cloudflaredns import sfp_cloudflaredns from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCloudflaredns(unittest.TestCase): def test_opts(self): module = sfp_cloudflaredns() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_cloudflaredns() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_cloudflaredns() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_cloudflaredns() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_coinblocker.py ================================================ import pytest import unittest from modules.sfp_coinblocker import sfp_coinblocker from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCoinblocker(unittest.TestCase): def test_opts(self): module = sfp_coinblocker() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_coinblocker() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_coinblocker() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_coinblocker() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_commoncrawl.py ================================================ import pytest import unittest from modules.sfp_commoncrawl import sfp_commoncrawl from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCommoncrawl(unittest.TestCase): def test_opts(self): module = sfp_commoncrawl() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_commoncrawl() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_commoncrawl() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_commoncrawl() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_comodo.py ================================================ import pytest import unittest from modules.sfp_comodo import sfp_comodo from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleComodo(unittest.TestCase): def test_opts(self): module = sfp_comodo() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_comodo() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_comodo() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_comodo() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_company.py ================================================ import pytest import unittest from modules.sfp_company import sfp_company from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleCompany(unittest.TestCase): def test_opts(self): module = sfp_company() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_company() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_company() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_company() self.assertIsInstance(module.producedEvents(), list) @unittest.skip("todo") def test_handleEvent_event_data_ssl_certificate_issued_containing_company_name_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_company() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'COMPANY_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "SpiderFoot Corporation" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_company) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'SSL_CERTIFICATE_ISSUED' event_data = 'O=SpiderFoot Corporation' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) @unittest.skip("todo") def test_handleEvent_event_data_domain_whois_containing_company_name_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_company() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'COMPANY_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "SpiderFoot Corporation" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_company) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'DOMAIN_WHOIS' event_data = 'Registrant Organization: SpiderFoot Corporation' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) @unittest.skip("todo") def test_handleEvent_event_data_target_web_content_containing_company_name_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_company() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'COMPANY_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "SpiderFoot Corporation" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_company) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = '

Copyright SpiderFoot Corporation 2022.

' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) ================================================ FILE: test/unit/modules/test_sfp_cookie.py ================================================ import pytest import unittest from modules.sfp_cookie import sfp_cookie from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleCookie(unittest.TestCase): def test_opts(self): module = sfp_cookie() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_cookie() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_cookie() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_cookie() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_containing_cookie_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_cookie() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'TARGET_WEB_COOKIE' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'example cookie' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_cookie) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'WEBSERVER_HTTPHEADERS' event_data = '{"cookie": "example cookie"}' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = "https://spiderfoot.net/example" with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_not_containing_cookie_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_cookie() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_cookie) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'WEBSERVER_HTTPHEADERS' event_data = '{"not cookie": "example cookie"}' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = "https://spiderfoot.net/example" result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_countryname.py ================================================ import pytest import unittest from modules.sfp_countryname import sfp_countryname from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleCountryName(unittest.TestCase): def test_opts(self): module = sfp_countryname() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_countryname() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_countryname() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_countryname() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_phone_number_event_data_containing_countrycode_should_create_country_name_event(self): sf = SpiderFoot(self.default_options) module = sfp_countryname() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'COUNTRY_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "United States" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_countryname) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'PHONE_NUMBER' event_data = '+12345678901' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_domain_whois_event_data_containing_countryname_string_should_create_country_name_event(self): sf = SpiderFoot(self.default_options) module = sfp_countryname() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'COUNTRY_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "United States" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_countryname) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'DOMAIN_WHOIS' event_data = 'example data 123 Fake St, Fakeville, "United States" example data' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_domain_whois_event_data_not_containing_countryname_string_should_not_create_event(self): sf = SpiderFoot(self.default_options) module = sfp_countryname() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_countryname) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'DOMAIN_WHOIS' event_data = 'example data' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_creditcard.py ================================================ import pytest import unittest from modules.sfp_creditcard import sfp_creditcard from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleCreditCard(unittest.TestCase): def test_opts(self): module = sfp_creditcard() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_creditcard() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_creditcard() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_creditcard() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_containing_creditcard_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_creditcard() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'CREDIT_CARD_NUMBER' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = '4111111111111111' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_creditcard) event_type = 'ROOT' event_data = 'example data 4111 1111 1111 1111 example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_not_containing_creditcard_string_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_creditcard() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_creditcard) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_crobat_api.py ================================================ import pytest import unittest from modules.sfp_crobat_api import sfp_crobat_api from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCrobatApi(unittest.TestCase): def test_opts(self): module = sfp_crobat_api() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_crobat_api() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_crobat_api() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_crobat_api() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200"] for code in http_codes: with self.subTest(code=code): module = sfp_crobat_api() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_crobat_api() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_crossref.py ================================================ import pytest import unittest from modules.sfp_crossref import sfp_crossref from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCrossref(unittest.TestCase): def test_opts(self): module = sfp_crossref() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_crossref() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_crossref() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_crossref() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_crt.py ================================================ import pytest import unittest from modules.sfp_crt import sfp_crt from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCrt(unittest.TestCase): def test_opts(self): module = sfp_crt() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_crt() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_crt() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_crt() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "404"] for code in http_codes: with self.subTest(code=code): module = sfp_crt() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "403", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_crt() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_crxcavator.py ================================================ import pytest import unittest from modules.sfp_crxcavator import sfp_crxcavator from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCrxcavator(unittest.TestCase): def test_opts(self): module = sfp_crxcavator() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_crxcavator() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_crxcavator() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_crxcavator() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_customfeed.py ================================================ import pytest import unittest from modules.sfp_customfeed import sfp_customfeed from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleCustomfeed(unittest.TestCase): def test_opts(self): module = sfp_customfeed() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_customfeed() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_customfeed() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_customfeed() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_feed_url_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_customfeed() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_cybercrimetracker.py ================================================ import pytest import unittest from modules.sfp_cybercrimetracker import sfp_cybercrimetracker from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCybercrimetracker(unittest.TestCase): def test_opts(self): module = sfp_cybercrimetracker() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_cybercrimetracker() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_cybercrimetracker() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_cybercrimetracker() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_debounce.py ================================================ import pytest import unittest from modules.sfp_debounce import sfp_debounce from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDebounce(unittest.TestCase): def test_opts(self): module = sfp_debounce() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_debounce() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_debounce() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_debounce() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_dehashed.py ================================================ import pytest import unittest from modules.sfp_dehashed import sfp_dehashed from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleDehashed(unittest.TestCase): def test_opts(self): module = sfp_dehashed() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_dehashed() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_dehashed() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_dehashed() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_dehashed() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_digitaloceanspace.py ================================================ import pytest import unittest from modules.sfp_digitaloceanspace import sfp_digitaloceanspace from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDigitaloceanspace(unittest.TestCase): def test_opts(self): module = sfp_digitaloceanspace() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_digitaloceanspace() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_digitaloceanspace() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_digitaloceanspace() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_dns_for_family.py ================================================ import pytest import unittest from modules.sfp_dns_for_family import sfp_dns_for_family from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDnsForFamily(unittest.TestCase): def test_opts(self): module = sfp_dns_for_family() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_dns_for_family() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_dns_for_family() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_dns_for_family() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_dnsbrute.py ================================================ import pytest import unittest from modules.sfp_dnsbrute import sfp_dnsbrute from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDnsBrute(unittest.TestCase): def test_opts(self): module = sfp_dnsbrute() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_dnsbrute() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_dnsbrute() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_dnsbrute() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_dnscommonsrv.py ================================================ import pytest import unittest from modules.sfp_dnscommonsrv import sfp_dnscommonsrv from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDnsCommonsrv(unittest.TestCase): def test_opts(self): module = sfp_dnscommonsrv() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_dnscommonsrv() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_dnscommonsrv() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_dnscommonsrv() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_dnsdb.py ================================================ import pytest import unittest from modules.sfp_dnsdb import sfp_dnsdb from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleDnsDb(unittest.TestCase): def test_opts(self): module = sfp_dnsdb() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_dnsdb() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_dnsdb() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_dnsdb() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_dnsdb() module.setup(sf, dict()) target_value = "example target value" target_type = "IP_ADDRESS" target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = "ROOT" event_data = "example data" event_module = "" source_event = "" evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_dnsdumpster.py ================================================ import pytest import unittest from modules.sfp_dnsdumpster import sfp_dnsdumpster from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDnsDumpster(unittest.TestCase): def test_opts(self): module = sfp_dnsdumpster() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_dnsdumpster() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_dnsdumpster() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_dnsdumpster() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_dnsgrep.py ================================================ import pytest import unittest from modules.sfp_dnsgrep import sfp_dnsgrep from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDnsGrep(unittest.TestCase): def test_opts(self): module = sfp_dnsgrep() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_dnsgrep() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_dnsgrep() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_dnsgrep() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_dnsneighbor.py ================================================ import pytest import unittest from modules.sfp_dnsneighbor import sfp_dnsneighbor from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDnsNeighbor(unittest.TestCase): def test_opts(self): module = sfp_dnsneighbor() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_dnsneighbor() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_dnsneighbor() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_dnsneighbor() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_dnsraw.py ================================================ import pytest import unittest from modules.sfp_dnsraw import sfp_dnsraw from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDnsRaw(unittest.TestCase): def test_opts(self): module = sfp_dnsraw() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_dnsraw() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_dnsraw() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_dnsraw() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_dnsresolve.py ================================================ import pytest import unittest from modules.sfp_dnsresolve import sfp_dnsresolve from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDnsResolve(unittest.TestCase): def test_opts(self): module = sfp_dnsresolve() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_dnsresolve() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_dnsresolve() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_dnsresolve() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_dnszonexfer.py ================================================ import pytest import unittest from modules.sfp_dnszonexfer import sfp_dnszonexfer from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDnsZonexfer(unittest.TestCase): def test_opts(self): module = sfp_dnszonexfer() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_dnszonexfer() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_dnszonexfer() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_dnszonexfer() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_dronebl.py ================================================ import pytest import unittest from modules.sfp_dronebl import sfp_dronebl from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDronebl(unittest.TestCase): def test_opts(self): module = sfp_dronebl() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_dronebl() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_dronebl() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_dronebl() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_duckduckgo.py ================================================ import pytest import unittest from modules.sfp_duckduckgo import sfp_duckduckgo from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleDuckduckgo(unittest.TestCase): def test_opts(self): module = sfp_duckduckgo() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_duckduckgo() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_duckduckgo() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_duckduckgo() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_email.py ================================================ import pytest import unittest from modules.sfp_email import sfp_email from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleEmail(unittest.TestCase): def test_opts(self): module = sfp_email() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_email() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_email() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_email() self.assertIsInstance(module.producedEvents(), list) @unittest.skip("todo") def test_handleEvent_event_data_target_web_content_containing_email_address_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_email() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'EMAILADDR' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'firstname.lastname@spiderfoot.net' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_email) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = '

sample data firstname.lastname@spiderfoot.net sample data.

' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) ================================================ FILE: test/unit/modules/test_sfp_emailcrawlr.py ================================================ import pytest import unittest from modules.sfp_emailcrawlr import sfp_emailcrawlr from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleEmailcrawlr(unittest.TestCase): def test_opts(self): module = sfp_emailcrawlr() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_emailcrawlr() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_emailcrawlr() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_emailcrawlr() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "404"] for code in http_codes: with self.subTest(code=code): module = sfp_emailcrawlr() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_emailcrawlr() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_emailcrawlr() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_emailformat.py ================================================ import pytest import unittest from modules.sfp_emailformat import sfp_emailformat from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleEmailformat(unittest.TestCase): def test_opts(self): module = sfp_emailformat() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_emailformat() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_emailformat() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_emailformat() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_emailrep.py ================================================ import pytest import unittest from modules.sfp_emailrep import sfp_emailrep from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleEmailrep(unittest.TestCase): def test_opts(self): module = sfp_emailrep() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_emailrep() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_emailrep() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_emailrep() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_emergingthreats.py ================================================ import pytest import unittest from modules.sfp_emergingthreats import sfp_emergingthreats from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleEmergingthreats(unittest.TestCase): def test_opts(self): module = sfp_emergingthreats() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_emergingthreats() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_emergingthreats() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_emergingthreats() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_errors.py ================================================ import pytest import unittest from modules.sfp_errors import sfp_errors from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleErrors(unittest.TestCase): def test_opts(self): module = sfp_errors() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_errors() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_errors() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_errors() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_should_only_handle_events_from_sfp_spider(self): sf = SpiderFoot(self.default_options) module = sfp_errors() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_errors) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data Internal Server Error example data' event_module = 'something else entirely' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = 'https://spiderfoot.net/' result = module.handleEvent(evt) self.assertIsNone(result) def test_handleEvent_should_only_handle_events_within_target_scope(self): sf = SpiderFoot(self.default_options) module = sfp_errors() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_errors) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data Internal Server Error example data' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = 'https://something.else.entirely/' result = module.handleEvent(evt) self.assertIsNone(result) def test_handleEvent_event_data_containing_error_string_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_errors() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'ERROR_MESSAGE' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'Generic Error' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_errors) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data Internal Server Error example data' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = 'https://spiderfoot.net/' with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_not_containing_error_string_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_errors() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_errors) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = 'https://spiderfoot.net/' result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_ethereum.py ================================================ import pytest import unittest from modules.sfp_ethereum import sfp_ethereum from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleEthereum(unittest.TestCase): def test_opts(self): module = sfp_ethereum() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_ethereum() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_ethereum() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_ethereum() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_containing_ethereum_string_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_ethereum() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'ETHEREUM_ADDRESS' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = '0x89205A3A3b2A69De6Dbf7f01ED13B2108B2c43e7' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_ethereum) event_type = 'ROOT' event_data = 'example data 0x89205A3A3b2A69De6Dbf7f01ED13B2108B2c43e7 example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_not_containing_ethereum_string_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_ethereum() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_ethereum) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_etherscan.py ================================================ import pytest import unittest from modules.sfp_etherscan import sfp_etherscan from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleEtherscan(unittest.TestCase): def test_opts(self): module = sfp_etherscan() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_etherscan() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_etherscan() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_etherscan() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_etherscan() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_filemeta.py ================================================ import pytest import unittest from modules.sfp_filemeta import sfp_filemeta from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleFilemeta(unittest.TestCase): def test_opts(self): module = sfp_filemeta() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_filemeta() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_filemeta() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_filemeta() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_flickr.py ================================================ import pytest import unittest from modules.sfp_flickr import sfp_flickr from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleFlickr(unittest.TestCase): def test_opts(self): module = sfp_flickr() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_flickr() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_flickr() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_flickr() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_focsec.py ================================================ import pytest import unittest from modules.sfp_focsec import sfp_focsec from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleFocsec(unittest.TestCase): def test_opts(self): module = sfp_focsec() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_focsec() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_focsec() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_focsec() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_focsec() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_fortinet.py ================================================ import pytest import unittest from modules.sfp_fortinet import sfp_fortinet from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleFortinet(unittest.TestCase): def test_opts(self): module = sfp_fortinet() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_fortinet() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_fortinet() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_fortinet() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_fraudguard.py ================================================ import pytest import unittest from modules.sfp_fraudguard import sfp_fraudguard from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleFraudguard(unittest.TestCase): def test_opts(self): module = sfp_fraudguard() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_fraudguard() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_fraudguard() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_fraudguard() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_fraudguard() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_fsecure_riddler.py ================================================ import pytest import unittest from modules.sfp_fsecure_riddler import sfp_fsecure_riddler from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleFsecureRiddler(unittest.TestCase): def test_opts(self): module = sfp_fsecure_riddler() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_fsecure_riddler() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_fsecure_riddler() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_fsecure_riddler() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_fsecure_riddler() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_fullcontact.py ================================================ import pytest import unittest from modules.sfp_fullcontact import sfp_fullcontact from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleFullcontact(unittest.TestCase): def test_opts(self): module = sfp_fullcontact() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_fullcontact() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_fullcontact() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_fullcontact() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_fullcontact() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_fullhunt.py ================================================ import pytest import unittest from modules.sfp_fullhunt import sfp_fullhunt from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleFullhunt(unittest.TestCase): def test_opts(self): module = sfp_fullhunt() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_fullhunt() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_fullhunt() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_fullhunt() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "400", "404"] for code in http_codes: with self.subTest(code=code): module = sfp_fullhunt() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "403", "429"] for code in http_codes: with self.subTest(code=code): module = sfp_fullhunt() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_fullhunt() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_github.py ================================================ import pytest import unittest from modules.sfp_github import sfp_github from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleGithub(unittest.TestCase): def test_opts(self): module = sfp_github() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_github() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_github() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_github() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_social_media_not_github_profile_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_github() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_github) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'SOCIAL_MEDIA' event_data = 'Not GitHub: example_username' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_gleif.py ================================================ import pytest import unittest from modules.sfp_gleif import sfp_gleif from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleGleif(unittest.TestCase): def test_opts(self): module = sfp_gleif() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_gleif() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_gleif() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_gleif() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_invalid_lei_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_gleif() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_gleif) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'LEI' event_data = 'invalid LEI' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_google_tag_manager.py ================================================ import pytest import unittest from modules.sfp_google_tag_manager import sfp_google_tag_manager from sflib import SpiderFoot @pytest.mark.usefixtures class TestModulesGoogleTagManager(unittest.TestCase): def test_opts(self): module = sfp_google_tag_manager() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_google_tag_manager() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_google_tag_manager() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_google_tag_manager() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_googlemaps.py ================================================ import pytest import unittest from modules.sfp_googlemaps import sfp_googlemaps from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleGoogleMaps(unittest.TestCase): def test_opts(self): module = sfp_googlemaps() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_googlemaps() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_googlemaps() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_googlemaps() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_googlemaps() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_googleobjectstorage.py ================================================ import pytest import unittest from modules.sfp_googleobjectstorage import sfp_googleobjectstorage from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleGoogleobjectstorage(unittest.TestCase): def test_opts(self): module = sfp_googleobjectstorage() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_googleobjectstorage() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_googleobjectstorage() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_googleobjectstorage() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_googlesafebrowsing.py ================================================ import pytest import unittest from modules.sfp_googlesafebrowsing import sfp_googlesafebrowsing from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleGoogleSafebrowsing(unittest.TestCase): def test_opts(self): module = sfp_googlesafebrowsing() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_googlesafebrowsing() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_googlesafebrowsing() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_googlesafebrowsing() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_googlesafebrowsing() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_googlesearch.py ================================================ import pytest import unittest from modules.sfp_googlesearch import sfp_googlesearch from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleGoogleSearch(unittest.TestCase): def test_opts(self): module = sfp_googlesearch() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_googlesearch() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_googlesearch() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_googlesearch() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_googlesearch() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_gravatar.py ================================================ import pytest import unittest from modules.sfp_gravatar import sfp_gravatar from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleGravatar(unittest.TestCase): def test_opts(self): module = sfp_gravatar() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_gravatar() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_gravatar() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_gravatar() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_grayhatwarfare.py ================================================ import pytest import unittest from modules.sfp_grayhatwarfare import sfp_grayhatwarfare from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleGrayhatWarfare(unittest.TestCase): def test_opts(self): module = sfp_grayhatwarfare() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_grayhatwarfare() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_grayhatwarfare() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_grayhatwarfare() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_grayhatwarfare() module.setup(sf, dict()) target_value = 'example target value' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_greensnow.py ================================================ import pytest import unittest from modules.sfp_greensnow import sfp_greensnow from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleGreensnow(unittest.TestCase): def test_opts(self): module = sfp_greensnow() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_greensnow() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_greensnow() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_greensnow() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_grep_app.py ================================================ import pytest import unittest from modules.sfp_grep_app import sfp_grep_app from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleGrepApp(unittest.TestCase): def test_opts(self): module = sfp_grep_app() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_grep_app() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_grep_app() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_grep_app() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_greynoise.py ================================================ import pytest import unittest from modules.sfp_greynoise import sfp_greynoise from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleGreynoise(unittest.TestCase): def test_opts(self): module = sfp_greynoise() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_greynoise() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_greynoise() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_greynoise() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_greynoise() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_h1nobbdde.py ================================================ import pytest import unittest from modules.sfp_h1nobbdde import sfp_h1nobbdde from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleH1nobbdde(unittest.TestCase): def test_opts(self): module = sfp_h1nobbdde() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_h1nobbdde() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_h1nobbdde() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_h1nobbdde() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_hackertarget.py ================================================ import pytest import unittest from modules.sfp_hackertarget import sfp_hackertarget from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleHackertarget(unittest.TestCase): def test_opts(self): module = sfp_hackertarget() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_hackertarget() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_hackertarget() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_hackertarget() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_hashes.py ================================================ import pytest import unittest from modules.sfp_hashes import sfp_hashes from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleHashes(unittest.TestCase): def test_opts(self): module = sfp_hashes() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_hashes() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_hashes() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_hashes() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_containing_hashes_string_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_hashes() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'HASH' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "[MD5] e17cff4eb3e8fbe6ca3b83fb47532dba" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_hashes) event_type = 'ROOT' event_data = 'example data e17cff4eb3e8fbe6ca3b83fb47532dba example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_not_containing_hashes_string_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_hashes() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_hashes) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_haveibeenpwned.py ================================================ import pytest import unittest from modules.sfp_haveibeenpwned import sfp_haveibeenpwned from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleHaveibeenpwned(unittest.TestCase): def test_opts(self): module = sfp_haveibeenpwned() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_haveibeenpwned() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_haveibeenpwned() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_haveibeenpwned() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_haveibeenpwned() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_honeypot.py ================================================ import pytest import unittest from modules.sfp_honeypot import sfp_honeypot from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleHoneypot(unittest.TestCase): def test_opts(self): module = sfp_honeypot() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_honeypot() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_honeypot() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_honeypot() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_honeypot() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_hosting.py ================================================ import pytest import unittest from modules.sfp_hosting import sfp_hosting from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleHosting(unittest.TestCase): def test_opts(self): module = sfp_hosting() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_hosting() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_hosting() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_hosting() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_hostio.py ================================================ import pytest import unittest from modules.sfp_hostio import sfp_hostio from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleHostio(unittest.TestCase): def test_opts(self): module = sfp_hostio() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_hostio() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_hostio() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_hostio() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_hostio() module.setup(sf, dict()) target_value = 'example target value' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_hunter.py ================================================ import pytest import unittest from modules.sfp_hunter import sfp_hunter from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleHunter(unittest.TestCase): def test_opts(self): module = sfp_hunter() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_hunter() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_hunter() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_hunter() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_hunter() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_hybrid_analysis.py ================================================ import pytest import unittest from modules.sfp_hybrid_analysis import sfp_hybrid_analysis from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleHybridAnalysis(unittest.TestCase): def test_opts(self): module = sfp_hybrid_analysis() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_hybrid_analysis() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_hybrid_analysis() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_hybrid_analysis() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "400"] for code in http_codes: with self.subTest(code=code): module = sfp_hybrid_analysis() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_hybrid_analysis() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_iban.py ================================================ import pytest import unittest from modules.sfp_iban import sfp_iban from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIban(unittest.TestCase): def test_opts(self): module = sfp_iban() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_iban() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_iban() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_iban() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_containing_iban_string_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_iban() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'IBAN_NUMBER' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'BE71096123456769' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_iban) event_type = 'ROOT' event_data = 'BE71096123456769' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_not_containing_iban_string_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_iban() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_iban) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_iknowwhatyoudownload.py ================================================ import pytest import unittest from modules.sfp_iknowwhatyoudownload import sfp_iknowwhatyoudownload from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIknowwhatyoudownload(unittest.TestCase): def test_opts(self): module = sfp_iknowwhatyoudownload() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_iknowwhatyoudownload() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_iknowwhatyoudownload() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_iknowwhatyoudownload() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_iknowwhatyoudownload() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_intelx.py ================================================ import pytest import unittest from modules.sfp_intelx import sfp_intelx from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntelx(unittest.TestCase): def test_opts(self): module = sfp_intelx() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_intelx() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_intelx() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_intelx() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_intelx() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_intfiles.py ================================================ import pytest import unittest from modules.sfp_intfiles import sfp_intfiles from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIntfiles(unittest.TestCase): def test_opts(self): module = sfp_intfiles() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_intfiles() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_intfiles() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_intfiles() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_internal_url_with_interesting_file_extension_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_intfiles() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'INTERESTING_FILE' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'https://spiderfoot.net/example.zip' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_intfiles) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'LINKED_URL_INTERNAL' event_data = 'https://spiderfoot.net/example.zip' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_internal_url_without_interesting_file_extension_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_intfiles() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_intfiles) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'LINKED_URL_INTERNAL' event_data = 'https://spiderfoot.net/example' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_ipapico.py ================================================ import pytest import unittest from modules.sfp_ipapico import sfp_ipapico from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleipApico(unittest.TestCase): def test_opts(self): module = sfp_ipapico() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_ipapico() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_ipapico() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_ipapico() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_ipapicom.py ================================================ import pytest import unittest from modules.sfp_ipapicom import sfp_ipapicom from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleipapicom(unittest.TestCase): def test_opts(self): module = sfp_ipapicom() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_ipapicom() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_ipapicom() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_ipapicom() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_ipapicom() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_ipinfo.py ================================================ import pytest import unittest from modules.sfp_ipinfo import sfp_ipinfo from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIpinfo(unittest.TestCase): def test_opts(self): module = sfp_ipinfo() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_ipinfo() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_ipinfo() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_ipinfo() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_ipinfo() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_ipqualityscore.py ================================================ import pytest import unittest from modules.sfp_ipqualityscore import sfp_ipqualityscore from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIpqualityscore(unittest.TestCase): def test_opts(self): module = sfp_ipqualityscore() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_ipqualityscore() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_ipqualityscore() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_ipqualityscore() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_ipqualityscore() module.setup(sf, dict()) target_value = 'example target value' target_type = 'PHONE_NUMBER' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_ipregistry.py ================================================ import pytest import unittest from modules.sfp_ipregistry import sfp_ipregistry from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIpRegistry(unittest.TestCase): def test_opts(self): module = sfp_ipregistry() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_ipregistry() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_ipregistry() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_ipregistry() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_ipregistry() module.setup(sf, dict()) target_value = "example target value" target_type = "IP_ADDRESS" target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = "ROOT" event_data = "example data" event_module = "" source_event = "" evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_ipstack.py ================================================ import pytest import unittest from modules.sfp_ipstack import sfp_ipstack from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleIpstack(unittest.TestCase): def test_opts(self): module = sfp_ipstack() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_ipstack() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_ipstack() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_ipstack() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_ipstack() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_isc.py ================================================ import pytest import unittest from modules.sfp_isc import sfp_isc from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleisc(unittest.TestCase): def test_opts(self): module = sfp_isc() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_isc() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_isc() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_isc() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_jsonwhoiscom.py ================================================ import pytest import unittest from modules.sfp_jsonwhoiscom import sfp_jsonwhoiscom from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleJsonwhoiscom(unittest.TestCase): def test_opts(self): module = sfp_jsonwhoiscom() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_jsonwhoiscom() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_jsonwhoiscom() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_jsonwhoiscom() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "404"] for code in http_codes: with self.subTest(code=code): module = sfp_jsonwhoiscom() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_jsonwhoiscom() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_jsonwhoiscom() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_junkfiles.py ================================================ import pytest import unittest from modules.sfp_junkfiles import sfp_junkfiles from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleJunkfiles(unittest.TestCase): def test_opts(self): module = sfp_junkfiles() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_junkfiles() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_junkfiles() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_junkfiles() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_keybase.py ================================================ import pytest import unittest from modules.sfp_keybase import sfp_keybase from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleKeybase(unittest.TestCase): def test_opts(self): module = sfp_keybase() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_keybase() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_keybase() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_keybase() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_koodous.py ================================================ import pytest import unittest from modules.sfp_koodous import sfp_koodous from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleKoodous(unittest.TestCase): def test_opts(self): module = sfp_koodous() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_koodous() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_koodous() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_koodous() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "404"] for code in http_codes: with self.subTest(code=code): module = sfp_koodous() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_koodous() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_koodous() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_leakix.py ================================================ import pytest import unittest from modules.sfp_leakix import sfp_leakix from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleLeakix(unittest.TestCase): def test_opts(self): module = sfp_leakix() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_leakix() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_leakix() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_leakix() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "404"] for code in http_codes: with self.subTest(code=code): module = sfp_leakix() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_leakix() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_maltiverse.py ================================================ import pytest import unittest from modules.sfp_maltiverse import sfp_maltiverse from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleMaltiverse(unittest.TestCase): def test_opts(self): module = sfp_maltiverse() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_maltiverse() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_maltiverse() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_maltiverse() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_malwarepatrol.py ================================================ import pytest import unittest from modules.sfp_malwarepatrol import sfp_malwarepatrol from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModulemalwarepatrol(unittest.TestCase): def test_opts(self): module = sfp_malwarepatrol() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_malwarepatrol() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_malwarepatrol() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_malwarepatrol() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_malwarepatrol() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_metadefender.py ================================================ import pytest import unittest from modules.sfp_metadefender import sfp_metadefender from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleMetadefender(unittest.TestCase): def test_opts(self): module = sfp_metadefender() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_metadefender() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_metadefender() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_metadefender() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "404"] for code in http_codes: with self.subTest(code=code): module = sfp_metadefender() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) # TODO http_codes = ["401", "402", "403", "429", "500", "502", "503"] http_codes = ["401", "429"] for code in http_codes: with self.subTest(code=code): module = sfp_metadefender() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_metadefender() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_mnemonic.py ================================================ import pytest import unittest from modules.sfp_mnemonic import sfp_mnemonic from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleMnemonic(unittest.TestCase): def test_opts(self): module = sfp_mnemonic() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_mnemonic() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_mnemonic() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_mnemonic() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_multiproxy.py ================================================ import pytest import unittest from modules.sfp_multiproxy import sfp_multiproxy from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleMultiproxy(unittest.TestCase): def test_opts(self): module = sfp_multiproxy() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_multiproxy() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_multiproxy() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_multiproxy() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_myspace.py ================================================ import pytest import unittest from modules.sfp_myspace import sfp_myspace from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleMyspace(unittest.TestCase): def test_opts(self): module = sfp_myspace() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_myspace() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_myspace() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_myspace() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_social_media_not_myspace_profile_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_myspace() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_myspace) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'SOCIAL_MEDIA' event_data = 'Not MySpace: example_username' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_nameapi.py ================================================ import pytest import unittest from modules.sfp_nameapi import sfp_nameapi from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleNameapi(unittest.TestCase): def test_opts(self): module = sfp_nameapi() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_nameapi() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_nameapi() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_nameapi() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_nameapi() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_names.py ================================================ import pytest import unittest from modules.sfp_names import sfp_names from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleNames(unittest.TestCase): def test_opts(self): module = sfp_names() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_names() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_names() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_names() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_email_address_containing_human_names_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_names() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'HUMAN_NAME' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "Firstname Lastname" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_names) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'EMAILADDR' event_data = 'firstname.lastname@spiderfoot.net' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_email_address_containing_human_names_containing_numbers_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_names() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_names) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'EMAILADDR' event_data = 'firstname.lastname1@spiderfoot.net' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) def test_handleEvent_event_data_email_address_not_containing_names_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_names() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_names) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'EMAILADDR' event_data = 'lastname@spiderfoot.net' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_networksdb.py ================================================ import pytest import unittest from modules.sfp_networksdb import sfp_networksdb from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleNetworksdb(unittest.TestCase): def test_opts(self): module = sfp_networksdb() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_networksdb() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_networksdb() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_networksdb() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "400"] for code in http_codes: with self.subTest(code=code): module = sfp_networksdb() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) # TODO: http_codes = ["401", "402", "403", "429", "500", "502", "503"] http_codes = ["403", "429"] for code in http_codes: with self.subTest(code=code): module = sfp_networksdb() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_networksdb() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_neutrinoapi.py ================================================ import pytest import unittest from modules.sfp_neutrinoapi import sfp_neutrinoapi from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleNeutrinoapi(unittest.TestCase): def test_opts(self): module = sfp_neutrinoapi() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_neutrinoapi() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_neutrinoapi() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_neutrinoapi() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "400"] for code in http_codes: with self.subTest(code=code): module = sfp_neutrinoapi() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) # TODO: http_codes = ["401", "402", "403", "429", "500", "502", "503"] http_codes = ["403"] for code in http_codes: with self.subTest(code=code): module = sfp_neutrinoapi() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_neutrinoapi() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_numverify.py ================================================ import pytest import unittest from modules.sfp_numverify import sfp_numverify from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleNumverify(unittest.TestCase): def test_opts(self): module = sfp_numverify() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_numverify() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_numverify() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_numverify() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_numverify() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_onioncity.py ================================================ import pytest import unittest from modules.sfp_onioncity import sfp_onioncity from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleonioncity(unittest.TestCase): def test_opts(self): module = sfp_onioncity() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_onioncity() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_onioncity() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_onioncity() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_onioncity() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_onionsearchengine.py ================================================ import pytest import unittest from modules.sfp_onionsearchengine import sfp_onionsearchengine from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleOnionsearchengine(unittest.TestCase): def test_opts(self): module = sfp_onionsearchengine() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_onionsearchengine() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_onionsearchengine() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_onionsearchengine() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_onyphe.py ================================================ import pytest import unittest from modules.sfp_onyphe import sfp_onyphe from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleOnyphe(unittest.TestCase): def test_opts(self): module = sfp_onyphe() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_onyphe() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_onyphe() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_onyphe() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_onyphe() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_openbugbounty.py ================================================ import pytest import unittest from modules.sfp_openbugbounty import sfp_openbugbounty from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleOpenbugbounty(unittest.TestCase): def test_opts(self): module = sfp_openbugbounty() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_openbugbounty() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_openbugbounty() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_openbugbounty() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_opencorporates.py ================================================ import pytest import unittest from modules.sfp_opencorporates import sfp_opencorporates from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleOpencorporates(unittest.TestCase): def test_opts(self): module = sfp_opencorporates() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_opencorporates() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_opencorporates() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_opencorporates() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_opendns.py ================================================ import pytest import unittest from modules.sfp_opendns import sfp_opendns from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleOpendns(unittest.TestCase): def test_opts(self): module = sfp_opendns() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_opendns() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_opendns() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_opendns() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_opennic.py ================================================ import pytest import unittest from modules.sfp_opennic import sfp_opennic from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleOpenNic(unittest.TestCase): def test_opts(self): module = sfp_opennic() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_opennic() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_opennic() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_opennic() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_openphish.py ================================================ import pytest import unittest from modules.sfp_openphish import sfp_openphish from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleOpenphish(unittest.TestCase): def test_opts(self): module = sfp_openphish() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_openphish() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_openphish() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_openphish() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_openstreetmap.py ================================================ import pytest import unittest from modules.sfp_openstreetmap import sfp_openstreetmap from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleopenstreetmap(unittest.TestCase): def test_opts(self): module = sfp_openstreetmap() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_openstreetmap() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_openstreetmap() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_openstreetmap() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_pageinfo.py ================================================ import pytest import unittest from modules.sfp_pageinfo import sfp_pageinfo from sflib import SpiderFoot @pytest.mark.usefixtures class TestModulePageInfo(unittest.TestCase): def test_opts(self): module = sfp_pageinfo() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_pageinfo() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_pageinfo() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_pageinfo() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_pastebin.py ================================================ import pytest import unittest from modules.sfp_pastebin import sfp_pastebin from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModulePastebin(unittest.TestCase): def test_opts(self): module = sfp_pastebin() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_pastebin() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_pastebin() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_pastebin() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_pastebin() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_pgp.py ================================================ import pytest import unittest from modules.sfp_pgp import sfp_pgp from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModulePgp(unittest.TestCase): def test_opts(self): module = sfp_pgp() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_pgp() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_pgp() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_pgp() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_keyserver_urls_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_pgp() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) module.opts['keyserver_search1'] = '' module.opts['keyserver_search2'] = '' module.opts['keyserver_fetch1'] = '' module.opts['keyserver_fetch2'] = '' result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_phishstats.py ================================================ import pytest import unittest from modules.sfp_phishstats import sfp_phishstats from sflib import SpiderFoot @pytest.mark.usefixtures class TestModulePhishstats(unittest.TestCase): def test_opts(self): module = sfp_phishstats() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_phishstats() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_phishstats() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_phishstats() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_phishtank.py ================================================ import pytest import unittest from modules.sfp_phishtank import sfp_phishtank from sflib import SpiderFoot @pytest.mark.usefixtures class TestModulePhishtank(unittest.TestCase): def test_opts(self): module = sfp_phishtank() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_phishtank() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_phishtank() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_phishtank() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_phone.py ================================================ import pytest import unittest from modules.sfp_phone import sfp_phone from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModulePhone(unittest.TestCase): def test_opts(self): module = sfp_phone() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_phone() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_phone() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_phone() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_domain_whois_event_data_containing_phone_string_should_create_phone_number_event(self): sf = SpiderFoot(self.default_options) module = sfp_phone() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'PHONE_NUMBER' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "+12025550111" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_phone) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'DOMAIN_WHOIS' event_data = 'example data +1 202 555 0111 example data' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_domain_whois_event_data_not_containing_phone_string_should_not_create_event(self): sf = SpiderFoot(self.default_options) module = sfp_phone() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_phone) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'DOMAIN_WHOIS' event_data = 'example data' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) def test_handleEvent_phone_number_event_data_containing_phone_string_should_return_provider_telco_event(self): sf = SpiderFoot(self.default_options) module = sfp_phone() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'PROVIDER_TELCO' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "Swisscom" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_phone) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'PHONE_NUMBER' event_data = '+41798765432' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) ================================================ FILE: test/unit/modules/test_sfp_portscan_tcp.py ================================================ import pytest import unittest from modules.sfp_portscan_tcp import sfp_portscan_tcp from sflib import SpiderFoot @pytest.mark.usefixtures class TestModulePortscanTcp(unittest.TestCase): def test_opts(self): module = sfp_portscan_tcp() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_portscan_tcp() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_portscan_tcp() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_portscan_tcp() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_projectdiscovery.py ================================================ import pytest import unittest from modules.sfp_projectdiscovery import sfp_projectdiscovery from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleProjectdiscovery(unittest.TestCase): def test_opts(self): module = sfp_projectdiscovery() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_projectdiscovery() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_projectdiscovery() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_projectdiscovery() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_projectdiscovery() module.setup(sf, dict()) target_value = "example target value" target_type = "IP_ADDRESS" target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = "ROOT" event_data = "example data" event_module = "" source_event = "" evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_psbdmp.py ================================================ import pytest import unittest from modules.sfp_psbdmp import sfp_psbdmp from sflib import SpiderFoot @pytest.mark.usefixtures class TestModulePsbdmp(unittest.TestCase): def test_opts(self): module = sfp_psbdmp() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_psbdmp() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_psbdmp() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_psbdmp() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_pulsedive.py ================================================ import pytest import unittest from modules.sfp_pulsedive import sfp_pulsedive from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModulePulsedive(unittest.TestCase): def test_opts(self): module = sfp_pulsedive() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_pulsedive() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_pulsedive() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_pulsedive() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_pulsedive() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_punkspider.py ================================================ import pytest import unittest from modules.sfp_punkspider import sfp_punkspider from sflib import SpiderFoot @pytest.mark.usefixtures class TestModulePunkspider(unittest.TestCase): def test_opts(self): module = sfp_punkspider() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_punkspider() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_punkspider() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_punkspider() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "404"] for code in http_codes: with self.subTest(code=code): module = sfp_punkspider() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_punkspider() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_quad9.py ================================================ import pytest import unittest from modules.sfp_quad9 import sfp_quad9 from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleQuad9(unittest.TestCase): def test_opts(self): module = sfp_quad9() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_quad9() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_quad9() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_quad9() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_reversewhois.py ================================================ import pytest import unittest from modules.sfp_reversewhois import sfp_reversewhois from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleReversewhois(unittest.TestCase): def test_opts(self): module = sfp_reversewhois() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_reversewhois() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_reversewhois() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_reversewhois() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_ripe.py ================================================ import pytest import unittest from modules.sfp_ripe import sfp_ripe from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleRipe(unittest.TestCase): def test_opts(self): module = sfp_ripe() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_ripe() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_ripe() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_ripe() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_riskiq.py ================================================ import pytest import unittest from modules.sfp_riskiq import sfp_riskiq from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleRiskiq(unittest.TestCase): def test_opts(self): module = sfp_riskiq() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_riskiq() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_riskiq() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_riskiq() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_riskiq() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_robtex.py ================================================ import pytest import unittest from modules.sfp_robtex import sfp_robtex from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleRobtex(unittest.TestCase): def test_opts(self): module = sfp_robtex() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_robtex() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_robtex() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_robtex() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_s3bucket.py ================================================ import pytest import unittest from modules.sfp_s3bucket import sfp_s3bucket from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleS3bucket(unittest.TestCase): def test_opts(self): module = sfp_s3bucket() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_s3bucket() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_s3bucket() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_s3bucket() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_searchcode.py ================================================ import pytest import unittest from modules.sfp_searchcode import sfp_searchcode from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleCodesearch(unittest.TestCase): def test_opts(self): module = sfp_searchcode() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_searchcode() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_searchcode() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_searchcode() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_securitytrails.py ================================================ import pytest import unittest from modules.sfp_securitytrails import sfp_securitytrails from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleSecuritytrails(unittest.TestCase): def test_opts(self): module = sfp_securitytrails() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_securitytrails() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_securitytrails() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_securitytrails() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_securitytrails() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_seon.py ================================================ import pytest import unittest from modules.sfp_seon import sfp_seon from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleSeon(unittest.TestCase): def test_opts(self): module = sfp_seon() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_seon() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_seon() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_seon() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_seon() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_shodan.py ================================================ import pytest import unittest from modules.sfp_shodan import sfp_shodan from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleShodan(unittest.TestCase): def test_opts(self): module = sfp_shodan() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_shodan() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_shodan() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_shodan() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_shodan() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_similar.py ================================================ import pytest import unittest from modules.sfp_similar import sfp_similar from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleSimilar(unittest.TestCase): def test_opts(self): module = sfp_similar() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_similar() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_similar() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_similar() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_skymem.py ================================================ import pytest import unittest from modules.sfp_skymem import sfp_skymem from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleSkymem(unittest.TestCase): def test_opts(self): module = sfp_skymem() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_skymem() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_skymem() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_skymem() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_slideshare.py ================================================ import pytest import unittest from modules.sfp_slideshare import sfp_slideshare from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleSlideshare(unittest.TestCase): def test_opts(self): module = sfp_slideshare() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_slideshare() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_slideshare() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_slideshare() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_social_media_not_slideshare_profile_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_slideshare() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_slideshare) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'SOCIAL_MEDIA' event_data = 'Not SlideShare: example_username' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_snov.py ================================================ import pytest import unittest from modules.sfp_snov import sfp_snov from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleSnov(unittest.TestCase): def test_opts(self): module = sfp_snov() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_snov() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_snov() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_snov() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_snov() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_social.py ================================================ import pytest import unittest from modules.sfp_social import sfp_social from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleSocial(unittest.TestCase): def test_opts(self): module = sfp_social() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_social() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_social() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_social() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_url_containing_social_media_profile_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_social() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'SOCIAL_MEDIA' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "LinkedIn (Individual): https://linkedin.com/in/spiderfoot" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_social) event_type = 'ROOT' event_data = 'https://linkedin.com/in/spiderfoot' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_url_not_containing_social_media_profile_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_social() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_social) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_sociallinks.py ================================================ import pytest import unittest from modules.sfp_sociallinks import sfp_sociallinks from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleSociallinks(unittest.TestCase): def test_opts(self): module = sfp_sociallinks() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_sociallinks() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_sociallinks() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_sociallinks() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_sociallinks() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_socialprofiles.py ================================================ import pytest import unittest from modules.sfp_socialprofiles import sfp_socialprofiles from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleSocialprofiles(unittest.TestCase): def test_opts(self): module = sfp_socialprofiles() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_socialprofiles() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_socialprofiles() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_socialprofiles() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_socialprofiles() module.setup(sf, dict()) target_value = 'example target value' target_type = 'HUMAN_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_sorbs.py ================================================ import pytest import unittest from modules.sfp_sorbs import sfp_sorbs from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleSorbs(unittest.TestCase): def test_opts(self): module = sfp_sorbs() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_sorbs() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_sorbs() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_sorbs() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_spamcop.py ================================================ import pytest import unittest from modules.sfp_spamcop import sfp_spamcop from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleSpamcop(unittest.TestCase): def test_opts(self): module = sfp_spamcop() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_spamcop() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_spamcop() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_spamcop() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_spamhaus.py ================================================ import pytest import unittest from modules.sfp_spamhaus import sfp_spamhaus from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleSpamhaus(unittest.TestCase): def test_opts(self): module = sfp_spamhaus() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_spamhaus() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_spamhaus() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_spamhaus() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_spider.py ================================================ import pytest import unittest from modules.sfp_spider import sfp_spider from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleSpider(unittest.TestCase): def test_opts(self): module = sfp_spider() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_spider() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_spider() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_spider() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_spur.py ================================================ import pytest import unittest from modules.sfp_spur import sfp_spur from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleSpur(unittest.TestCase): def test_opts(self): module = sfp_spur() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_spur() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_spur() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_spur() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_spur() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_spyonweb.py ================================================ import pytest import unittest from modules.sfp_spyonweb import sfp_spyonweb from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleSpyonweb(unittest.TestCase): def test_opts(self): module = sfp_spyonweb() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_spyonweb() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_spyonweb() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_spyonweb() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_spyonweb() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_sslcert.py ================================================ import pytest import unittest from modules.sfp_sslcert import sfp_sslcert from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleSslCert(unittest.TestCase): def test_opts(self): module = sfp_sslcert() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_sslcert() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_sslcert() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_sslcert() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_stackoverflow.py ================================================ import pytest import unittest from modules.sfp_stackoverflow import sfp_stackoverflow from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleStackoverflow(unittest.TestCase): def test_opts(self): module = sfp_stackoverflow() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_stackoverflow() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_stackoverflow() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_stackoverflow() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_stevenblack_hosts.py ================================================ import pytest import unittest from modules.sfp_stevenblack_hosts import sfp_stevenblack_hosts from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleStevenblackHosts(unittest.TestCase): def test_opts(self): module = sfp_stevenblack_hosts() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_stevenblack_hosts() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_stevenblack_hosts() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_stevenblack_hosts() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_strangeheaders.py ================================================ import pytest import unittest from modules.sfp_strangeheaders import sfp_strangeheaders from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleStrangeHeaders(unittest.TestCase): def test_opts(self): module = sfp_strangeheaders() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_strangeheaders() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_strangeheaders() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_strangeheaders() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_containing_unusual_header_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_strangeheaders() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'WEBSERVER_STRANGEHEADER' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = 'unusual header: example header value' if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_strangeheaders) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'WEBSERVER_HTTPHEADERS' event_data = '{"unusual header": "example header value"}' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = "https://spiderfoot.net/example" with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_not_containing_unusual_header_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_strangeheaders() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_strangeheaders) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'WEBSERVER_HTTPHEADERS' event_data = '{"server": "example server"}' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = "https://spiderfoot.net/example" result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_subdomain_takeover.py ================================================ import pytest import unittest from modules.sfp_subdomain_takeover import sfp_subdomain_takeover from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleSubdomain_takeover(unittest.TestCase): def test_opts(self): module = sfp_subdomain_takeover() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_subdomain_takeover() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_subdomain_takeover() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_subdomain_takeover() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_sublist3r.py ================================================ import pytest import unittest from modules.sfp_sublist3r import sfp_sublist3r from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleSublist3r(unittest.TestCase): def test_opts(self): module = sfp_sublist3r() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_sublist3r() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_sublist3r() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_sublist3r() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_surbl.py ================================================ import pytest import unittest from modules.sfp_surbl import sfp_surbl from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleSurbl(unittest.TestCase): def test_opts(self): module = sfp_surbl() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_surbl() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_surbl() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_surbl() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_talosintel.py ================================================ import pytest import unittest from modules.sfp_talosintel import sfp_talosintel from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleTalosintel(unittest.TestCase): def test_opts(self): module = sfp_talosintel() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_talosintel() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_talosintel() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_talosintel() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_template.py ================================================ import pytest import unittest from modules.sfp_template import sfp_template from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleTemplate(unittest.TestCase): def test_opts(self): module = sfp_template() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_template() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_template() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_template() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_textmagic.py ================================================ import pytest import unittest from modules.sfp_textmagic import sfp_textmagic from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleTextmagic(unittest.TestCase): def test_opts(self): module = sfp_textmagic() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_textmagic() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_textmagic() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_textmagic() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_textmagic() module.setup(sf, dict()) target_value = 'example target value' target_type = 'PHONE_NUMBER' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_threatcrowd.py ================================================ import pytest import unittest from modules.sfp_threatcrowd import sfp_threatcrowd from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleThreatcrowd(unittest.TestCase): def test_opts(self): module = sfp_threatcrowd() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_threatcrowd() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_threatcrowd() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_threatcrowd() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_threatfox.py ================================================ import pytest import unittest from modules.sfp_threatfox import sfp_threatfox from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleThreatFox(unittest.TestCase): def test_opts(self): module = sfp_threatfox() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_threatfox() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_threatfox() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_threatfox() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_threatminer.py ================================================ import pytest import unittest from modules.sfp_threatminer import sfp_threatminer from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleThreatminer(unittest.TestCase): def test_opts(self): module = sfp_threatminer() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_threatminer() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_threatminer() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_threatminer() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_tldsearch.py ================================================ import pytest import unittest from modules.sfp_tldsearch import sfp_tldsearch from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleTldsearch(unittest.TestCase): def test_opts(self): module = sfp_tldsearch() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tldsearch() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tldsearch() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tldsearch() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_tool_cmseek.py ================================================ import pytest import unittest from modules.sfp_tool_cmseek import sfp_tool_cmseek from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleToolCmseek(unittest.TestCase): def test_opts(self): module = sfp_tool_cmseek() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_cmseek() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_cmseek() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_cmseek() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_tool_path_configured_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_tool_cmseek() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_tool_dnstwist.py ================================================ import pytest import unittest from modules.sfp_tool_dnstwist import sfp_tool_dnstwist from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleToolDnstwist(unittest.TestCase): def test_opts(self): module = sfp_tool_dnstwist() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_dnstwist() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_dnstwist() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_dnstwist() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_tool_nbtscan.py ================================================ import pytest import unittest from modules.sfp_tool_nbtscan import sfp_tool_nbtscan from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleToolNbtscan(unittest.TestCase): def test_opts(self): module = sfp_tool_nbtscan() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_nbtscan() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_nbtscan() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_nbtscan() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_tool_path_configured_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_tool_nbtscan() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_tool_nmap.py ================================================ import pytest import unittest from modules.sfp_tool_nmap import sfp_tool_nmap from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleToolNmap(unittest.TestCase): def test_opts(self): module = sfp_tool_nmap() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_nmap() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_nmap() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_nmap() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_tool_path_configured_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_tool_nmap() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_tool_nuclei.py ================================================ import pytest import unittest from modules.sfp_tool_nuclei import sfp_tool_nuclei from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleToolNuclei(unittest.TestCase): def test_opts(self): module = sfp_tool_nuclei() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_nuclei() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_nuclei() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_nuclei() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_tool_path_configured_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_tool_nuclei() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_tool_onesixtyone.py ================================================ import pytest import unittest from modules.sfp_tool_onesixtyone import sfp_tool_onesixtyone from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleToolOnesixtyone(unittest.TestCase): def test_opts(self): module = sfp_tool_onesixtyone() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_onesixtyone() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_onesixtyone() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_onesixtyone() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_tool_path_configured_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_tool_onesixtyone() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_tool_retirejs.py ================================================ import pytest import unittest from modules.sfp_tool_retirejs import sfp_tool_retirejs from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleToolRetirejs(unittest.TestCase): def test_opts(self): module = sfp_tool_retirejs() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_retirejs() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_retirejs() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_retirejs() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_tool_path_configured_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_tool_retirejs() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_tool_snallygaster.py ================================================ import pytest import unittest from modules.sfp_tool_snallygaster import sfp_tool_snallygaster from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleToolSnallygaster(unittest.TestCase): def test_opts(self): module = sfp_tool_snallygaster() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_snallygaster() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_snallygaster() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_snallygaster() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_tool_path_configured_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_tool_snallygaster() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_tool_testsslsh.py ================================================ import pytest import unittest from modules.sfp_tool_testsslsh import sfp_tool_testsslsh from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleToolTestsslsh(unittest.TestCase): def test_opts(self): module = sfp_tool_testsslsh() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_testsslsh() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_testsslsh() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_testsslsh() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_tool_path_configured_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_tool_testsslsh() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_tool_trufflehog.py ================================================ import pytest import unittest from modules.sfp_tool_trufflehog import sfp_tool_trufflehog from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleToolTrufflehog(unittest.TestCase): def test_opts(self): module = sfp_tool_trufflehog() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_trufflehog() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_trufflehog() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_trufflehog() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_tool_path_configured_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_tool_trufflehog() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example value' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_tool_wafw00f.py ================================================ import pytest import unittest from modules.sfp_tool_wafw00f import sfp_tool_wafw00f from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleToolWafw00f(unittest.TestCase): def test_opts(self): module = sfp_tool_wafw00f() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_wafw00f() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_wafw00f() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_wafw00f() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_tool_path_configured_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_tool_wafw00f() module.setup(sf, dict()) target_value = 'example target value' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_tool_wappalyzer.py ================================================ import pytest import unittest from modules.sfp_tool_wappalyzer import sfp_tool_wappalyzer from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleWappalyzer(unittest.TestCase): def test_opts(self): module = sfp_tool_wappalyzer() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_wappalyzer() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_wappalyzer() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_wappalyzer() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_tool_path_configured_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_tool_wappalyzer() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_tool_whatweb.py ================================================ import pytest import unittest from modules.sfp_tool_whatweb import sfp_tool_whatweb from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleToolWhatweb(unittest.TestCase): def test_opts(self): module = sfp_tool_whatweb() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_tool_whatweb() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_tool_whatweb() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_tool_whatweb() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_tool_path_configured_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_tool_whatweb() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_torch.py ================================================ import pytest import unittest from modules.sfp_torch import sfp_torch from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleTorch(unittest.TestCase): def test_opts(self): module = sfp_torch() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_torch() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_torch() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_torch() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_torexits.py ================================================ import pytest import unittest from modules.sfp_torexits import sfp_torexits from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleTorexits(unittest.TestCase): def test_opts(self): module = sfp_torexits() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_torexits() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_torexits() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_torexits() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_trashpanda.py ================================================ import pytest import unittest from modules.sfp_trashpanda import sfp_trashpanda from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleTrashpanda(unittest.TestCase): def test_opts(self): module = sfp_trashpanda() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_trashpanda() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_trashpanda() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_trashpanda() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_trashpanda() module.setup(sf, dict()) target_value = 'example target value' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_trumail.py ================================================ import pytest import unittest from modules.sfp_trumail import sfp_trumail from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleTrumail(unittest.TestCase): def test_opts(self): module = sfp_trumail() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_trumail() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_trumail() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_trumail() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_twilio.py ================================================ import pytest import unittest from modules.sfp_twilio import sfp_twilio from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuletwilio(unittest.TestCase): def test_opts(self): module = sfp_twilio() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_twilio() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_twilio() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_twilio() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_twilio() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_twitter.py ================================================ import pytest import unittest from modules.sfp_twitter import sfp_twitter from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleTwitter(unittest.TestCase): def test_opts(self): module = sfp_twitter() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_twitter() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_twitter() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_twitter() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_social_media_not_twitter_profile_should_not_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_twitter() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_twitter) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'SOCIAL_MEDIA' event_data = 'Not Twitter: example_username' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_uceprotect.py ================================================ import pytest import unittest from modules.sfp_uceprotect import sfp_uceprotect from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleUceprotect(unittest.TestCase): def test_opts(self): module = sfp_uceprotect() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_uceprotect() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_uceprotect() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_uceprotect() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_urlscan.py ================================================ import pytest import unittest from modules.sfp_urlscan import sfp_urlscan from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleUrlscan(unittest.TestCase): def test_opts(self): module = sfp_urlscan() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_urlscan() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_urlscan() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_urlscan() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_venmo.py ================================================ import pytest import unittest from modules.sfp_venmo import sfp_venmo from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleVenmo(unittest.TestCase): def test_opts(self): module = sfp_venmo() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_venmo() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_venmo() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_venmo() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_viewdns.py ================================================ import pytest import unittest from modules.sfp_viewdns import sfp_viewdns from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModulViewdns(unittest.TestCase): def test_opts(self): module = sfp_viewdns() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_viewdns() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_viewdns() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_viewdns() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_viewdns() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_virustotal.py ================================================ import pytest import unittest from modules.sfp_virustotal import sfp_virustotal from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleVirustotal(unittest.TestCase): def test_opts(self): module = sfp_virustotal() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_virustotal() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_virustotal() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_virustotal() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_virustotal() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_voipbl.py ================================================ import pytest import unittest from modules.sfp_voipbl import sfp_voipbl from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleVoipbl(unittest.TestCase): def test_opts(self): module = sfp_voipbl() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_voipbl() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_voipbl() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_voipbl() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_vxvault.py ================================================ import pytest import unittest from modules.sfp_vxvault import sfp_vxvault from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleVxvault(unittest.TestCase): def test_opts(self): module = sfp_vxvault() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_vxvault() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_vxvault() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_vxvault() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_webanalytics.py ================================================ import pytest import unittest from modules.sfp_webanalytics import sfp_webanalytics from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleWebAnalytics(unittest.TestCase): def test_opts(self): module = sfp_webanalytics() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_webanalytics() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_webanalytics() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_webanalytics() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_target_web_content_containing_web_analytics_string_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_webanalytics() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'WEB_ANALYTICS_ID' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "Google Analytics: ua-1111111111-123" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_webanalytics) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = '

example data ua-1111111111-123 example data

' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_target_web_content_not_containing_web_analytics_string_should_not_create_event(self): sf = SpiderFoot(self.default_options) module = sfp_webanalytics() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_webanalytics) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) def test_handleEvent_event_dns_text_containing_web_analytics_string_should_return_event(self): sf = SpiderFoot(self.default_options) module = sfp_webanalytics() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'WEB_ANALYTICS_ID' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "Google Site Verification: abcdefghijklmnopqrstuvwxyz1234567890abc_def" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_webanalytics) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'DNS_TEXT' event_data = 'google-site-verification=abcdefghijklmnopqrstuvwxyz1234567890abc_def' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_dns_text_not_containing_web_analytics_string_should_not_create_event(self): sf = SpiderFoot(self.default_options) module = sfp_webanalytics() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_webanalytics) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'DNS_TEXT' event_data = 'example data' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_webframework.py ================================================ import pytest import unittest from modules.sfp_webframework import sfp_webframework from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleWebFramework(unittest.TestCase): def test_opts(self): module = sfp_webframework() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_webframework() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_webframework() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_webframework() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_event_data_web_content_containing_webframework_string_should_create_url_web_framework_event(self): sf = SpiderFoot(self.default_options) module = sfp_webframework() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): expected = 'URL_WEB_FRAMEWORK' if str(event.eventType) != expected: raise Exception(f"{event.eventType} != {expected}") expected = "Wordpress" if str(event.data) != expected: raise Exception(f"{event.data} != {expected}") raise Exception("OK") module.notifyListeners = new_notifyListeners.__get__(module, sfp_webframework) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data /wp-includes/ example data' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = "https://spiderfoot.net/" with self.assertRaises(Exception) as cm: module.handleEvent(evt) self.assertEqual("OK", str(cm.exception)) def test_handleEvent_event_data_web_content_not_containing_webframework_string_should_not_create_event(self): sf = SpiderFoot(self.default_options) module = sfp_webframework() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_webframework) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data' event_module = 'example module' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = "https://spiderfoot.net/" result = module.handleEvent(evt) self.assertIsNone(result) def test_handleEvent_event_data_web_content_from_external_url_containing_webframework_string_should_not_create_event(self): sf = SpiderFoot(self.default_options) module = sfp_webframework() module.setup(sf, dict()) target_value = 'spiderfoot.net' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) def new_notifyListeners(self, event): raise Exception(f"Raised event {event.eventType}: {event.data}") module.notifyListeners = new_notifyListeners.__get__(module, sfp_webframework) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data /wp-includes/ example data' event_module = 'sfp_spider' source_event = evt evt = SpiderFootEvent(event_type, event_data, event_module, source_event) evt.actualSource = "https://externalhost.local/" result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_webserver.py ================================================ import pytest import unittest from modules.sfp_webserver import sfp_webserver from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleWebserver(unittest.TestCase): def test_opts(self): module = sfp_webserver() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_webserver() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_webserver() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_webserver() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_whatcms.py ================================================ import pytest import unittest from modules.sfp_whatcms import sfp_whatcms from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleWhatCMS(unittest.TestCase): def test_opts(self): module = sfp_whatcms() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_whatcms() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_whatcms() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_whatcms() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200"] for code in http_codes: with self.subTest(code=code): module = sfp_whatcms() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_whatcms() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_whatcms() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_whois.py ================================================ import pytest import unittest from modules.sfp_whois import sfp_whois from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleWhois(unittest.TestCase): def test_opts(self): module = sfp_whois() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_whois() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_whois() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_whois() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_whoisology.py ================================================ import pytest import unittest from modules.sfp_whoisology import sfp_whoisology from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleWhoisology(unittest.TestCase): def test_opts(self): module = sfp_whoisology() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_whoisology() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_whoisology() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_whoisology() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_whoisology() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_whoxy.py ================================================ import pytest import unittest from modules.sfp_whoxy import sfp_whoxy from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleWhoxy(unittest.TestCase): def test_opts(self): module = sfp_whoxy() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_whoxy() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_whoxy() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_whoxy() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_whoxy() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_wigle.py ================================================ import pytest import unittest from modules.sfp_wigle import sfp_wigle from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleWigle(unittest.TestCase): def test_opts(self): module = sfp_wigle() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_wigle() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_wigle() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_wigle() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_wigle() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) ================================================ FILE: test/unit/modules/test_sfp_wikileaks.py ================================================ import pytest import unittest from modules.sfp_wikileaks import sfp_wikileaks from sflib import SpiderFoot @pytest.mark.usefixtures class TestModulewikileaks(unittest.TestCase): def test_opts(self): module = sfp_wikileaks() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_wikileaks() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_wikileaks() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_wikileaks() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_wikipediaedits.py ================================================ import pytest import unittest from modules.sfp_wikipediaedits import sfp_wikipediaedits from sflib import SpiderFoot @pytest.mark.usefixtures class TestModulewikipediaedits(unittest.TestCase): def test_opts(self): module = sfp_wikipediaedits() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_wikipediaedits() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_wikipediaedits() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_wikipediaedits() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_xforce.py ================================================ import pytest import unittest from modules.sfp_xforce import sfp_xforce from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleXforce(unittest.TestCase): def test_opts(self): module = sfp_xforce() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_xforce() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_xforce() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_xforce() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200", "400", "404"] for code in http_codes: with self.subTest(code=code): module = sfp_xforce() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "402", "403", "429"] for code in http_codes: with self.subTest(code=code): module = sfp_xforce() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_xforce() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_yandexdns.py ================================================ import pytest import unittest from modules.sfp_yandexdns import sfp_yandexdns from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleYandexdns(unittest.TestCase): def test_opts(self): module = sfp_yandexdns() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_yandexdns() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_yandexdns() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_yandexdns() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/modules/test_sfp_zetalytics.py ================================================ import pytest import unittest from modules.sfp_zetalytics import sfp_zetalytics from sflib import SpiderFoot from spiderfoot import SpiderFootEvent, SpiderFootTarget @pytest.mark.usefixtures class TestModuleZetalytics(unittest.TestCase): def test_opts(self): module = sfp_zetalytics() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_zetalytics() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_zetalytics() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_zetalytics() self.assertIsInstance(module.producedEvents(), list) def test_handleEvent_no_api_key_should_set_errorState(self): sf = SpiderFoot(self.default_options) module = sfp_zetalytics() module.setup(sf, dict()) target_value = 'example target value' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_zonefiles.py ================================================ import pytest import unittest from modules.sfp_zonefiles import sfp_zonefiles from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleZoneFiles(unittest.TestCase): def test_opts(self): module = sfp_zonefiles() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_zonefiles() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_zonefiles() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_zonefiles() self.assertIsInstance(module.producedEvents(), list) def test_parseApiResponse_nonfatal_http_response_code_should_not_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["200"] for code in http_codes: with self.subTest(code=code): module = sfp_zonefiles() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertFalse(module.errorState) def test_parseApiResponse_fatal_http_response_error_code_should_set_errorState(self): sf = SpiderFoot(self.default_options) http_codes = ["401", "403", "404", "429", "500", "502", "503"] for code in http_codes: with self.subTest(code=code): module = sfp_zonefiles() module.setup(sf, dict()) result = module.parseApiResponse({"code": code, "content": None}) self.assertIsNone(result) self.assertTrue(module.errorState) ================================================ FILE: test/unit/modules/test_sfp_zoneh.py ================================================ import pytest import unittest from modules.sfp_zoneh import sfp_zoneh from sflib import SpiderFoot @pytest.mark.usefixtures class TestModuleZoneh(unittest.TestCase): def test_opts(self): module = sfp_zoneh() self.assertEqual(len(module.opts), len(module.optdescs)) def test_setup(self): sf = SpiderFoot(self.default_options) module = sfp_zoneh() module.setup(sf, dict()) def test_watchedEvents_should_return_list(self): module = sfp_zoneh() self.assertIsInstance(module.watchedEvents(), list) def test_producedEvents_should_return_list(self): module = sfp_zoneh() self.assertIsInstance(module.producedEvents(), list) ================================================ FILE: test/unit/spiderfoot/test_spiderfootcorrelator.py ================================================ # test_spiderfootcorrelator.py import unittest from spiderfoot import SpiderFootCorrelator, SpiderFootDb class TestSpiderFootCorrelator(unittest.TestCase): """ Test SpiderFootCorrelator """ def test_init_argument_dbh_invalid_type_should_raise_TypeError(self): invalid_types = [None, str(), list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): SpiderFootCorrelator(invalid_type, {}) def test_init_argument_ruleset_invalid_type_should_raise_TypeError(self): invalid_types = [None, str(), list(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): SpiderFootCorrelator(None, invalid_type) def test_init_argument_ruleset_invalid_rule_should_raise_SyntaxError(self): sfdb = SpiderFootDb(self.default_options, False) ruleset = {"sample rule": "invalid yaml"} with self.assertRaises(SyntaxError): SpiderFootCorrelator(sfdb, ruleset) def test_run_correlations_invalid_scan_instance_should_raise_ValueError(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}, 'example scan id') with self.assertRaises(ValueError): correlator.run_correlations() def test_build_db_criteria_argument_matchrule_invalid_type_should_raise_TypeError(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}) invalid_types = [None, str(), list(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): correlator.build_db_criteria(invalid_type) def test_enrich_event_sources_argument_rule_invalid_type_should_raise_TypeError(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}) invalid_types = [None, str(), list(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): correlator.enrich_event_sources(invalid_type) def test_enrich_event_children_argument_rule_invalid_type_should_raise_TypeError(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}) invalid_types = [None, str(), list(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): correlator.enrich_event_children(invalid_type) def test_enrich_event_entities_argument_rule_invalid_type_should_raise_TypeError(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}) invalid_types = [None, str(), list(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): correlator.enrich_event_entities(invalid_type) def test_process_rule_argument_rule_invalid_type_should_raise_TypeError(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}) invalid_types = [None, str(), list(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): correlator.process_rule(invalid_type) def test_build_correlation_title_argument_rule_invalid_type_should_raise_TypeError(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}) invalid_types = [None, str(), list(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): correlator.build_correlation_title(invalid_type, []) def test_build_correlation_title_argument_data_invalid_type_should_raise_TypeError(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}) invalid_types = [None, str(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): correlator.build_correlation_title({}, invalid_type) def test_create_correlation_argument_rule_invalid_type_should_raise_TypeError(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}) invalid_types = [None, str(), list(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): correlator.create_correlation(invalid_type, [], readonly=True) def test_create_correlation_argument_data_invalid_type_should_raise_TypeError(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}) invalid_types = [None, str(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): correlator.create_correlation({}, invalid_type, readonly=True) def test_check_ruleset_validity_should_return_bool(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}) ruleset = [{"sample": "sample"}] self.assertIsInstance(correlator.check_ruleset_validity(ruleset), bool) invalid_types = [None, str(), list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): self.assertIsInstance(correlator.check_ruleset_validity(invalid_type), bool) def test_check_rule_validity_invalid_rule_should_return_false(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}) invalid_types = [None, str(), list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): self.assertFalse(correlator.check_rule_validity(invalid_type)) def test_check_rule_validity_rule_missing_mandatory_field_should_return_false(self): sfdb = SpiderFootDb(self.default_options, False) correlator = SpiderFootCorrelator(sfdb, {}) rule = { "id": "sample", "collections": [], "headline": "sample" } self.assertFalse(correlator.check_rule_validity(rule)) rule = { "id": "sample", "meta": "sample", "headline": "sample" } self.assertFalse(correlator.check_rule_validity(rule)) rule = { "id": "sample", "meta": "sample", "collections": [] } self.assertFalse(correlator.check_rule_validity(rule)) ================================================ FILE: test/unit/spiderfoot/test_spiderfootdb.py ================================================ # test_spiderfootdb.py import pytest import unittest from spiderfoot import SpiderFootDb, SpiderFootEvent @pytest.mark.usefixtures class TestSpiderFootDb(unittest.TestCase): """ Test SpiderFootDb """ def test_init_argument_opts_of_invalid_type_should_raise_TypeError(self): """ Test __init__(self, opts, init=False) """ invalid_types = [None, "", list(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): SpiderFootDb(invalid_type) def test_init_argument_opts_with_empty_value_should_raise_ValueError(self): """ Test __init__(self, opts, init=False) """ with self.assertRaises(ValueError): SpiderFootDb(dict()) def test_init_argument_opts_with_empty_key___database_value_should_raise_ValueError(self): """ Test __init__(self, opts, init=False) """ with self.assertRaises(ValueError): opts = dict() opts['__database'] = None SpiderFootDb(opts) def test_init_should_create_SpiderFootDb_object(self): """ Test __init__(self, opts, init=False) """ sfdb = SpiderFootDb(self.default_options, False) self.assertIsInstance(sfdb, SpiderFootDb) @unittest.skip("todo") def test_create_should_create_database_schema(self): """ Test create(self) """ sfdb = SpiderFootDb(self.default_options, False) sfdb.create() self.assertEqual('TBD', 'TBD') def test_close_should_close_database_connection(self): """ Test close(self) """ sfdb = SpiderFootDb(self.default_options, False) sfdb.close() def test_search_should_return_a_list(self): """ Test search(self, criteria, filterFp=False) """ sfdb = SpiderFootDb(self.default_options, False) criteria = { 'scan_id': "example scan id", 'type': "example type", 'value': "example value", 'regex': "example regex" } search_results = sfdb.search(criteria, False) self.assertIsInstance(search_results, list) self.assertFalse(search_results) def test_search_argument_criteria_of_invalid_type_should_raise_TypeError(self): """ Test search(self, criteria, filterFp=False) """ sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, "", list(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.search(invalid_type, False) def test_search_argument_criteria_key_of_invalid_type_should_raise_TypeError(self): """ Test search(self, criteria, filterFp=False) """ sfdb = SpiderFootDb(self.default_options, False) criteria = { 'type': "example type", 'value': "example value", 'regex': [] } with self.assertRaises(TypeError): sfdb.search(criteria, False) def test_search_argument_criteria_no_valid_criteria_should_raise_ValueError(self): """ Test search(self, criteria, filterFp=False) """ sfdb = SpiderFootDb(self.default_options, False) criteria = { 'invalid_criteria': "example invalid criteria" } with self.assertRaises(ValueError): sfdb.search(criteria, False) def test_search_argument_criteria_one_criteria_should_raise_ValueError(self): """ Test search(self, criteria, filterFp=False) """ sfdb = SpiderFootDb(self.default_options, False) criteria = { 'type': "example type" } with self.assertRaises(ValueError): sfdb.search(criteria, False) def test_eventTypes_should_return_a_list(self): """ Test eventTypes(self) """ sfdb = SpiderFootDb(self.default_options, False) event_types = sfdb.eventTypes() self.assertIsInstance(event_types, list) def test_scanLogEvent_should_create_a_scan_log_event(self): """ Test scanLogEvent(self, instanceId, classification, message, component=None) """ sfdb = SpiderFootDb(self.default_options, False) sfdb.scanLogEvent("", "", "", None) self.assertEqual('TBD', 'TBD') def test_scanLogEvent_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanLogEvent(self, instanceId, classification, message, component=None) """ sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanLogEvent(invalid_type, "", "") def test_scanLogEvent_argument_classification_of_invalid_type_should_raise_TypeError(self): """ Test scanLogEvent(self, instanceId, classification, message, component=None) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanLogEvent(instance_id, invalid_type, "") def test_scanLogEvent_argument_message_of_invalid_type_should_raise_TypeError(self): """ Test scanLogEvent(self, instanceId, classification, message, component=None) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" invalid_types = [None, list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanLogEvent(instance_id, "", invalid_type) @unittest.skip("todo") def test_scanInstanceCreate_should_create_a_scan_instance(self): """ Test scanInstanceCreate(self, instanceId, scanName, scanTarget) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_name = "example scan name" scan_target = "example scan target" sfdb.scanInstanceCreate(instance_id, scan_name, scan_target) self.assertEqual('TBD', 'TBD') @unittest.skip("todo") def test_scanInstanceCreate_argument_instanceId_already_exists_should_halt_and_catch_fire(self): """ Test scanInstanceCreate(self, instanceId, scanName, scanTarget) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_name = "example scan name" scan_target = "example scan target" sfdb.scanInstanceCreate(instance_id, scan_name, scan_target) instance_id = "example instance id" scan_name = "example scan name" scan_target = "example scan target" with self.assertRaises(IOError): sfdb.scanInstanceCreate(instance_id, scan_name, scan_target) self.assertEqual('TBD', 'TBD') def test_scanInstanceCreate_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanInstanceCreate(self, instanceId, scanName, scanTarget) """ sfdb = SpiderFootDb(self.default_options, False) scan_name = "" scan_target = "spiderfoot.net" invalid_types = [None, list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanInstanceCreate(invalid_type, scan_name, scan_target) def test_scanInstanceCreate_argument_scanName_of_invalid_type_should_raise_TypeError(self): """ Test scanInstanceCreate(self, instanceId, scanName, scanTarget) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_target = "spiderfoot.net" invalid_types = [None, list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanInstanceCreate(instance_id, invalid_type, scan_target) def test_scanInstanceCreate_argument_scanTarget_of_invalid_type_should_raise_TypeError(self): """ Test scanInstanceCreate(self, instanceId, scanName, scanTarget) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_name = "" invalid_types = [None, list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanInstanceCreate(instance_id, scan_name, invalid_type) def test_scanInstanceSet(self): """ Test scanInstanceSet(self, instanceId, started=None, ended=None, status=None) """ sfdb = SpiderFootDb(self.default_options, False) scan_instance = 'example scan instance' sfdb.scanInstanceSet(scan_instance, None, None, None) self.assertEqual('TBD', 'TBD') def test_scanInstanceSet_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanInstanceSet(self, instanceId, started=None, ended=None, status=None) """ sfdb = SpiderFootDb(self.default_options, False) started = None ended = None status = None invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanInstanceSet(invalid_type, started, ended, status) def test_scanInstanceGet_should_return_scan_info(self): """ Test scanInstanceGet(self, instanceId) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_name = "example scan name" scan_target = "example scan target" sfdb.scanInstanceCreate(instance_id, scan_name, scan_target) scan_instance_get = sfdb.scanInstanceGet(instance_id) self.assertEqual(len(scan_instance_get), 6) self.assertIsInstance(scan_instance_get[0], str) self.assertEqual(scan_instance_get[0], scan_name) self.assertIsInstance(scan_instance_get[1], str) self.assertEqual(scan_instance_get[1], scan_target) self.assertIsInstance(scan_instance_get[2], float) self.assertIsInstance(scan_instance_get[3], float) self.assertIsInstance(scan_instance_get[4], float) self.assertIsInstance(scan_instance_get[5], str) self.assertEqual(scan_instance_get[5], 'CREATED') def test_scanInstanceGet_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanInstanceGet(self, instanceId) """ sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanInstanceGet(invalid_type) def test_scanResultSummary_should_return_a_list(self): """ Test scanResultSummary(self, instanceId, by="type") """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_results_summary = sfdb.scanResultSummary(instance_id, "type") self.assertIsInstance(scan_results_summary, list) def test_scanResultSummary_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanResultSummary(self, instanceId, by="type") """ sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanResultSummary(invalid_type) def test_scanResultSummary_argument_by_of_invalid_type_should_raise_TypeError(self): """ Test scanResultSummary(self, instanceId, by="type") """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanResultSummary(instance_id, invalid_type) with self.assertRaises(ValueError): sfdb.scanResultSummary(instance_id, "invalid filter type") def test_scanResultSummary_argument_by_invalid_value_should_raise_ValueError(self): """ Test scanResultSummary(self, instanceId, by="type") """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" with self.assertRaises(ValueError): sfdb.scanResultSummary(instance_id, "invalid filter type") def test_scanResultEvent_should_return_a_list(self): """ Test scanResultEvent(self, instanceId, eventType='ALL', filterFp=False) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_result_event = sfdb.scanResultEvent(instance_id, "", False) self.assertIsInstance(scan_result_event, list) def test_scanResultEvent_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanResultEvent(self, instanceId, eventType='ALL', filterFp=False) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ALL' filter_fp = None invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanResultEvent(invalid_type, event_type, filter_fp) def test_scanResultEvent_argument_eventType_of_invalid_type_should_raise_TypeError(self): """ Test scanResultEvent(self, instanceId, eventType='ALL', filterFp=False) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" invalid_types = [None, dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanResultEvent(instance_id, invalid_type, None) def test_scanResultEventUnique_should_return_a_list(self): """ Test scanResultEventUnique(self, instanceId, eventType='ALL', filterFp=False) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_result_event = sfdb.scanResultEventUnique(instance_id, "", False) self.assertIsInstance(scan_result_event, list) def test_scanResultEventUnique_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanResultEventUnique(self, instanceId, eventType='ALL', filterFp=False) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ALL' filter_fp = None invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanResultEventUnique(invalid_type, event_type, filter_fp) def test_scanResultEventUnique_argument_eventType_of_invalid_type_should_raise_TypeError(self): """ Test scanResultEventUnique(self, instanceId, eventType='ALL', filterFp=False) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" invalid_types = [None, list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanResultEventUnique(instance_id, invalid_type, None) def test_scanLogs_should_return_a_list(self): """ Test scanLogs(self, instanceId, limit=None, fromRowId=None, reverse=False) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_logs = sfdb.scanLogs(instance_id, None, None, None) self.assertIsInstance(scan_logs, list) self.assertEqual('TBD', 'TBD') def test_scanLogs_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanLogs(self, instanceId, limit=None, fromRowId=None, reverse=False) """ sfdb = SpiderFootDb(self.default_options, False) limit = None from_row_id = None reverse = None invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanLogs(invalid_type, limit, from_row_id, reverse) def test_scanErrors_should_return_a_list(self): """ Test scanErrors(self, instanceId, limit=None) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_instance = sfdb.scanErrors(instance_id) self.assertIsInstance(scan_instance, list) def test_scanErrors_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanErrors(self, instanceId, limit=None) """ sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanErrors(invalid_type) def test_scanInstanceDelete(self): """ Test scanInstanceDelete(self, instanceId) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" sfdb.scanInstanceDelete(instance_id) self.assertEqual('TBD', 'TBD') def test_scanInstanceDelete_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanInstanceDelete(self, instanceId) """ sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanInstanceDelete(invalid_type) @unittest.skip("todo") def test_scanResultsUpdateFP(self): """ Test scanResultsUpdateFP(self, instanceId, resultHashes, fpFlag) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_name = "example scan name" scan_target = "example scan target" sfdb.scanInstanceCreate(instance_id, scan_name, scan_target) result_hashes = None fp_flag = None sfdb.scanResultsUpdateFP(instance_id, result_hashes, fp_flag) self.assertEqual('TBD', 'TBD') def test_scanResultsUpdateFP_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanResultsUpdateFP(self, instanceId, resultHashes, fpFlag) """ sfdb = SpiderFootDb(self.default_options, False) result_hashes = [] fp_flag = None invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanResultsUpdateFP(invalid_type, result_hashes, fp_flag) def test_scanResultsUpdateFP_argument_resultHashes_of_invalid_type_should_raise_TypeError(self): """ Test scanResultsUpdateFP(self, instanceId, resultHashes, fpFlag) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" fp_flag = None invalid_types = [None, "", dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanResultsUpdateFP(instance_id, invalid_type, fp_flag) def test_configSet_should_set_config_opts(self): """ Test configSet(self, optMap=dict()) """ sfdb = SpiderFootDb(self.default_options, False) opts = dict() opts['example'] = 'example non-default config opt' sfdb.configSet(opts) config = sfdb.configGet() self.assertIsInstance(config, dict) self.assertIn('example', config) self.assertEqual('TBD', 'TBD') def test_configSet_argument_optmap_of_invalid_type_should_raise_TypeError(self): """ Test configSet(self, optMap=dict()) """ sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, "", list()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.configSet(invalid_type) def test_configGet_should_return_a_dict(self): """ Test configGet(self) """ sfdb = SpiderFootDb(self.default_options, False) config = sfdb.configGet() self.assertIsInstance(config, dict) def test_configClear_should_clear_config(self): """ Test configClear(self) """ sfdb = SpiderFootDb(self.default_options, False) opts = dict() opts['example'] = 'example non-default config opt' sfdb.configSet(opts) config = sfdb.configGet() self.assertIsInstance(config, dict) self.assertIn('example', config) sfdb.configClear() config = sfdb.configGet() self.assertIsInstance(config, dict) self.assertNotIn('example', config) def test_scanConfigSet_argument_optMap_of_invalid_type_should_raise_TypeError(self): """ Test scanConfigSet(self, id, optMap=dict()) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" invalid_types = [None, ""] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanConfigSet(instance_id, invalid_type) def test_scanConfigSet_argument_instanceId_with_empty_value_should_raise_ValueError(self): """ Test scanConfigSet(self, id, optMap=dict()) """ sfdb = SpiderFootDb(self.default_options, False) with self.assertRaises(ValueError): sfdb.scanConfigSet("", dict()) def test_scanConfigGet_should_return_a_dict(self): """ Test scanConfigGet(self, instanceId) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_config = sfdb.scanConfigGet(instance_id) self.assertIsInstance(scan_config, dict) def test_scanConfigGet_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanConfigGet(self, instanceId) """ sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanConfigGet(invalid_type) def test_scanEventStore_should_store_a_scan_event(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event = "" invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanEventStore(invalid_type, event) def test_scanEventStore_argument_instanceId_with_empty_value_should_raise_ValueError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event = "" with self.assertRaises(ValueError): sfdb.scanEventStore("", event) def test_scanEventStore_argument_sfEvent_of_invalid_type_should_raise_TypeError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" invalid_types = [None, "", list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanEventStore(instance_id, invalid_type) def test_scanEventStore_argument_sfEvent_with_invalid_eventType_property_type_should_raise_TypeError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" invalid_types = [None, list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): event = SpiderFootEvent(event_type, event_data, module, source_event) event.eventType = invalid_type sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_sfEvent_with_empty_eventType_property_value_should_raise_ValueError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" with self.assertRaises(ValueError): event.eventType = '' sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_sfEvent_with_invalid_data_property_type_should_raise_TypeError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" invalid_types = [None, list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): event = SpiderFootEvent(event_type, event_data, module, source_event) event.data = invalid_type sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_sfEvent_with_empty_data_property_value_should_raise_ValueError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" with self.assertRaises(ValueError): event.data = '' sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_sfEvent_with_invalid_module_property_type_should_raise_TypeError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" invalid_types = [None, list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): event = SpiderFootEvent(event_type, event_data, module, source_event) event.module = invalid_type sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_sfEvent_with_empty_module_property_value_should_raise_ValueError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" with self.assertRaises(ValueError): event.module = '' sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_sfEvent_with_invalid_confidence_property_type_should_raise_TypeError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" invalid_types = [None, list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): event = SpiderFootEvent(event_type, event_data, module, source_event) event.confidence = invalid_type sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_sfEvent_with_empty_confidence_property_value_should_raise_ValueError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" invalid_values = [-1, 101] for invalid_value in invalid_values: with self.subTest(invalid_value=invalid_value): with self.assertRaises(ValueError): event = SpiderFootEvent(event_type, event_data, module, source_event) event.confidence = invalid_value sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_sfEvent_with_invalid_visibility_property_type_should_raise_TypeError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" invalid_types = [None, list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): event = SpiderFootEvent(event_type, event_data, module, source_event) event.visibility = invalid_type sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_sfEvent_with_empty_visibility_property_value_should_raise_ValueError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" invalid_values = [-1, 101] for invalid_value in invalid_values: with self.subTest(invalid_value=invalid_value): with self.assertRaises(ValueError): event = SpiderFootEvent(event_type, event_data, module, source_event) event.visibility = invalid_value sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_sfEvent_with_invalid_risk_property_type_should_raise_TypeError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" invalid_types = [None, list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): event = SpiderFootEvent(event_type, event_data, module, source_event) event.risk = invalid_type sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_sfEvent_with_empty_risk_property_value_should_raise_ValueError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" invalid_values = [-1, 101] for invalid_value in invalid_values: with self.subTest(invalid_value=invalid_value): with self.assertRaises(ValueError): event = SpiderFootEvent(event_type, event_data, module, source_event) event.risk = invalid_value sfdb.scanEventStore(instance_id, event) def test_scanEventStore_argument_sfEvent_with_invalid_sourceEvent_property_type_should_raise_TypeError(self): """ Test scanEventStore(self, instanceId, sfEvent, truncateSize=0) """ sfdb = SpiderFootDb(self.default_options, False) event_type = 'ROOT' event_data = 'example data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = 'example module' event = SpiderFootEvent(event_type, event_data, module, source_event) instance_id = "example instance id" invalid_types = [None, "", list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): event = SpiderFootEvent(event_type, event_data, module, source_event) event.sourceEvent = invalid_type sfdb.scanEventStore(instance_id, event) def test_scanInstanceList_should_return_a_list(self): """ Test scanInstanceList(self) """ sfdb = SpiderFootDb(self.default_options, False) scan_instances = sfdb.scanInstanceList() self.assertIsInstance(scan_instances, list) def test_scanResultHistory_should_return_a_list(self): """ Test scanResultHistory(self, instanceId) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_result_history = sfdb.scanResultHistory(instance_id) self.assertIsInstance(scan_result_history, list) def test_scanResultHistory_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanResultHistory(self, instanceId) """ sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanResultHistory(invalid_type) def test_scanElementSourcesDirect_should_return_a_list(self): """ Test scanElementSourcesDirect(self, instanceId, elementIdList) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" element_id_list = [] scan_element_sources_direct = sfdb.scanElementSourcesDirect(instance_id, element_id_list) self.assertIsInstance(scan_element_sources_direct, list) self.assertEqual('TBD', 'TBD') def test_scanElementSourcesDirect_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanElementSourcesDirect(self, instanceId, elementIdList) """ sfdb = SpiderFootDb(self.default_options, False) element_id_list = [] invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanElementSourcesDirect(invalid_type, element_id_list) def test_scanElementSourcesDirect_argument_elementIdList_of_invalid_type_should_raise_TypeError(self): """ Test scanElementSourcesDirect(self, instanceId, elementIdList) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" invalid_types = [None, "", dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanElementSourcesDirect(instance_id, invalid_type) def test_scanElementChildrenDirect_should_return_a_list(self): """ Test scanElementChildrenDirect(self, instanceId, elementIdList) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_element_children_direct = sfdb.scanElementChildrenDirect(instance_id, list()) self.assertIsInstance(scan_element_children_direct, list) self.assertEqual('TBD', 'TBD') def test_scanElementChildrenDirect_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanElementChildrenDirect(self, instanceId, elementIdList) """ sfdb = SpiderFootDb(self.default_options, False) element_id_list = [] invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanElementChildrenDirect(invalid_type, element_id_list) def test_scanElementChildrenDirect_argument_elementIdList_of_invalid_type_should_raise_TypeError(self): """ Test scanElementChildrenDirect(self, instanceId, elementIdList) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" invalid_types = [None, "", dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanElementChildrenDirect(instance_id, invalid_type) def test_scanElementSourcesAll_should_return_a_list(self): """ Test scanElementSourcesAll(self, instanceId, childData) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" child_data = ["example child", "example child"] scan_element_sources_all = sfdb.scanElementSourcesAll(instance_id, child_data) self.assertIsInstance(scan_element_sources_all, list) self.assertEqual('TBD', 'TBD') def test_scanElementSourcesAll_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanElementSourcesAll(self, instanceId, childData) """ sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, list(), dict(), int()] child_data = [] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanElementSourcesAll(invalid_type, child_data) def test_scanElementSourcesAll_argument_childData_of_invalid_type_should_raise_TypeError(self): """ Test scanElementSourcesAll(self, instanceId, childData) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" invalid_types = [None, "", dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanElementSourcesAll(instance_id, invalid_type) def test_scanElementSourcesAll_argument_childData_with_empty_value_should_raise_ValueError(self): """ Test scanElementSourcesAll(self, instanceId, childData) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" child_data = [] with self.assertRaises(ValueError): sfdb.scanElementSourcesAll(instance_id, child_data) def test_scanElementChildrenAll_should_return_a_list(self): """ Test scanElementChildrenAll(self, instanceId, parentIds) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_element_children_all = sfdb.scanElementChildrenAll(instance_id, list()) self.assertIsInstance(scan_element_children_all, list) self.assertEqual('TBD', 'TBD') def test_scanElementChildrenAll_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanElementChildrenAll(self, instanceId, parentIds) """ sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, list(), dict(), int()] parent_ids = [] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanElementChildrenAll(invalid_type, parent_ids) def test_scanElementChildrenAll_argument_parentIds_of_invalid_type_should_raise_TypeError(self): """ Test scanElementChildrenAll(self, instanceId, parentIds) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" invalid_types = [None, "", dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.scanElementChildrenAll(instance_id, invalid_type) def test_correlationResultCreate_arguments_of_invalid_type_should_raise_TypeError(self): sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.correlationResultCreate(invalid_type, "", "", "", "", "", "", []) with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.correlationResultCreate("", invalid_type, "", "", "", "", "", []) with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.correlationResultCreate("", "", invalid_type, "", "", "", "", []) with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.correlationResultCreate("", "", "", invalid_type, "", "", "", []) with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.correlationResultCreate("", "", "", "", invalid_type, "", "", []) with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.correlationResultCreate("", "", "", "", "", invalid_type, "", []) with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): sfdb.correlationResultCreate("", "", "", "", "", "", invalid_type, []) ================================================ FILE: test/unit/spiderfoot/test_spiderfootevent.py ================================================ # test_spiderfootevent.py import unittest from spiderfoot import SpiderFootEvent class TestSpiderFootEvent(unittest.TestCase): def test_init_root_event_should_create_event(self): event_data = 'example event data' module = 'example module' source_event = '' event_type = 'ROOT' evt = SpiderFootEvent(event_type, event_data, module, source_event) self.assertIsInstance(evt, SpiderFootEvent) def test_init_nonroot_event_with_root_sourceEvent_should_create_event(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example non-root event type' event_data = 'example event data' module = 'example module' evt = SpiderFootEvent(event_type, event_data, module, source_event) self.assertIsInstance(evt, SpiderFootEvent) def test_init_argument_eventType_of_invalid_type_should_raise_TypeError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) module = 'example module' invalid_types = [None, bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): SpiderFootEvent(invalid_type, event_data, module, source_event) def test_init_argument_eventType_with_empty_value_should_raise_ValueError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = '' module = 'example module' with self.assertRaises(ValueError): SpiderFootEvent(event_type, event_data, module, source_event) def test_init_argument_data_of_invalid_type_should_raise_TypeError(self): event_type = 'ROOT' module = '' source_event = '' invalid_types = [None, bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): SpiderFootEvent(event_type, invalid_type, module, source_event) def test_init_argument_data_with_empty_value_should_raise_ValueError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = '' module = 'example module' with self.assertRaises(ValueError): SpiderFootEvent(event_type, event_data, module, source_event) def test_init_argument_module_of_invalid_type_should_raise_TypeError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = SpiderFootEvent(event_type, event_data, module, "ROOT") event_type = 'example non-root event type' invalid_types = [None, bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): SpiderFootEvent(event_type, event_data, invalid_type, source_event) def test_init_argument_module_with_empty_value_should_raise_ValueError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example event type' event_data = 'example event data' module = '' with self.assertRaises(ValueError): SpiderFootEvent(event_type, event_data, module, source_event) def test_init_argument_sourceEvent_of_invalid_type_should_raise_TypeError(self): event_type = 'ROOT' event_data = 'example event data' module = '' event_type = 'example non-root event type' module = 'example module' invalid_types = [None, "", bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): SpiderFootEvent(event_type, event_data, module, invalid_type) def test_init_argument_confidence_of_invalid_type_should_raise_TypeError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' invalid_types = [None, "", bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): evt = SpiderFootEvent(event_type, event_data, module, source_event) evt.confidence = invalid_type def test_init_argument_confidence_invalid_value_should_raise_ValueError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' invalid_values = [-1, 101] for invalid_value in invalid_values: with self.subTest(invalid_value=invalid_value): with self.assertRaises(ValueError): evt = SpiderFootEvent(event_type, event_data, module, source_event) evt.confidence = invalid_value def test_init_argument_visibility_of_invalid_type_should_raise_TypeError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' invalid_types = [None, "", bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): evt = SpiderFootEvent(event_type, event_data, module, source_event) evt.visibility = invalid_type def test_init_argument_visibility_invalid_value_should_raise_ValueError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' invalid_values = [-1, 101] for invalid_value in invalid_values: with self.subTest(invalid_value=invalid_value): with self.assertRaises(ValueError): evt = SpiderFootEvent(event_type, event_data, module, source_event) evt.visibility = invalid_value def test_init_argument_risk_of_invalid_type_should_raise_TypeError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' invalid_types = [None, "", bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): evt = SpiderFootEvent(event_type, event_data, module, source_event) evt.risk = invalid_type def test_init_argument_risk_invalid_value_should_raise_ValueError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' invalid_values = [-1, 101] for invalid_value in invalid_values: with self.subTest(invalid_value=invalid_value): with self.assertRaises(ValueError): evt = SpiderFootEvent(event_type, event_data, module, source_event) evt.risk = invalid_value def test_confidence_attribute_should_return_confidence_as_integer(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' confidence = 100 evt = SpiderFootEvent(event_type, event_data, module, source_event) evt.confidence = confidence self.assertEqual(confidence, evt.confidence) def test_confidence_attribute_setter_invalid_type_should_raise_TypeError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, module, source_event) invalid_types = [None, "", bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): evt.confidence = invalid_type def test_visibility_attribute_should_return_visibility_as_integer(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' visibility = 100 evt = SpiderFootEvent(event_type, event_data, module, source_event) evt.visibility = visibility self.assertEqual(visibility, evt.visibility) def test_visibility_attribute_setter_invalid_type_should_raise_TypeError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, module, source_event) invalid_types = [None, "", bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): evt.visibility = invalid_type def test_risk_attribute_should_return_risk_as_integer(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' risk = 100 evt = SpiderFootEvent(event_type, event_data, module, source_event) evt.risk = risk self.assertEqual(risk, evt.risk) def test_risk_attribute_setter_invalid_type_should_raise_TypeError(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, module, source_event) invalid_types = [None, "", bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): evt.risk = invalid_type def test_actualSource_attribute_should_return_actual_source_as_string(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, module, source_event) actual_source = 'example actual source' evt.actualSource = actual_source self.assertEqual(actual_source, evt.actualSource) def test_sourceEventHash_attribute_should_return_source_event_hash_as_string(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, module, source_event) self.assertEqual("ROOT", evt.sourceEventHash) def test_moduleDataSource_attribute_should_return_module_data_source_as_string(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, module, source_event) module_data_source = 'example module data source' evt.moduleDataSource = module_data_source self.assertEqual(module_data_source, evt.moduleDataSource) def test_asdict_root_event_should_return_event_as_a_dict(self): event_data = 'example event data' module = 'example module data' source_event = '' event_type = 'ROOT' evt = SpiderFootEvent(event_type, event_data, module, source_event) evt_dict = evt.asDict() self.assertIsInstance(evt_dict, dict) self.assertEqual(evt_dict['type'], event_type) self.assertEqual(evt_dict['data'], event_data) self.assertEqual(evt_dict['module'], module) def test_asdict_nonroot_event_should_return_event_as_a_dict(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' source_event = SpiderFootEvent(event_type, event_data, module, source_event) event_type = 'example non-root event type' event_data = 'example event data' module = 'example_module' evt = SpiderFootEvent(event_type, event_data, module, source_event) evt_dict = evt.asDict() self.assertIsInstance(evt_dict, dict) self.assertEqual(evt_dict['type'], event_type) self.assertEqual(evt_dict['data'], event_data) self.assertEqual(evt_dict['module'], module) def test_hash_attribute_root_event_should_return_root_as_a_string(self): event_type = 'ROOT' event_data = 'example event data' module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, module, source_event) evt_hash = evt.hash self.assertEqual('ROOT', evt_hash) def test_hash_attribute_nonroot_event_should_return_a_string(self): event_type = 'ROOT' event_data = 'example event data' module = 'example module' source_event = SpiderFootEvent(event_type, event_data, module, "ROOT") event_type = 'not ROOT' evt = SpiderFootEvent(event_type, event_data, module, source_event) evt_hash = evt.hash self.assertIsInstance(evt_hash, str) ================================================ FILE: test/unit/spiderfoot/test_spiderfoothelpers.py ================================================ # test_spiderfoot.py import pytest import unittest from spiderfoot import SpiderFootHelpers @pytest.mark.usefixtures class TestSpiderFootHelpers(unittest.TestCase): def test_data_path_should_return_a_string(self): data_path = SpiderFootHelpers.dataPath() self.assertIsInstance(data_path, str) def test_cache_path_should_return_a_string(self): cache_path = SpiderFootHelpers.cachePath() self.assertIsInstance(cache_path, str) def test_log_path_should_return_a_string(self): log_path = SpiderFootHelpers.logPath() self.assertIsInstance(log_path, str) def test_target_type(self): target_type = SpiderFootHelpers.targetTypeFromString("0.0.0.0") self.assertEqual('IP_ADDRESS', target_type) target_type = SpiderFootHelpers.targetTypeFromString("noreply@spiderfoot.net") self.assertEqual('EMAILADDR', target_type) target_type = SpiderFootHelpers.targetTypeFromString("0.0.0.0/0") self.assertEqual('NETBLOCK_OWNER', target_type) target_type = SpiderFootHelpers.targetTypeFromString("+1234567890") self.assertEqual('PHONE_NUMBER', target_type) target_type = SpiderFootHelpers.targetTypeFromString('"Human Name"') self.assertEqual('HUMAN_NAME', target_type) target_type = SpiderFootHelpers.targetTypeFromString('"abc123"') self.assertEqual('USERNAME', target_type) target_type = SpiderFootHelpers.targetTypeFromString("1234567890") self.assertEqual('BGP_AS_OWNER', target_type) target_type = SpiderFootHelpers.targetTypeFromString("::1") self.assertEqual('IPV6_ADDRESS', target_type) target_type = SpiderFootHelpers.targetTypeFromString("spiderfoot.net") self.assertEqual('INTERNET_NAME', target_type) target_type = SpiderFootHelpers.targetTypeFromString("1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R") self.assertEqual('BITCOIN_ADDRESS', target_type) def test_target_type_invalid_seed_should_return_none(self): target_type = SpiderFootHelpers.targetTypeFromString(None) self.assertEqual(None, target_type) target_type = SpiderFootHelpers.targetTypeFromString("") self.assertEqual(None, target_type) target_type = SpiderFootHelpers.targetTypeFromString('""') self.assertEqual(None, target_type) def test_urlRelativeToAbsolute_argument_url_relative_path_should_return_an_absolute_path(self): relative_url = SpiderFootHelpers.urlRelativeToAbsolute('/somewhere/else/../../path?param=value#fragment') self.assertIsInstance(relative_url, str) self.assertEqual('/path?param=value#fragment', relative_url) def test_url_base_dir_should_return_a_string(self): base_dir = SpiderFootHelpers.urlBaseDir('http://localhost.local/path?param=value#fragment') self.assertIsInstance(base_dir, str) self.assertEqual('http://localhost.local/', base_dir) def test_url_base_url_should_return_a_string(self): base_url = SpiderFootHelpers.urlBaseUrl('http://localhost.local/path?param=value#fragment') self.assertIsInstance(base_url, str) self.assertEqual('http://localhost.local', base_url) def test_dictionaryWordsFromWordlists_should_return_a_set(self): words = SpiderFootHelpers.dictionaryWordsFromWordlists() self.assertIsInstance(words, set) self.assertTrue(len(words)) def test_dictionaryWordsFromWordlists_argument_wordlists_missing_wordlist_should_raise_IOError(self): with self.assertRaises(IOError): SpiderFootHelpers.dictionaryWordsFromWordlists(['does not exist']) def test_humanNamesFromWordlists_should_return_a_set(self): names = SpiderFootHelpers.humanNamesFromWordlists() self.assertIsInstance(names, set) self.assertTrue(len(names)) def test_humanNamesFromWordlists_argument_wordlists_missing_wordlist_should_raise_IOError(self): with self.assertRaises(IOError): SpiderFootHelpers.humanNamesFromWordlists(['does not exist']) def test_usernamesFromWordlists_should_return_a_set(self): users = SpiderFootHelpers.usernamesFromWordlists() self.assertIsInstance(users, set) self.assertTrue(len(users)) def test_usernamesFromWordlists_argument_wordlists_missing_wordlist_should_raise_IOError(self): with self.assertRaises(IOError): SpiderFootHelpers.usernamesFromWordlists(['does not exist']) def test_buildGraphData_invalid_data_type_should_raise_TypeError(self): invalid_types = [None, "", bytes(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): SpiderFootHelpers.buildGraphData(invalid_type) def test_buildGraphData_empty_data_should_raise_ValueError(self): with self.assertRaises(ValueError): SpiderFootHelpers.buildGraphData([]) def test_buildGraphData_data_row_with_invalid_number_of_columns_should_raise_ValueError(self): with self.assertRaises(ValueError): SpiderFootHelpers.buildGraphData( [ ['only one column'] ] ) def test_buildGraphData_should_return_a_set(self): graph_data = SpiderFootHelpers.buildGraphData( [ ["test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test"] ] ) self.assertIsInstance(graph_data, set) self.assertEqual('TBD', 'TBD') def test_buildGraphGexf_should_return_bytes(self): gexf = SpiderFootHelpers.buildGraphGexf('test root', 'test title', [["test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "ENTITY", "test", "test", "test"]]) self.assertIsInstance(gexf, bytes) self.assertEqual('TBD', 'TBD') def test_buildGraphJson_should_return_a_string(self): json = SpiderFootHelpers.buildGraphJson('test root', [["test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "test", "ENTITY", "test", "test", "test"]]) self.assertIsInstance(json, str) self.assertEqual('TBD', 'TBD') def test_dataParentChildToTree_invalid_data_type_should_return_TypeError(self): invalid_types = [None, "", bytes(), list(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): SpiderFootHelpers.dataParentChildToTree(invalid_type) def test_dataParentChildToTree_empty_data_should_return_ValueError(self): with self.assertRaises(ValueError): SpiderFootHelpers.dataParentChildToTree(dict()) def test_dataParentChildToTree_should_return_dict(self): tree = SpiderFootHelpers.dataParentChildToTree({"test": {"123": "456"}}) self.assertIsInstance(tree, dict) def test_genScanInstanceId_should_return_a_string(self): scan_instance_id = SpiderFootHelpers.genScanInstanceId() self.assertIsInstance(scan_instance_id, str) def test_extractLinksFromHtml_argument_data_not_containing_links_should_return_an_empty_dict(self): parse_links = SpiderFootHelpers.extractLinksFromHtml('url', 'example html content', 'domains') self.assertIsInstance(parse_links, dict) self.assertFalse(parse_links) def test_extractLinksFromHtml_argument_data_containing_malformed_html_with_links_should_return_a_dict_of_urls(self): url = 'http://spiderfoot.net/' parse_links = SpiderFootHelpers.extractLinksFromHtml( url, 'example html content', 'domains' ) self.assertIsInstance(parse_links, dict) self.assertEqual( parse_links, { 'http://spiderfoot.net/xmlrpc.php': {'source': url, 'original': 'http://spiderfoot.net/xmlrpc.php'}, 'http://spiderfoot.net/path': {'source': url, 'original': 'http://spiderfoot.net/path'}, 'http://spiderfoot.net/relative-path': {'source': url, 'original': '/relative-path'}, } ) def test_extractLinksFromHtml_invalid_url_should_raise_TypeError(self): invalid_types = [None, bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): SpiderFootHelpers.extractLinksFromHtml(invalid_type, 'example html content', 'domains') def test_extractLinksFromHtml_invalid_data_should_raise_TypeError(self): invalid_types = [None, bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError): SpiderFootHelpers.extractLinksFromHtml("", invalid_type, 'domains') def test_extractLinksFromHtml_invalid_domains_should_return_a_dict(self): invalid_types = [None, "", bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): parse_links = SpiderFootHelpers.extractLinksFromHtml('url', 'example html content', invalid_type) self.assertIsInstance(parse_links, dict) def test_extractHashesFromText_should_return_a_list(self): invalid_types = [None, list(), bytes(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): hashes = SpiderFootHelpers.extractHashesFromText(invalid_type) self.assertIsInstance(hashes, list) def test_extractHashesFromText_argument_data_should_return_hahes(self): md5_hash = "e17cff4eb3e8fbe6ca3b83fb47532dba" sha1_hash = "f81efbe70f8116fcf3dc4e9b37725dcb949719f5" sha256_hash = "7cd444af3d8de9e195b1f1cb55e7b7d9409dcd4648247c853a2f64b7578dc9b7" sha512_hash = "a55a2fe120d7d7d6e2ba930e6c56faa30b9d24a3178a0aff1d89312a89d61d8a9d5b7743e3af6b1a318d99974a1145ed76f85aa8c6574074dfb347613ccd3249" hashes = SpiderFootHelpers.extractHashesFromText(f"spiderfoot{md5_hash}spiderfoot{sha1_hash}spiderfoot{sha256_hash}spiderfoot{sha512_hash}spiderfoot") self.assertIsInstance(hashes, list) self.assertIn(("MD5", md5_hash), hashes) self.assertIn(("SHA1", sha1_hash), hashes) self.assertIn(("SHA256", sha256_hash), hashes) self.assertIn(("SHA512", sha512_hash), hashes) def test_valid_email_should_return_a_boolean(self): invalid_types = [None, "", bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): valid_email = SpiderFootHelpers.validEmail(invalid_type) self.assertIsInstance(valid_email, bool) self.assertFalse(valid_email) valid_email = SpiderFootHelpers.validEmail('%@localhost.local') self.assertIsInstance(valid_email, bool) self.assertFalse(valid_email) valid_email = SpiderFootHelpers.validEmail('...@localhost.local') self.assertIsInstance(valid_email, bool) self.assertFalse(valid_email) valid_email = SpiderFootHelpers.validEmail('root@localhost.local\n.com') self.assertIsInstance(valid_email, bool) self.assertFalse(valid_email) valid_email = SpiderFootHelpers.validEmail('root@localhost') self.assertIsInstance(valid_email, bool) self.assertFalse(valid_email) valid_email = SpiderFootHelpers.validEmail('root@localhost.local') self.assertIsInstance(valid_email, bool) self.assertTrue(valid_email) def test_validPhoneNumber_should_return_a_boolean(self): invalid_types = [None, "", bytes(), list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): valid_phone = SpiderFootHelpers.validPhoneNumber(invalid_type) self.assertIsInstance(valid_phone, bool) self.assertFalse(valid_phone) valid_phone = SpiderFootHelpers.validPhoneNumber('+1234567890') self.assertIsInstance(valid_phone, bool) self.assertFalse(valid_phone) valid_phone = SpiderFootHelpers.validPhoneNumber('+12345678901234567890') self.assertIsInstance(valid_phone, bool) self.assertFalse(valid_phone) valid_phone = SpiderFootHelpers.validPhoneNumber('+12345678901') self.assertIsInstance(valid_phone, bool) self.assertTrue(valid_phone) def test_validLEI_should_return_a_boolean(self): invalid_types = [None, "", bytes(), list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): valid_phone = SpiderFootHelpers.validLEI(invalid_type) self.assertIsInstance(valid_phone, bool) self.assertFalse(valid_phone) valid_lei = SpiderFootHelpers.validLEI('7ZW8QJWVPR4P1J1KQYZZ') self.assertIsInstance(valid_lei, bool) self.assertFalse(valid_lei) valid_lei = SpiderFootHelpers.validLEI('7ZW8QJWVPR4P1J1KQY45') self.assertIsInstance(valid_lei, bool) self.assertTrue(valid_lei) def test_countryCodes_should_return_a_dict(self): country_code_dict = SpiderFootHelpers.countryCodes() self.assertIsInstance(country_code_dict, dict) def test_countryNameFromCountryCode_argument_countryCode_should_return_country_as_a_string(self): country_name = SpiderFootHelpers.countryNameFromCountryCode('US') self.assertIsInstance(country_name, str) self.assertEqual(country_name, "United States") def test_countryNameFromTld_argument_tld_should_return_country_as_a_string(self): tlds = ['com', 'net', 'org', 'gov', 'mil'] for tld in tlds: with self.subTest(tld=tld): country_name = SpiderFootHelpers.countryNameFromTld(tld) self.assertIsInstance(country_name, str) self.assertEqual(country_name, "United States") def test_extractEmailsFromText_should_return_list_of_emails_from_string(self): emails = SpiderFootHelpers.extractEmailsFromText("

From:noreply@spiderfoot.net

Subject:Hello user@spiderfoot.net, here's some text

") self.assertIsInstance(emails, list) self.assertIn('noreply@spiderfoot.net', emails) self.assertIn('user@spiderfoot.net', emails) def test_extractEmailsFromText_invalid_data_should_return_list(self): invalid_types = [None, "", bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): emails = SpiderFootHelpers.extractEmailsFromText(invalid_type) self.assertIsInstance(emails, list) def test_extractPgpKeysFromText_should_return_list_of_pgp_keys_from_string(self): key1 = "-----BEGIN PGP PUBLIC KEY BLOCK-----Version: software v1.2.3\nComment: sample comment\n\nmQINBFRUAGoBEACuk6ze2V2pZtScf1Ul25N2CX19AeL7sVYwnyrTYuWdG2FmJx4x\nDLTLVUazp2AEm/JhskulL/7VCZPyg7ynf+o20Tu9/6zUD7p0rnQA2k3Dz+7dKHHh\neEsIl5EZyFy1XodhUnEIjel2nGe6f1OO7Dr3UIEQw5JnkZyqMcbLCu9sM2twFyfa\na8JNghfjltLJs3/UjJ8ZnGGByMmWxrWQUItMpQjGr99nZf4L+IPxy2i8O8WQewB5\nfvfidBGruUYC+mTw7CusaCOQbBuZBiYduFgH8hRW97KLmHn0xzB1FV++KI7syo8q\nXGo8Un24WP40IT78XjKO\n=nUop\n-----END PGP PUBLIC KEY BLOCK-----" key2 = "-----BEGIN PGP PRIVATE KEY BLOCK-----Version: software v1.2.3\nComment: sample comment\n\nmQINBFRUAGoBEACuk6ze2V2pZtScf1Ul25N2CX19AeL7sVYwnyrTYuWdG2FmJx4x\nDLTLVUazp2AEm/JhskulL/7VCZPyg7ynf+o20Tu9/6zUD7p0rnQA2k3Dz+7dKHHh\neEsIl5EZyFy1XodhUnEIjel2nGe6f1OO7Dr3UIEQw5JnkZyqMcbLCu9sM2twFyfa\na8JNghfjltLJs3/UjJ8ZnGGByMmWxrWQUItMpQjGr99nZf4L+IPxy2i8O8WQewB5\nfvfidBGruUYC+mTw7CusaCOQbBuZBiYduFgH8hRW97KLmHn0xzB1FV++KI7syo8q\nXGo8Un24WP40IT78XjKO\n=nUop\n-----END PGP PRIVATE KEY BLOCK-----" keys = SpiderFootHelpers.extractPgpKeysFromText(f"

sample{key1}sample

sample{key2}sample

") self.assertIsInstance(keys, list) self.assertIn(key1, keys) self.assertIn(key2, keys) self.assertEqual(len(keys), 2) def test_extractIbansFromText_should_return_a_list(self): invalid_types = [None, "", bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): ibans = SpiderFootHelpers.extractIbansFromText(invalid_type) self.assertIsInstance(ibans, list) # Example IBANS from https://www.iban.com/structure ibans = [ "AL35202111090000000001234567", "AD1400080001001234567890", "AT483200000012345864", "AZ96AZEJ00000000001234567890", "BH02CITI00001077181611", "BY86AKBB10100000002966000000", "BE71096123456769", "BA393385804800211234", "BR1500000000000010932840814P2", "BG18RZBB91550123456789", "CR23015108410026012345", "HR1723600001101234565", "CY21002001950000357001234567", "CZ5508000000001234567899", "DK9520000123456789", "DO22ACAU00000000000123456789", "EG800002000156789012345180002", "SV43ACAT00000000000000123123", "EE471000001020145685", "FO9264600123456789", "FI1410093000123458", "FR7630006000011234567890189", "GE60NB0000000123456789", "DE75512108001245126199", "GI04BARC000001234567890", "GR9608100010000001234567890", "GL8964710123456789", "GT20AGRO00000000001234567890", "VA59001123000012345678", "HU93116000060000000012345676", "IS750001121234563108962099", "IQ20CBIQ861800101010500", "IE64IRCE92050112345678", "IL170108000000012612345", "IT60X0542811101000000123456", "JO71CBJO0000000000001234567890", "KZ563190000012344567", "XK051212012345678906", "KW81CBKU0000000000001234560101", "LV97HABA0012345678910", "LB92000700000000123123456123", "LI7408806123456789012", "LT601010012345678901", "LU120010001234567891", "MT31MALT01100000000000000000123", "MR1300020001010000123456753", "MU43BOMM0101123456789101000MUR", "MD21EX000000000001234567", "MC5810096180790123456789085", "ME25505000012345678951", "NL02ABNA0123456789", "MK07200002785123453", "NO8330001234567", "PK36SCBL0000001123456702", "PS92PALS000000000400123456702", "PL10105000997603123456789123", "PT50002700000001234567833", "QA54QNBA000000000000693123456", "RO09BCYP0000001234567890", "LC14BOSL123456789012345678901234", "SM76P0854009812123456789123", "ST23000200000289355710148", "SA4420000001234567891234", "RS35105008123123123173", "SC52BAHL01031234567890123456USD", "SK8975000000000012345671", "SI56192001234567892", "ES7921000813610123456789", "SE7280000810340009783242", "CH5604835012345678009", "TL380010012345678910106", "TN5904018104004942712345", "TR320010009999901234567890", "UA903052992990004149123456789", "AE460090000000123456789", "GB33BUKB20201555555555", "VG21PACG0000000123456789" ] for iban in ibans: with self.subTest(iban=iban): extract_ibans = SpiderFootHelpers.extractIbansFromText(iban) self.assertIsInstance(extract_ibans, list) self.assertIn(iban, extract_ibans) # Invalid IBANs ibans = [ # Invalid country code "ZZ21PACG0000000123456789", # Invalid length for country code "VG123456789012345", # Invalid mod 97 remainder "VG21PACG0000000123456111" ] for iban in ibans: with self.subTest(iban=iban): extract_ibans = SpiderFootHelpers.extractIbansFromText(iban) self.assertIsInstance(extract_ibans, list) self.assertNotIn(iban, extract_ibans) def test_extractCreditCardsFromText_should_return_a_list(self): invalid_types = [None, "", bytes(), list(), dict()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): cards = SpiderFootHelpers.extractCreditCardsFromText(invalid_type) self.assertIsInstance(cards, list) cards = SpiderFootHelpers.extractCreditCardsFromText("spiderfoot4111 1111 1111 1111spiderfoot") self.assertIsInstance(cards, list) self.assertEqual(["4111111111111111"], cards) def test_sslDerToPem_should_return_a_certificate_as_a_string(self): pem = SpiderFootHelpers.sslDerToPem( b"0\x82\x07J0\x82\x062\xa0\x03\x02\x01\x02\x02\x10\x0c\x1f\xcb\x18E\x18\xc7\xe3\x86gA#mks\xf10\r\x06\t*\x86H\x86\xf7\r\x01\x01\x0b\x05\x000O1\x0b0\t\x06\x03U\x04\x06\x13\x02US1\x150\x13\x06\x03U\x04\n\x13\x0cDigiCert Inc1)0\'\x06\x03U\x04\x03\x13 DigiCert TLS RSA SHA256 2020 CA10\x1e\x17\r230113000000Z\x17\r240213235959Z0\x81\x961\x0b0\t\x06\x03U\x04\x06\x13\x02US1\x130\x11\x06\x03U\x04\x08\x13\nCalifornia1\x140\x12\x06\x03U\x04\x07\x13\x0bLos Angeles1B0@\x06\x03U\x04\n\x0c9Internet\xc2\xa0Corporation\xc2\xa0for\xc2\xa0Assigned\xc2\xa0Names\xc2\xa0and\xc2\xa0Numbers1\x180\x16\x06\x03U\x04\x03\x13\x0fwww.example.org0\x82\x01\x220\r\x06\t*\x86H\x86\xf7\r\x01\x01\x01\x05\x00\x03\x82\x01\x0f\x000\x82\x01\n\x02\x82\x01\x01\x00\xc2\x80w\x89Y\xb8Eo\xbaJ\xd9\x11\xfa{\xad\xc7W\xd0z\xfb\xb6\xfa\xdd\x05\xbb\xa2\x81q\xbb\xe1\x7f!\xd2_.\xf0\xd2rNu4\xf8\x8db\xe3J\xdaQ\x90\xd4\x01=\x9c\x0c\xc0q\xf7\xe6/\xb6\xd6\x07g&\xd0\xde\xff\x17\xce\xf0\x85\xfd1\xc1f\xca\x87e\x05G*_\xc0\xab\xb8\x8c\xc3\xbf\xd0\x17\x7fc\xa3\\\xf0F\xfb\x86\xaa\xfbM\xd7*^\x7f\x9a\xe0\x13\x97}\xbe\xfb}5W\r]^\x81\x985\xea\x16B\xa2\xd3\xb0t\xf7Y-\xed8\xe7\xfez\x1b\xb36\xe6~\xae?\x9e\xa6\x16\x83\xdeS\x01N\x81\x00\xae\xbbB\xf5\x1fu)4\xcd\xe9\x84\x808\xae<7\x14\xc0\xf0\'\xce0R\xb9\x8a\xdc_\x22\xa0y\xf8ONI\x04\xe2u|\xaa/*\x1e\x03\xecqL\xa3*a\xfco\xca\x91\x1e\x93Z.x\x08X\xf6\xee\xbb4 ]\x9a\xe6\xaf\xc6\xd7\xf2\xbf\n{\xfa\x8e\x92w\xe3l{\x0c@\x86dJ\x15\xecp\xd7r\x8ec0\xe1\x0b\xefZ0\x97.%\x02\x03\x01\x00\x01\xa3\x82\x03\xd80\x82\x03\xd40\x1f\x06\x03U\x1d#\x04\x180\x16\x80\x14\xb7k\xa2\xea\xa8\xaa\x84\x8cy\xea\xb4\xda\x0f\x98\xb2\xc5\x95v\xb9\xf40\x1d\x06\x03U\x1d\x0e\x04\x16\x04\x14\xb0\x93?\xe8\x17\x82\xfdl\xb2\xb6\x17\x87\xcb\xe3\x80\xfe\x82\x9b\x01\x9e0\x81\x81\x06\x03U\x1d\x11\x04z0x\x82\x0fwww.example.org\x82\x0bexample.net\x82\x0bexample.edu\x82\x0bexample.com\x82\x0bexample.org\x82\x0fwww.example.com\x82\x0fwww.example.edu\x82\x0fwww.example.net0\x0e\x06\x03U\x1d\x0f\x01\x01\xff\x04\x04\x03\x02\x05\xa00\x1d\x06\x03U\x1d%\x04\x160\x14\x06\x08+\x06\x01\x05\x05\x07\x03\x01\x06\x08+\x06\x01\x05\x05\x07\x03\x020\x81\x8f\x06\x03U\x1d\x1f\x04\x81\x870\x81\x840@\xa0>\xa0<\x86:http://crl3.digicert.com/DigiCertTLSRSASHA2562020CA1-4.crl0@\xa0>\xa0<\x86:http://crl4.digicert.com/DigiCertTLSRSASHA2562020CA1-4.crl0>\x06\x03U\x1d \x0470503\x06\x06g\x81\x0c\x01\x02\x020)0\'\x06\x08+\x06\x01\x05\x05\x07\x02\x01\x16\x1bhttp://www.digicert.com/CPS0\x7f\x06\x08+\x06\x01\x05\x05\x07\x01\x01\x04s0q0$\x06\x08+\x06\x01\x05\x05\x070\x01\x86\x18http://ocsp.digicert.com0I\x06\x08+\x06\x01\x05\x05\x070\x02\x86=http://cacerts.digicert.com/DigiCertTLSRSASHA2562020CA1-1.crt0\t\x06\x03U\x1d\x13\x04\x020\x000\x82\x01\x7f\x06\n+\x06\x01\x04\x01\xd6y\x02\x04\x02\x04\x82\x01o\x04\x82\x01k\x01i\x00v\x00\xee\xcd\xd0d\xd5\xdb\x1a\xce\xc5\\\xb7\x9d\xb4\xcd\x13\xa22\x87F|\xbc\xec\xde\xc3QHYFq\x1f\xb5\x9b\x00\x00\x01\x85\xabH\x05#\x00\x00\x04\x03\x00G0E\x02!\x00\xaa\xdf\x9f+\xa8\xc5t`:\xb6\xfd\x04Z\xdfkk\x1d\x16`\x15x\xad\xefc\x81\x98*\xd38\xb8\xd9\x05\x02 @a\xd7\x22\xa9>\xf8\x17\xd4\x1a\xde\x13L\x01Rj\xe29U!%.\xfb*\x01u\xf7w\xd3\xdb\xce\xfb\x00w\x00s\xd9\x9e\x89\x1bL\x96x\xa0 }G\x9d\xe6\xb2\xc6\x1c\xd0Q^q\x19*\x8ck\x80\x10z\xc1wr\xb5\x00\x00\x01\x85\xabH\x05\x9f\x00\x00\x04\x03\x00H0F\x02!\x00\xd7d\x94\x14\xaek\x80\xba\x91\xce\xf8\x1c\xaf\xb6sW\x89\xe5\xf9\x9b}\x96Z\x00\xcd\x12\xdf=\xce\xefH\xf0\x02!\x00\x97=\xbc\x12s\x1dk\x13\xe0c\x15\xac\x19\x95X\xcb\x8f\xfdO\xb0\xcd\nA\x07,|p\xd9%D\xcb\xc0\x00v\x00H\xb0\xe3k\xda\xa6G4\x0f\xe5j\x02\xfa\x9d0\xeb\x1cR\x01\xcbV\xdd,\x81\xd9\xbb\xbf\xab9\xd8\x84s\x00\x00\x01\x85\xabH\x05^\x00\x00\x04\x03\x00G0E\x02!\x00\xde[\x84{a\xa3%\x8c\'p\x90\x07\xfdb`Q!2\x05\x15\x90XG\x0c\xcf\xe7\x94OS\x84,!\x02 \x0f\xbc\xf2W\xca\x9e\xda\xdaL\xf0%}\xcf\xed\xfa\x87\xe5y(\xde\xb3\xe1\x0b4h]\x87z[\xe4$\n0\r\x06\t*\x86H\x86\xf7\r\x01\x01\x0b\x05\x00\x03\x82\x01\x01\x00Y\xe4J\xd8\xa9\x82\xba\x9aJ\xf1c\x0cmv&u\xb3some HTML with \"some quotes\" & some JavaScript\n

", "Some more input. This function accepts a list" ]) self.assertIsInstance(clean_input, list) self.assertEqual(clean_input, [ '<p>some HTML with "some quotes" & some JavaScript\n<script>alert('JavaScript')</script></p>', "Some more input. This function accepts a list" ]) def test_search_base_should_return_a_list(self): """ Test searchBase(self, id=None, eventType=None, value=None) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) search_results = sfwebui.searchBase(None, None, None) self.assertIsInstance(search_results, list) search_results = sfwebui.searchBase(None, None, "//") self.assertIsInstance(search_results, list) @unittest.skip("todo") def test_scan_correlations_export(self): opts = self.default_options opts['__modules__'] = dict() scan_id = "" sfwebui = SpiderFootWebUi(self.web_default_options, opts) search_results = sfwebui.scancorrelationsexport(scan_id, "csv", "excel") self.assertIsInstance(search_results, bytes) search_results = sfwebui.scancorrelationsexport(scan_id, "xlxs", "excel") self.assertIsInstance(search_results, bytes) @unittest.skip("todo") def test_scan_event_result_export_should_return_bytes(self): """ Test scaneventresultexport(self, id, type, filetype="csv", dialect="excel") """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) search_results = sfwebui.scaneventresultexport("", "") self.assertIsInstance(search_results, bytes) search_results = sfwebui.scaneventresultexport("", "", "excel") self.assertIsInstance(search_results, bytes) @unittest.skip("todo") def test_scan_event_result_export_multi(self): """ Test scaneventresultexportmulti(self, ids, filetype="csv", dialect="excel") """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) search_results = sfwebui.scaneventresultexportmulti("", "") self.assertIsInstance(search_results, bytes) search_results = sfwebui.scaneventresultexportmulti("", "excel") self.assertIsInstance(search_results, bytes) @unittest.skip("todo") def test_scan_search_result_export(self): """ Test scansearchresultexport(self, id, eventType=None, value=None, filetype="csv", dialect="excel") """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) search_results = sfwebui.scansearchresultexport("") self.assertIsInstance(search_results, bytes) search_results = sfwebui.scansearchresultexport("", None, None, "excel") self.assertIsInstance(search_results, bytes) def test_scan_export_logs_invalid_scan_id_should_return_string(self): """ Test scanexportlogs(self: 'SpiderFootWebUi', id: str, dialect: str = "excel") -> str """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) logs = sfwebui.scanexportlogs(None, "excel") self.assertIsInstance(logs, str) self.assertIn("Scan ID not found.", logs) @unittest.skip("todo") def test_scan_export_logs_should_return_bytes(self): """ Test scanexportlogs(self: 'SpiderFootWebUi', id: str, dialect: str = "excel") -> str """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) logs = sfwebui.scanexportlogs("scan id", "excel") self.assertIsInstance(logs, bytes) @unittest.skip("todo") def test_scan_export_json_multi(self): """ Test scanexportjsonmulti(self, ids) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) search_results = sfwebui.scanexportjsonmulti(None) self.assertIsInstance(search_results, bytes) @unittest.skip("todo") def test_scan_viz_should_return_a_string(self): """ Test scanviz(self, id, gexf="0") """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_viz = sfwebui.scanviz(None, None) self.assertIsInstance(scan_viz, str) @unittest.skip("todo") def test_scan_viz_multi_should_return_a_string(self): """ Test scanvizmulti(self, ids, gexf="1") """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_viz_multi = sfwebui.scanvizmulti(None, None) self.assertIsInstance(scan_viz_multi, str) def test_scanopts_should_return_dict(self): opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_opts = sfwebui.scanopts("example scan instance") self.assertIsInstance(scan_opts, dict) self.assertEqual(scan_opts, dict()) def test_rerunscan_invalid_scan_id_should_return_error(self): opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) rerunscan = sfwebui.rerunscan("example scan instance") self.assertIsInstance(rerunscan, str) self.assertIn("Invalid scan ID", rerunscan) @unittest.skip("todo") def test_rerunscanmulti(self): """ Test rerunscanmulti(self, ids) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) rerunscanmulti = sfwebui.rerunscanmulti("example scan instance") self.assertIsInstance(rerunscanmulti, str) @unittest.skip("todo") def test_newscan(self): """ Test newscan(self) """ self.assertEqual('TBD', 'TBD') def test_clonescan(self): """ Test clonescan(self, id) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) clone_scan = sfwebui.clonescan("example scan instance") self.assertIsInstance(clone_scan, str) def test_index(self): """ Test index(self) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) index = sfwebui.index() self.assertIsInstance(index, str) def test_scaninfo(self): """ Test scaninfo(self, id) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_info = sfwebui.scaninfo("example scan instance") self.assertIsInstance(scan_info, str) def test_opts(self): opts = self.default_options opts['__modules__'] = dict() opts['__globaloptdescs__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) opts_page = sfwebui.opts() self.assertIsInstance(opts_page, str) self.assertIn('Settings', opts_page) def test_optsexport_should_return_a_string(self): opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) opts_export = sfwebui.optsexport(None) self.assertIsInstance(opts_export, str) def test_optsraw_should_return_a_list(self): opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) opts_raw = sfwebui.optsraw() self.assertIsInstance(opts_raw, list) self.assertEqual(opts_raw[0], 'SUCCESS') def test_error(self): opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) message = "example message" scan_error = sfwebui.error(message) self.assertIsInstance(scan_error, str) self.assertIn("example message", scan_error) def test_scandelete_invalid_scanid_should_return_an_error(self): """ Test scandelete(self, id) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_delete = sfwebui.scandelete("example scan id") self.assertIsInstance(scan_delete, dict) self.assertEqual("Scan example scan id does not exist", scan_delete.get('error').get('message')) def test_savesettings_invalid_csrf_token_should_return_an_error(self): """ Test savesettings(self, allopts, token, configFile=None) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) save_settings = sfwebui.savesettings(None, "invalid token", None) self.assertIsInstance(save_settings, str) self.assertIn("Invalid token", save_settings) @unittest.skip("todo") def test_savesettings(self): self.assertEqual('TBD', 'TBD') def test_savesettingsraw_invalid_csrf_token_should_return_an_error(self): opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) save_settings_raw = sfwebui.savesettingsraw(None, "invalid token") self.assertIsInstance(save_settings_raw, bytes) self.assertIn(b"Invalid token", save_settings_raw) @unittest.skip("todo") def test_savesettingsraw(self): self.assertEqual('TBD', 'TBD') def test_reset_settings_should_return_true(self): opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) reset_settings = sfwebui.reset_settings() self.assertIsInstance(reset_settings, bool) self.assertTrue(reset_settings) @unittest.skip("todo") def test_result_set_fp(self): """ Test resultsetfp(self, id, resultids, fp) """ self.assertEqual('TBD', 'TBD') def test_eventtypes_should_return_list(self): opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) event_types = sfwebui.eventtypes() self.assertIsInstance(event_types, list) def test_modules_should_return_list(self): opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) modules = sfwebui.modules() self.assertIsInstance(modules, list) def test_correlationrules_should_return_list(self): opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) correlationrules = sfwebui.correlationrules() self.assertIsInstance(correlationrules, list) def test_ping_should_return_list(self): """ Test ping(self) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) ping = sfwebui.ping() self.assertIsInstance(ping, list) self.assertEqual(ping[0], 'SUCCESS') def test_query_should_perform_sql_query(self): """ Test query(self, query) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) select = "12345" query = sfwebui.query(f"SELECT {select}") self.assertIsInstance(query, list) self.assertEqual(len(query), 1) self.assertEqual(str(query[0].get(select)), str(select)) def test_query_invalid_query_should_return_error(self): """ Test query(self, query) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) query = sfwebui.query(None) self.assertIsInstance(query, dict) self.assertEqual("Invalid query.", query.get('error').get('message')) query = sfwebui.query("UPDATE 1") self.assertIsInstance(query, dict) self.assertEqual("Non-SELECTs are unpredictable and not recommended.", query.get('error').get('message')) @unittest.skip("todo") def test_start_scan_should_start_a_scan(self): """ Test startscan(self, scanname, scantarget, modulelist, typelist, usecase) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) start_scan = sfwebui.startscan('example scan name', 'spiderfoot.net', 'example module list', None, None) self.assertEqual(start_scan, start_scan) self.assertEqual('TBD', 'TBD') def test_start_scan_invalid_scanname_should_return_error(self): """ Test startscan(self, scanname, scantarget, modulelist, typelist, usecase) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) start_scan = sfwebui.startscan(None, 'example scan target', None, None, None) self.assertIn('Invalid request: scan name was not specified.', start_scan) start_scan = sfwebui.startscan('', 'example scan target', None, None, None) self.assertIn('Invalid request: scan name was not specified.', start_scan) def test_start_scan_invalid_scantarget_should_return_error(self): """ Test startscan(self, scanname, scantarget, modulelist, typelist, usecase) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) start_scan = sfwebui.startscan('example scan name', None, None, None, None) self.assertIn('Invalid request: scan target was not specified.', start_scan) start_scan = sfwebui.startscan('example scan name', '', None, None, None) self.assertIn('Invalid request: scan target was not specified.', start_scan) def test_start_scan_unrecognized_scantarget_type_should_return_error(self): """ Test startscan(self, scanname, scantarget, modulelist, typelist, usecase) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) start_scan = sfwebui.startscan('example scan name', 'example scan target', 'example module list', None, None) self.assertIn('Invalid target type. Could not recognize it as a target SpiderFoot supports.', start_scan) def test_start_scan_invalid_modules_should_return_error(self): """ Test startscan(self, scanname, scantarget, modulelist, typelist, usecase) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) start_scan = sfwebui.startscan('example scan name', 'spiderfoot.net', None, None, None) self.assertIn('Invalid request: no modules specified for scan.', start_scan) start_scan = sfwebui.startscan('example scan name', 'spiderfoot.net', '', '', '') self.assertIn('Invalid request: no modules specified for scan.', start_scan) def test_start_scan_invalid_typelist_should_return_error(self): """ Test startscan(self, scanname, scantarget, modulelist, typelist, usecase) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) start_scan = sfwebui.startscan('example scan name', 'spiderfoot.net', None, 'invalid type list', None) self.assertIn('Invalid request: no modules specified for scan.', start_scan) start_scan = sfwebui.startscan('example scan name', 'spiderfoot.net', '', 'invalid type list', '') self.assertIn('Invalid request: no modules specified for scan.', start_scan) def test_stopscan_invalid_scanid_should_return_an_error(self): """ Test stopscan(self, id) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) stop_scan = sfwebui.stopscan("example scan id") self.assertIsInstance(stop_scan, dict) self.assertEqual("Scan example scan id does not exist", stop_scan.get('error').get('message')) def test_scanlog_should_return_a_list(self): """ Test scanlog(self, id, limit=None, rowId=None, reverse=None) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_log = sfwebui.scanlog(None, None, None, None) self.assertIsInstance(scan_log, list) scan_log = sfwebui.scanlog('', '', '', '') self.assertIsInstance(scan_log, list) def test_scanerrors_should_return_a_list(self): """ Test scanerrors(self, id, limit=None) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_errors = sfwebui.scanerrors(None, None) self.assertIsInstance(scan_errors, list) scan_errors = sfwebui.scanerrors('', '') self.assertIsInstance(scan_errors, list) def test_scanlist_should_return_a_list(self): """ Test scanlist(self) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_list = sfwebui.scanlist() self.assertIsInstance(scan_list, list) def test_scanstatus_should_return_a_list(self): """ Test scanstatus(self, id) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_status = sfwebui.scanstatus("example scan instance") self.assertIsInstance(scan_status, list) def test_scansummary_should_return_a_list(self): """ Test scansummary(self, id, by) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_summary = sfwebui.scansummary(None, None) self.assertIsInstance(scan_summary, list) scan_summary = sfwebui.scansummary('', '') self.assertIsInstance(scan_summary, list) def test_scaneventresults_should_return_a_list(self): """ Test scaneventresults(self, id, eventType, filterfp=False) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_results = sfwebui.scaneventresults(None, None, None) self.assertIsInstance(scan_results, list) scan_results = sfwebui.scaneventresults('', '', '') self.assertIsInstance(scan_results, list) def test_scaneventresultsunique_should_return_a_list(self): """ Test scaneventresultsunique(self, id, eventType, filterfp=False) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_results = sfwebui.scaneventresultsunique(None, None, None) self.assertIsInstance(scan_results, list) scan_results = sfwebui.scaneventresultsunique('', '', '') self.assertIsInstance(scan_results, list) def test_search_should_return_a_list(self): """ Test search(self, id=None, eventType=None, value=None) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) search_results = sfwebui.search(None, None, None) self.assertIsInstance(search_results, list) search_results = sfwebui.search('', '', '') self.assertIsInstance(search_results, list) def test_scan_history_missing_scanid_should_return_error(self): """ Test scanhistory(self, id) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_history = sfwebui.scanhistory(None) self.assertIsInstance(scan_history, dict) self.assertEqual("No scan specified", scan_history.get('error').get('message')) scan_history = sfwebui.scanhistory("example scan id") self.assertIsInstance(scan_history, list) def test_scan_history_should_return_a_list(self): """ Test scanhistory(self, id) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_history = sfwebui.scanhistory("example scan id") self.assertIsInstance(scan_history, list) def test_scan_element_type_discovery_should_return_a_dict(self): """ Test scanelementtypediscovery(self, id, eventType) """ opts = self.default_options opts['__modules__'] = dict() sfwebui = SpiderFootWebUi(self.web_default_options, opts) scan_element_type_discovery = sfwebui.scanelementtypediscovery(None, None) self.assertIsInstance(scan_element_type_discovery, dict) scan_element_type_discovery = sfwebui.scanelementtypediscovery('', '') self.assertIsInstance(scan_element_type_discovery, dict) ================================================ FILE: test/update-requirements ================================================ #!/bin/bash # Update and lock all pip packages to latest version. # # Must be run from SpiderFoot root directory; ie: # ./test/update-requirements # # Requires lock-requirements: # pip3 install lock-requirements lock requirements.txt git diff requirements.txt lock test/requirements.txt git diff test/requirements.txt # python3 -m safety check -r test/requirements.txt