[ { "path": ".github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Describe the bug**\nA clear and concise description of what the bug is.\n\n**To Reproduce**\nSteps to reproduce the behavior:\n1. Go to '...'\n2. Click on '....'\n3. Scroll down to '....'\n4. See error\n\n**Expected behavior**\nA clear and concise description of what you expected to happen.\n\n**Screenshots**\nIf applicable, add screenshots to help explain your problem.\n\n**Desktop (please complete the following information):**\n - OS: [e.g. iOS]\n - Browser [e.g. chrome, safari]\n - Version [e.g. 22]\n\n**Smartphone (please complete the following information):**\n - Device: [e.g. iPhone6]\n - OS: [e.g. iOS8.1]\n - Browser [e.g. stock browser, safari]\n - Version [e.g. 22]\n\n**Additional context**\nAdd any other context about the problem here.\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.md", "content": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Is your feature request related to a problem? Please describe.**\nA clear and concise description of what the problem is. Ex. I'm always frustrated when [...]\n\n**Describe the solution you'd like**\nA clear and concise description of what you want to happen.\n\n**Describe alternatives you've considered**\nA clear and concise description of any alternative solutions or features you've considered.\n\n**Additional context**\nAdd any other context or screenshots about the feature request here.\n" }, { "path": ".github/workflows/codeql-analysis.yml", "content": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# You may wish to alter this file to override the set of languages analyzed,\n# or to provide custom queries or build logic.\n#\n# ******** NOTE ********\n# We have attempted to detect the languages in your repository. Please check\n# the `language` matrix defined below to confirm you have the correct set of\n# supported CodeQL languages.\n#\nname: \"CodeQL\"\n\non:\n push:\n branches: [ master ]\n pull_request:\n # The branches below must be a subset of the branches above\n branches: [ master ]\n schedule:\n - cron: '16 22 * * 2'\n\njobs:\n analyze:\n name: Analyze\n runs-on: ubuntu-latest\n\n strategy:\n fail-fast: false\n matrix:\n language: [ 'javascript', 'python' ]\n # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]\n # Learn more:\n # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v2\n\n # Initializes the CodeQL tools for scanning.\n - name: Initialize CodeQL\n uses: github/codeql-action/init@v1\n with:\n languages: ${{ matrix.language }}\n # If you wish to specify custom queries, you can do so here or in a config file.\n # By default, queries listed here will override any specified in a config file.\n # Prefix the list here with \"+\" to use these queries and those in the config file.\n # queries: ./path/to/local/query, your-org/your-repo/queries@main\n\n # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).\n # If this step fails, then you should remove it and run the build manually (see below)\n - name: Autobuild\n uses: github/codeql-action/autobuild@v1\n\n # ℹ️ Command-line programs to run using the OS shell.\n # 📚 https://git.io/JvXDl\n\n # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines\n # and modify them (or add more) to build your code if your project\n # uses a compiled language\n\n #- run: |\n # make bootstrap\n # make release\n\n - name: Perform CodeQL Analysis\n uses: github/codeql-action/analyze@v1\n" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# celery beat schedule file\ncelerybeat-schedule\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\nrooted\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n*.db\ntinder.json\ndatabase.db\nchromedriver_linux64/\n/openface/\nchromedriver\n/darta/\n/boe/*\n/instagram/*\n/twitter/*\n/facebook/*\n/google/*\n/data/*\n.DS_Store\n" }, { "path": "CHANGELOG.md", "content": "# Changelog\n\nAll notable changes to this project will be documented in this file.\n\nThe format is based on [Keep a Changelog](http://keepachangelog.com/)\nand this project adheres to [Semantic Versioning](http://semver.org/).\n\n## [1.1.0] - 2020-12-01\n\nFixed\n\n- Fix problems in scraping module\n\n## [1.0.0] - 2019-09-14\n\nAdded\n\n- Adding web interface.\n- CLI\n" }, { "path": "Dockerfile", "content": "FROM ubuntu:18.04\nRUN apt-get update \\\n && apt-get install -y python3-pip python3-dev \\\n && cd /usr/local/bin \\\n && ln -s /usr/bin/python3 python \\\n && pip3 install --upgrade pip\nRUN apt-get install -y --fix-missing \\\n build-essential \\\n cmake \\\n gfortran \\\n git \\\n wget \\\n curl \\\n pkg-config \\\n python3-dev \\\n python3-numpy \\\n software-properties-common \\\n zip \\\n unzip \\\n && apt-get clean && rm -rf /tmp/* /var/tmp/*\n\nRUN cd ~ && \\\n mkdir -p dlib && \\\n git clone -b 'v19.9' --single-branch https://github.com/davisking/dlib.git dlib/ && \\\n cd dlib/ && \\\n python3 setup.py install --yes USE_AVX_INSTRUCTIONS\n\nRUN mkdir spyscrap\nWORKDIR spyscrap\n\n# Install Google Chrome\nRUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb\nRUN dpkg -i google-chrome-stable_current_amd64.deb; apt-get -fy install\n\n# Install Chrome WebDriver\nRUN CHROMEDRIVER_VERSION=`curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE` && \\\n mkdir -p /opt/chromedriver-$CHROMEDRIVER_VERSION && \\\n curl -sS -o /tmp/chromedriver_linux64.zip http://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip && \\\n unzip -qq /tmp/chromedriver_linux64.zip -d /opt/chromedriver-$CHROMEDRIVER_VERSION && \\\n rm /tmp/chromedriver_linux64.zip && \\\n chmod +x /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver && \\\n ln -fs /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver /usr/local/bin/chromedriver\n\nCOPY ./src/requirements.txt .\nRUN pip3 install -r ./requirements.txt\nRUN python -m spacy download es_core_news_sm\nCOPY ./src/ .\nRUN cp /usr/local/bin/chromedriver .\nENV PYTHONIOENCODING=utf-8\nENTRYPOINT [\"python3\",\"main.py\"]\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2020 Ruth Gonzalez - Miguel Hernández\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "![Last update](https://img.shields.io/badge/last%20update-11%20FEB%202021-green.svg?style=flat-square)\n![OSINT](https://img.shields.io/badge/OSINT-brightgreen.svg?style=flat-square)\n![SOCMINT](https://img.shields.io/badge/SOCMINT-brightgreen.svg?style=flat-square)\n![TOOL](https://img.shields.io/badge/TOOL-brightgreen.svg?style=flat-square)\n![CYBERSECURITY](https://img.shields.io/badge/CYBERSECURITY-brightgreen.svg?style=flat-square)\n![SPYSCRAP](https://img.shields.io/badge/SPYSCRAP-brightgreen.svg?style=flat-square)\n\n# SpyScrap\n\n![alt text](./SpyScrap.png)\n\nThis is an [OSINT](https://en.wikipedia.org/wiki/Open-source_intelligence) tool. The main purpose is recolect information from different sources like Google, Tinder, Twitter and more. It combines facial recognition methods to filter the results and uses natural language processing for obtaining important entities from the website the user appears. The tool is able to calculate a final score which indicates the amount of public exposition an user has on the Internet.\nIt has two different modules that can work indepently: CLI and Web Interface.\nBoth modules are built using docker and are easy to deploy.\n\nIf you like the tool, give us a star! :star:\n\n# CLI\n\nCLI Module for web scraping:\n\n* Tinder\n* Instagram\n* Yandex\n* Google\n* Facebook\n* BOE\n* Twitter\n\n\n## Prerequisites\n\nDocker and docker-compose\n\n\n### Installation\n```bash\ndocker build -t spyscrap .\ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap [options]\n```\nYou must put the image you want to be used for facial recognition under the shared volume in docker as in the next example:\n```\ndocker run -ti -v /Users/ruthgnz/Documents/osint/SpyScrap/src/data:/spyscrap/data sp -t twitter -n \"ruth gonzalez novillo\" -i ./data/descarga.jpeg\n```\n\n## Usage\n\n```bash\ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap [options]\n```\n\nGet Tinder users and store data in sqlite3 database. Tinder Token must be capturen when logging into Tinder App under Local Storage.\n```bash\ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t tinder -k TOKEN\t\t\n```\n\nSearch in google.\nAdd -i to download images and do facial recognition\nAdd -p to only search in an specific site Ex: Linkedin\n\n```bash\ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap --tag google -n \"\"\ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap --tag google -n \"\" -i \ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap --tag google -n \"\" -i \t-p \"\"\n```\n\nSearch twitter profiles\n```bash\ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t twitter -n \"\" -s \t\t\n```\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\n\nSearch facebook profiles\nAdd -i to download images do facial recognition\t\t\n```bash\ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t facebook -n \"\"\ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap --tag facebook -n \"\"\t-i \t\t\t\n```\n\nSearch instagram profiles\nAdd -i to download instagram profile image and do facial recognition\n```bash\ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t instagram -n \"\"\ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t instagram -n \"\" -i \t\t\t\n```\n\nSearch DNI, Names and Surnames in BOE\n```bash\ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t boe -n \"\" -s \ndocker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t boe -n \"\" -s \t-e -d -f \t\t\t\n```\n\nOTHER EXAMPLES:\n``` docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap [options]\n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t tinder -k TOKEN\t\t\t\n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap --tag google -n \"\"\t\t\n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap --tag google -n \"\" -i \t\t\t\t\t\t\t\t\n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap --tag google -n \"\" -i \t-p \"\"\t\t\t\t\t\t\t\t\n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t twitter -n \"\" -s \t\t\t\t\t\t\n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t facebook -n \"\"\t\t\t\t\t\t\t\t\t\t\t\n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap --tag facebook -n \"\"\t-i \t\t\t\t\t\t\t\t\n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t instagram -n \"\"\t\t\t\t\t\t\t\t\t\t\t\t\n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t instagram -n \"\" -i \t\t\t\t\t\t\t\t\t\t\t\n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t boe -n \"\" -s \n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap -t boe -n \"\" -s \t-e -d -f \n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap main.py -t yandex -k -i \n docker run -ti -v /PATH/TO/SpyScrap/src/data:/spyscrap/data spyscrap main.py -t yandex -i \n ```\nAll the results are stored in the docker shared volume you must have configured on your localhost when running the container. The first part is the path for your local folder and you can change it. The second part must be the one in the example (/spyscrap/data)\n```bash\n-v /PATH/TO/SpyScrap/src/data:/spyscrap/data\n ```\n# Web Interface\n\nThis is a wrapper for the CLI.\n\n![alt text](./GUI.png)\n\n## Prerequisites\n\nDocker and docker-compose\n\n### Installation\n```bash\ncd web\ndocker-compose up\n```\nOnce the images are built, open the browser:\n```\nhttp:\\\\localhost\n```\nFor searching in Tinder you must put the database.db file created using the CLI in the volume inside the folder:\n\n```\nSpyScrap\\web\\data\n```\nYou will also find in this folder the results of all your searches on the web interface.\n\n## DISCLAIMER ⚠️\nThis tool is for educational purposes only.\n\nPlease only use this tool on systems you have permission to access! Ethical use only.\n\nAny actions and or activities related to the tools we have created is solely your responsibility. The misuse of the tools we have created can result in criminal charges brought against the persons in question. We will not be held responsible in the event any criminal charges be brought against any individuals misusing the tools we have made to break the law.\n\n## Authors ✒️\n\n* **Ruth González** - [@RuthGnz](https://twitter.com/RuthGnz)\n* **Miguel Hernández** - [@MiguelHzBz](https://twitter.com/MiguelHzBz)\n\n\n## Thanks 🎁\n\n* BBVA Next Technologies SecLab Team\n\n
\nFeel free to collaborate!!\n
\n\n> NOTE: Facial recognition is slow. The tool doesn't implement threading, and depends on your computer power. Be patient when the dataset is huge and you are using images to filter the results, specially on the Tinder module.\n\n\n---\n⌨️ with ❤️ by [@RuthGnz](https://twitter.com/RuthGnz) & [@MiguelHzBz](https://twitter.com/MiguelHzBz)\n" }, { "path": "src/data/.gitignore", "content": "# Ignore everything in this directory\n*\n# Except this file\n!.gitignore" }, { "path": "src/main.py", "content": "from osint_sources.model import create_tables\nfrom osint_sources.scraper import *\nimport sys, getopt\nimport argparse\n\ndef banner():\n print(r\"\"\"\n\t------------------------------------------\n\t| SpyScrap |\n\t|\t\t\t\t\t |\n\t| Authors: Ruth Gonzalez (@RuthGnz) |\n\t|\t Miguel Hernandez (@MiguelHzBz) |\n\t| Version: 1.0 |\n\t|\t\t\t\t\t |\n\t| |\n\t------------------------------------------\n\n \"\"\")\n\ndef getArguments(args):\n\n\targuments={}\n\tparser = argparse.ArgumentParser(description='EI - This tool scrapping social media to get information from a target')\n\tparser.add_argument('-t','--tag',dest='tag', help='Insert the option to scrapper, options: tinder, twitter, google, facebook, instagram or all')\n\tparser.add_argument('-k','--token',dest='token', help='If you choose tinder/yandex option, provide a valid token')\n\tparser.add_argument('-n','--name',dest='name', help='Name of person you like to search.')\n\tparser.add_argument(\"-p\",'--place',dest='place', help=\"Location you like to search\")\n\tparser.add_argument(\"-i\",'--image',dest='image', help=\"Image you like to search\")\n\tparser.add_argument(\"-s\",'--size',dest='size', help=\"Limit for searches\")\n\tparser.add_argument(\"-e\",'--explicit', dest='explicit', help=\"Default True. If true it search the exact text, if false it can search each word separately\")\n\tparser.add_argument(\"-d\",'--initdate',dest='initdate',help=\"Format is dd/mm/aaaa\")\n\tparser.add_argument('-f','--finaldate',dest='finaldate', help=\"Format is dd/mm/aaaa\")\n\tparser.add_argument(\"-v\",'--verbose', action=\"store_true\",help=\"Increase output verbosity\")\n\targs = parser.parse_args()\n\n\tif not args.tag:\n\t\tprint (\"--------------\")\n\t\tprint (\"Error in input arguments: \")\n\t\tprint (\"Need one tag of input, -t/--tag twitter/facebook/instagram/google/tinder/boe/yandex/all\")\n\t\tprint (\"--------------\")\n\t\tparser.print_help()\n\t\tsys.exit(-1)\n\tif args.tag.lower() == \"tinder\":\n\t\tif not args.token:\n\t\t\tprint (\"--------------\")\n\t\t\tprint (\"Tinder token must be provided\")\n\t\t\tprint (\"--------------\")\n\t\t\tparser.print_help()\n\t\t\tsys.exit(-1)\n\t\telse:\n\t\t\tprint (\"Starting Tinder scrapper...\")\n\t\t\ttinder(args.token)\n\n\tif args.tag.lower() == \"google\":\n\t\tif not args.name:\n\t\t\tprint (\"--------------\")\n\t\t\tprint (\"Name option must be provided\")\n\t\t\tprint (\"--------------\")\n\t\t\tparser.print_help()\n\t\t\tsys.exit(-1)\n\t\telse:\n\t\t\tif not args.place:\n\t\t\t\targs.place=''\n\t\t\tprint (\"Starting Google scrapper...\")\n\t\t\tgoogle(args.name,args.place,args.image,args.size,args.verbose)\n\n\tif args.tag.lower() == \"twitter\":\n\t\tif not args.name:\n\t\t\tprint (\"--------------\")\n\t\t\tprint (\"Name option must be provided\")\n\t\t\tprint (\"--------------\")\n\t\t\tparser.print_help()\n\t\t\tsys.exit(-1)\n\t\tif not args.size:\n\t\t\tprint (\"Size default: 2\")\n\t\t\targs.size=2\n\t\tprint (\"Starting Twitter scrapper...\")\n\t\ttwitter_scrapper(args.name,args.size,args.image,args.verbose)\n\n\tif args.tag.lower() == \"facebook\":\n\t\tif not args.name:\n\t\t\tprint (\"--------------\")\n\t\t\tprint (\"Name must be provided\")\n\t\t\tprint (\"--------------\")\n\t\t\tparser.print_help()\n\t\t\tsys.exit(-1)\n\t\tif not args.size:\n\t\t\tprint (\"Size default: 2\")\n\t\t\targs.size=2\n\t\tprint (\"Starting Facebook scrapper...\")\n\t\tfacebook_scrapper(args.name,args.image,args.size,args.verbose)\n\n\tif args.tag.lower() == \"instagram\":\n\t\tif not args.name:\n\t\t\tprint (\"--------------\")\n\t\t\tprint (\"Name option must be provided\")\n\t\t\tprint (\"--------------\")\n\t\t\tparser.print_help()\n\t\t\tsys.exit(-1)\n\t\telse:\n\t\t\tprint (\"Starting Instagram scrapper...\")\n\t\t\tinstagram_scrapper(args.name,args.image,args.verbose)\n\n\tif args.tag.lower()==\"boe\":\n\t\tif not args.name:\n\t\t\tprint (\"--------------\")\n\t\t\tprint (\"Name option must be provided\")\n\t\t\tprint (\"--------------\")\n\t\t\tparser.print_help()\n\t\t\tsys.exit(-1)\n\t\telse:\n\t\t\tif not args.size:\n\t\t\t\targs.size=1\n\t\t\tprint (\"Starting Boe scrapper...\")\n\t\t\tboe_scrapper(args.name,args.initdate,args.finaldate,args.size,args.explicit,args.verbose)\n\tif args.tag.lower()==\"yandex\":\n\t\tif not args.image:\n\t\t\tprint (\"--------------\")\n\t\t\tprint (\"image option must be provided\")\n\t\t\tprint (\"--------------\")\n\t\t\tparser.print_help()\n\t\t\tsys.exit(-1)\n\t\tif not re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+] |[!*\\(\\), ]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', args.image) and not args.token:\n\t\t\tprint (\"--------------\")\n\t\t\tprint (\"if you upload your own photo, client-id imgur must be provided with token option \" + args.image)\n\t\t\tprint (\"--------------\")\n\t\t\tparser.print_help()\n\t\t\tsys.exit(-1)\n\t\telse:\n\t\t\tprint (\"Starting Yandex scrapper...\")\n\t\t\tyandex_scrapper(args.image, args.token,args.verbose)\n\n\tif args.tag.lower() == \"all\":\n\t\tprint (\"--------------\")\n\t\tprint (\"TBD\")\n\t\tprint (\"--------------\")\n\t\tsys.exit(-1)\n\treturn args\n\n\ndef main(argv):\n\tbanner()\n\tif not os.path.isdir(\"data\"):\n\t\tos.mkdir(\"data\");\n\targs = getArguments(argv)\n\tprint(\"--------------------\")\n\tprint (\"Thanks for use SypScrap tool\")\n\nif __name__ == '__main__':\n\tcreate_tables()\n\tmain(sys.argv)\n \t#sys.exit(-1)\n" }, { "path": "src/osint_sources/__init__.py", "content": "" }, { "path": "src/osint_sources/boe.py", "content": "#!/usr/bin/python\n# coding: utf-8\n# encoding=utf8\nimport sys\nimport datetime\nfrom selenium.webdriver.common.keys import Keys\nimport time\nfrom selenium import webdriver\nimport os\nfrom parsel import Selector\nimport urllib.parse\nfrom selenium.common.exceptions import NoSuchElementException\nimport json\nfrom selenium.webdriver.chrome.options import Options\nimport shutil\nimport requests\nfrom io import BytesIO\nimport xml.etree.ElementTree as ET\n\ndef boe (text_to_search,initDate,outDate,pages,exact,verbose):\n if exact:\n text_to_search='\"'+text_to_search+'\"'\n pages=int(pages)\n chrome_options = Options()\n jsonData=[]\n chrome_options.add_argument(\"--headless\")\n chrome_options.add_argument('--no-sandbox')\n chrome_options.add_argument('--disable-dev-shm-usage')\n if initDate!=None and outDate!=None:\n url = 'https://www.boe.es/buscar/boe.php?campo%5B0%5D=ORI&dato%5B0%5D%5B1%5D=1&dato%5B0%5D%5B2%5D=2&dato%5B0%5D%5B3%5D=3&dato%5B0%5D%5B4%5D=4&dato%5B0%5D%5B5%5D=5&dato%5B0%5D%5BT%5D=T&operador%5B0%5D=and&campo%5B1%5D=TIT&dato%5B1%5D=&operador%5B1%5D=and&campo%5B2%5D=DEM&dato%5B2%5D=&operador%5B2%5D=and&campo%5B3%5D=DOC&dato%5B3%5D='+text_to_search+'&operador%5B3%5D=and&campo%5B4%5D=NBO&dato%5B4%5D=&operador%5B4%5D=and&campo%5B5%5D=NOF&dato%5B5%5D=&operador%5B5%5D=and&operador%5B6%5D=and&campo%5B6%5D=FPU&dato%5B6%5D%5B0%5D='+initDate+'&dato%5B6%5D%5B1%5D='+outDate+'&page_hits=50&sort_field%5B0%5D=fpu&sort_order%5B0%5D=desc&sort_field%5B1%5D=ori&sort_order%5B1%5D=asc&sort_field%5B2%5D=ref&sort_order%5B2%5D=asc&accion=Buscar'\n else:\n url ='https://www.boe.es/buscar/boe.php?campo%5B0%5D=ORI&dato%5B0%5D%5B1%5D=1&dato%5B0%5D%5B2%5D=2&dato%5B0%5D%5B3%5D=3&dato%5B0%5D%5B4%5D=4&dato%5B0%5D%5B5%5D=5&dato%5B0%5D%5BT%5D=T&operador%5B0%5D=and&campo%5B1%5D=TIT&dato%5B1%5D=&operador%5B1%5D=and&campo%5B2%5D=DEM&dato%5B2%5D=&operador%5B2%5D=and&campo%5B3%5D=DOC&dato%5B3%5D='+text_to_search+'&operador%5B3%5D=and&campo%5B4%5D=NBO&dato%5B4%5D=&operador%5B4%5D=and&campo%5B5%5D=NOF&dato%5B5%5D=&operador%5B5%5D=and&operador%5B6%5D=and&campo%5B6%5D=FPU&dato%5B6%5D%5B0%5D=&dato%5B6%5D%5B1%5D=&page_hits=50&sort_field%5B0%5D=fpu&sort_order%5B0%5D=desc&sort_field%5B1%5D=ori&sort_order%5B1%5D=asc&sort_field%5B2%5D=ref&sort_order%5B2%5D=asc&accion=Buscar'\n\n chrome_path = './chromedriver'\n driver = webdriver.Chrome(chrome_path,chrome_options=chrome_options)\n\n driver.get(url)\n driver.implicitly_wait(20)\n links=[]\n for page in range(pages):\n elements=driver.find_elements_by_tag_name('li')\n for link in elements:\n li=link.get_attribute('class')\n if li=='resultado-busqueda':\n date=link.find_elements_by_tag_name('h4')[0].get_attribute('innerHTML').split(' ')[3]\n a=link.find_elements_by_tag_name('a')\n for i in a:\n cl=i.get_attribute('class')\n if cl=='resultado-busqueda-link-defecto':\n href=i.get_attribute('href')\n href=href.split('=')[1]\n newUrl='https://www.boe.es/diario_boe/xml.php?id='+href\n links.append(newUrl)\n nextPage=driver.find_elements_by_class_name('pagSig')\n if len(nextPage) == 0:\n break\n else:\n nextPage=nextPage[0]\n nextPageLink= nextPage.find_element_by_xpath('..').get_attribute('href')\n driver.get(nextPageLink)\n\n driver.quit()\n boe_data=[]\n for url in links:\n boe={}\n boe['url']=url\n remoteFile = urllib.request.urlopen(url).read()\n memoryFile = BytesIO(remoteFile)\n tree = ET.parse(memoryFile)\n root = tree.getroot()\n text=root.find('texto')\n tables=text.findall('table')\n results=[]\n for table in tables:\n is_important=False\n headings=[]\n content=[]\n thead=table.find('thead')\n if thead!=None:\n tr=thead.find('tr')\n th=tr.findall('th')\n if len(th)>0:\n for t in th:\n if t!=None:\n data=t.text\n if data!=None:\n headings.append(data)\n if 'nombre' in data.lower() or 'apellido' in data.lower() or 'dni' in data.lower() or 'd.n.i' in data.lower() or 'nif' in data.lower():\n is_important=True\n else:\n td=tr.findall('td')\n for t in td:\n p=t.find('p')\n if p!=None:\n data=p.text\n headings.append(data)\n if 'nombre' in data.lower() or 'apellido' in data.lower() or 'dni' in data.lower() or 'd.n.i' in data.lower() or 'nif' in data.lower():\n is_important=True\n else:\n tr=table.findall('tr')\n for i,tri in enumerate(tr):\n td=tri.findall('td')\n for tdi in td:\n p=tdi.findall('p')\n heading=\"\"\n for pi in p:\n if pi.get('class')!=None:\n if 'cabeza_tabla' in pi.get('class') :\n heading=heading+pi.text\n if 'nombre' in heading.lower() or 'apellido' in heading.lower() or 'dni' in heading.lower() or 'd.n.i' in heading.lower() or 'nif' in heading.lower():\n is_important=True\n else:\n if pi.text != None:\n data=pi.text\n if 'nombre' in data.lower() or 'apellido' in data.lower() or 'dni' in data.lower() or 'd.n.i' in data.lower() or 'nif' in data.lower():\n heading=heading+pi.text\n is_important=True\n if i==2:\n break\n if 'ANEXO' not in heading:\n headings.append(heading)\n\n\n\n\n if is_important:\n tbody=table.find('tbody')\n if tbody!=None:\n tr=tbody.findall('tr')\n else:\n tr=table.findall('tr')\n for t in tr:\n td=t.findall('td')\n\n dataTable={}\n for i,tdi in enumerate(td):\n info=''\n if tdi.text.strip()==None or tdi.text.strip()==\"\":\n p=tdi.find('p')\n if p!=None:\n info=p.text\n else:\n info=tdi.text\n if i>len(headings)-1:\n dataTable[i]=info\n else:\n dataTable[headings[i]]=info\n content.append(dataTable)\n results.append({'table':content,'headings':headings})\n\n boe['datatables']=results\n\n texto=[]\n if len(results)==0:\n p=text.findall('p')\n is_important_line=len(p)\n for i,pi in enumerate(p):\n if pi.text != None:\n data=pi.text\n if 'nombre' in data.lower() or 'apellido' in data.lower() or 'dni' in data.lower() or 'd.n.i' in data.lower() or 'nif' in data.lower():\n is_important_line=i\n if is_important_line<=i:\n texto.append(pi.text)\n #try to search the text\n boe['texto']=texto\n boe_data.append(boe)\n\n\n\n now = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n if not os.path.isdir(\"data/boe\"):\n os.mkdir( \"data/boe\" );\n path=os.path.join('data/boe',str(now) + '_boe_data.json')\n with open(path, 'w+') as outfile:\n json.dump(boe_data, outfile)\n if verbose:\n for i in boe_data:\n print(i[\"url\"])\n print(\"--------------------\")\n if i[\"datatables\"]:\n print(i[\"datatables\"])\n if i[\"texto\"]:\n print(i[\"texto\"])\n print(\"Results BOE in: \" + str(path))\n response={'results':str(path)}\n return response\n" }, { "path": "src/osint_sources/facebook.py", "content": "#!/usr/bin/python\n# coding: utf-8\n# encoding=utf8\nimport sys\nimport datetime\nfrom selenium.webdriver.common.keys import Keys\nimport time\nfrom selenium import webdriver\nimport os\nfrom parsel import Selector\nimport urllib.parse\nfrom selenium.common.exceptions import NoSuchElementException\nimport json\nfrom selenium.webdriver.chrome.options import Options\nimport shutil\nimport requests\nfrom osint_sources.recognition import *\ndef facebook (name_to_search,knownimage,size,verbose):\n\n chrome_options = Options()\n jsonData=[]\n chrome_options.add_argument(\"--headless\")\n chrome_options.add_argument(\"--no-sandbox\")\n chrome_options.add_argument(\"--disable-dev-shm-usage\")\n\n chrome_path = './chromedriver'\n driver = webdriver.Chrome(chrome_path,chrome_options=chrome_options)\n\n\n driver.get(\"https://es-la.facebook.com/public/\"+name_to_search)\n print(\"https://es-la.facebook.com/public/\"+name_to_search)\n driver.implicitly_wait(20)\n\n isMoreButton=True\n for i in range(1,int(size)):\n isEnd=driver.find_elements_by_id('browse_end_of_results_footer')\n print(isEnd)\n if len(isEnd)>0:\n isMoreButton=False\n driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n time.sleep(2)\n links=[]\n results=driver.find_elements_by_id('BrowseResultsContainer')\n if len(results)>0:\n results=results[0]\n info=results.find_elements_by_tag_name('a')\n else:\n info=[]\n\n for user in info:\n user_class=user.get_attribute('class')\n if user_class=='_32mo':\n links.append(user.get_attribute('href'))\n user={'name':user.get_attribute('title'),'profile':user.get_attribute('href')}\n jsonData.append(user)\n\n isMoreButton=True\n i=0\n id_value=\"fbBrowseScrollingPagerContainer\"\n while isMoreButton:\n more=driver.find_elements_by_id(id_value+str(i))\n i=i+1\n if len(more)==0:\n isMoreButton=False\n else:\n div = more[0]\n info=div.find_elements_by_tag_name('a')\n for user in info:\n user_class=user.get_attribute('class')\n if user_class=='_32mo':\n links.append(user.get_attribute('href'))\n user={'name':user.get_attribute('title'),'profile':user.get_attribute('href')}\n jsonData.append(user)\n\n now = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n if not os.path.isdir(\"data/facebook\"):\n os.mkdir( \"data/facebook\");\n os.mkdir(\"data/facebook/\"+str(now)+\"_images\")\n path= os.path.join('data/facebook',str(now)+'_facebook_data.json')\n with open(path, 'w+') as outfile:\n json.dump(jsonData, outfile)\n if verbose:\n print(jsonData)\n print(\"Results Facebook in: \" + str(path))\n\n j=0\n response={'results':str(path)}\n if knownimage:\n for ind,l in enumerate(links):\n user=jsonData[ind]\n driver.get(l)\n try:\n div=driver.find_elements_by_class_name('profilePicThumb')[0]\n img=div.find_elements_by_tag_name('img')[0]\n url=img.get_attribute('src')\n name=os.path.join('data/facebook/'+str(now)+'_images',str(j)+\"-\"+name_to_search+\".jpg\")\n j=j+1\n urllib.request.urlretrieve(url, name)\n user['image']=name\n except:\n pass\n with open(path, 'w+') as outfile:\n json.dump(jsonData, outfile)\n driver.quit()\n print(\"Start compare images\")\n\n face_identification(knownimage,'./data/facebook/'+str(now)+'_images/')\n response['images']='./data/facebook/'+str(now)+'_images/'\n response['recognized']='./data/facebook/'+str(now)+'_images/recognized/'\n\n return response\n" }, { "path": "src/osint_sources/google.py", "content": "from selenium.webdriver.common.keys import Keys\nimport time\nimport urllib.request\nfrom urllib.parse import unquote\nfrom selenium import webdriver\nimport os\nimport json\nimport datetime\nimport face_recognition\nfrom os import listdir,remove\nfrom os.path import isfile, join\nfrom selenium.webdriver.chrome.options import Options\nfrom osint_sources.recognition import *\nimport spacy\nimport requests as req\nimport re\n\ndef containsAny(str, set):\n \"\"\" Check whether sequence str contains ANY of the items in set. \"\"\"\n return 1 in [c in str for c in set]\n\ndef google(toSearch,placeToSearch,knownImage,number,verbose):\n\tchrome_options = Options()\n\tchrome_options.add_argument(\"--headless\")\n\tchrome_options.add_argument(\"--no-sandbox\")\n\tchrome_options.add_argument(\"--disable-dev-shm-usage\")\n\tchrome_path = './chromedriver'\n\tdriver = webdriver.Chrome(chrome_path,chrome_options=chrome_options)\n\tif placeToSearch != None and len(placeToSearch)>0:\n\t\tdriver.get(\"https://www.google.com/search?q=site:\"+placeToSearch+\"+AND+%22\"+toSearch+\"%22&source=lnms&tbm=isch&sa=X&ved=0ahUKEwiz2eSN_9vgAhUJoRQKHU8YCuwQ_AUIDigB&biw=1181&bih=902\")\n\telse:\n\t\tdriver.get(\"https://www.google.com/search?q=\"+toSearch+\"&source=lnms&tbm=isch&sa=X&ved=0ahUKEwiz2eSN_9vgAhUJoRQKHU8YCuwQ_AUIDigB&biw=1181&bih=902\")\n\n\tdriver.implicitly_wait(50)\n\tif number == None:\n\t\tnumber=len(search)\n\tisMoreButton=True\n\twhile isMoreButton:\n\t\tlast_height = driver.execute_script(\"return document.body.scrollHeight\")\n\t\twhile True:\n\t\t\tdriver.execute_script(\"window.scrollTo(0, document.body.scrollHeight-1000);\")\n\t\t\t# Wait to load the page.\n\t\t\tdriver.implicitly_wait(30) # seconds\n\t\t\tnew_height = driver.execute_script(\"return document.body.scrollHeight\")\n\t\t\tif new_height == last_height:\n\t\t\t\tbreak\n\t\t\tlast_height = new_height\n # sleep for 30s\n\t\t\tdriver.implicitly_wait(30) # seconds\n\t\tinputs=driver.find_elements_by_class_name('mye4qd')\n\t\tinput_elem=None\n\t\tfor inp in inputs:\n\t\t\tmore=inp.get_attribute(\"type\")\n\t\t\tif more=='button':\n\t\t\t\tinput_elem=inp\n\t\t\t\tbreak\n\t\tif input_elem==None:\n\t\t\tisMoreButton=False\n\t\telse:\n\t\t\tprint('More Elements')\n\t\t\ttry:\n\t\t\t\tprint('Click')\n\t\t\t\tinput_elem.click()\n\t\t\t\tdriver.implicitly_wait(30)\n\t\t\texcept Exception as e:\n\t\t\t\tprint('break',e)\n\t\t\t\tdriver.implicitly_wait(30)\n\t\t\t\tbreak\n\n\n\tout = []\n\tjsonfile={}\n\tt =\"\"\n\tmy_set = {\"{\", \"}\", \"&\", \"#\", \"_\", \"=\",\":\",\"(\",\")\", \"+\",\".\"}\n\tnlp = spacy.load(\"es_core_news_sm\")\n\n\tsearch=driver.find_elements_by_xpath(\"//img[contains(@class,'Q4LuWd')]\")\n\tj=1\n\tnotRepeatPhotos = []\n\tnotRepeatFromUrl = []\n\tnumber = int(number)\n\ttotalPages = number*20\n\tnow = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n\tif not os.path.isdir(\"data/google\"):\n\t\tos.mkdir( \"data/google\");\n\tos.mkdir(\"data/google/\"+str(now)+\"_images\")\n\tfor i in range(0,len(search)):\n\t\timg=search[i]\n\t\ttry:\n\t\t\timg.click()\n\t\t\ttime.sleep(2)\n\t\t\tactual_images = driver.find_elements_by_css_selector('img.n3VNCb')\n\t\t\tfor actual_image in actual_images:\n\t\t\t\tif actual_image.get_attribute('src') and 'https' in actual_image.get_attribute('src'):\n\t\t\t\t\turl_image = actual_image.get_attribute('src')\n\t\t\t\t\ttext_image = actual_image.get_attribute('alt')\n\t\t\t\t\tn = actual_image.find_element_by_xpath('..')\n\t\t\t\t\turl_content = n.get_attribute(\"href\")\n\t\t\t\t\tjsonfile[\"photos\"]=url_image\n\t\t\t\t\tjsonfile[\"from_url\"]=url_content\n\t\t\t\t\tjsonfile[\"info\"] = text_image\n\t\t\t\t\tif not url_content.endswith(\".pdf\"):\n\t\t\t\t\t\ttry:\n\t\t\t\t\t\t\tresp = req.get(url_content)\n\t\t\t\t\t\t\tcontent = resp.text\n\t\t\t\t\t\t\tstripped = re.sub('<[^<]+?>', '', content)\n\t\t\t\t\t\t\tstripped_filter = re.sub('\\n', '', stripped)\n\t\t\t\t\t\t\tstripped_filter2 = re.sub('\\t', '', stripped_filter)\n\t\t\t\t\t\t\tdoc = nlp(stripped_filter2)\n\t\t\t\t\t\t\tlocs = []\n\t\t\t\t\t\t\tfor e in doc.ents:\n\t\t\t\t\t\t\t\tif e.label_ == \"LOC\":\n\t\t\t\t\t\t\t\t\tif not containsAny(e.text,my_set) and not e.text in locs:\n\t\t\t\t\t\t\t\t\t\tlocs.append(e.text)\n\t\t\t\t\t\t\tjsonfile[\"LOC_LIST\"] = locs\n\t\t\t\t\t\texcept:\n\t\t\t\t\t\t\tpass\n\t\t\t\t\telse:\n\t\t\t\t\t\tjsonfile[\"LOC_LIST\"] = []\n\n\t\t\t\t\tif placeToSearch != None:\n\t\t\t\t\t\tname=os.path.join('data/google/'+str(now)+'_images',str(i)+\"-\"+placeToSearch+\"-\"+toSearch+\".jpg\")\n\t\t\t\t\telse:\n\t\t\t\t\t\tname=os.path.join('data/google/'+str(now)+'_images',str(i)+\"-\"+toSearch+\".jpg\")\n\n\t\t\t\t\ttry:\n\t\t\t\t\t\turllib.request.urlretrieve(url_image, name)\n\t\t\t\t\t\tjsonfile['storedImage']=name\n\t\t\t\t\texcept:\n\t\t\t\t\t\tsrc=actual_image.get_attribute('src')\n\t\t\t\t\t\tif src != None:\n\t\t\t\t\t\t\turllib.request.urlretrieve(src, name)\n\t\t\t\t\t\t\tjsonfile['storedImage']=name\n\t\t\t\t\tout.append(jsonfile)\n\t\t\t\t\tjsonfile={}\n\t\t\t\t\tif len(out)>= int(totalPages):\n\t\t\t\t\t\tbreak\n\t\texcept Exception as er:\n\t\t\tprint(er)\n\t\tif len(out)>= int(totalPages):\n\t\t\tbreak\n\n\tpath= os.path.join('data/google',str(now)+'_google_data.json')\n\tresponse={'results':str(path)}\n\twith open(path, 'w+') as outfile:\n\t\tjson.dump(out, outfile)\n\tprint(\"Results Google in: \" + str(path))\n\tif verbose:\n\t\tprint(out)\n\tif knownImage:\n\t\tface_identification(knownImage,'data/google/'+str(now)+'_images/')\n\t\tresponse['images']='./data/google/'+str(now)+'_images/'\n\t\tresponse['recognized']='./data/google/'+str(now)+'_images/recognized/'\n\treturn response\n" }, { "path": "src/osint_sources/instagram.py", "content": "#!/usr/bin/python\n# coding: utf-8\n# encoding=utf8\nimport sys\nimport datetime\nimport time\nimport os\nimport requests\nimport json\nimport urllib.request\nfrom osint_sources.recognition import *\n\ndef instagram (name_to_search,knownimage,verbose):\n resp = requests.get(url='https://www.instagram.com/web/search/topsearch/?context=blended&query='+name_to_search)\n now = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n if not os.path.isdir(\"data/instagram\"):\n os.mkdir(\"data/instagram\");\n\n path=os.path.join('data/instagram',str(now)+'_instagram_data.json')\n users=resp.json()['users']\n jsonData=[]\n j=0\n for u in users:\n if verbose:\n print('Username: '+u['user']['username'])\n print('Full Name: '+u['user']['full_name'])\n print('Profile: https://www.instagram.com/'+u['user']['username'])\n print('Is Private: '+str(u['user']['is_private']))\n print('Is Verified: '+str(u['user']['is_verified']))\n print()\n if knownimage:\n if not os.path.isdir(\"data/instagram/\"+str(now)+\"_images\"):\n os.mkdir(\"data/instagram/\"+str(now)+\"_images\");\n image_name=os.path.join('data/instagram/'+str(now)+'_images',str(j)+\"-\"+'instagram.jpg')\n try:\n urllib.request.urlretrieve(u['user']['profile_pic_url'], image_name)\n except Exception as e:\n print(e)\n\n user={'username':u['user']['username'],'full_name':u['user']['full_name'],'profile':'https://www.instagram.com/'+u['user']['username'],'is_private':u['user']['is_private'],'is_verified':u['user']['is_verified'],'image':image_name}\n j=j+1\n else:\n user={'username':u['user']['username'],'full_name':u['user']['full_name'],'profile':'https://www.instagram.com/'+u['user']['username'],'is_private':u['user']['is_private'],'is_verified':u['user']['is_verified']}\n\n jsonData.append(user)\n with open(path, 'w+') as outfile:\n json.dump(jsonData, outfile)\n print(\"Results Instagram in: \" + str(path))\n response={'results':str(path)}\n if len(users)>0:\n if knownimage:\n print(\"Start compare images.\")\n face_identification(knownimage,\"data/instagram/\"+str(now)+\"_images/\")\n response['images']='./data/instagram/'+str(now)+'_images/'\n response['recognized']='./data/instagram/'+str(now)+'_images/recognized/'\n if verbose:\n for r, d, f in os.walk('./data/instagram/'+str(now)+'_images/recognized/'):\n for file in f:\n index_json = int(file.replace(\"-instagram.jpg\",\"\"))\n print(\"-----------\")\n print(\" MATCH\")\n print(\"-----------\")\n print('Username: '+jsonData[index_json]['username'])\n print('Full Name: '+jsonData[index_json]['full_name'])\n print('Profile: https://www.instagram.com/'+jsonData[index_json]['username'])\n print('Is Private: '+str(jsonData[index_json]['is_private']))\n print('Is Verified: '+str(jsonData[index_json]['is_verified']))\n print()\n return response\n" }, { "path": "src/osint_sources/model.py", "content": "from peewee import *\nimport datetime\nDATABASE = './data/database.db'\ndatabase = SqliteDatabase(DATABASE)\n\n\n\n# model definitions -- the standard \"pattern\" is to define a base model class\n# that specifies which database to use. then, any subclasses will automatically\n# use the correct storage. for more information, see:\n# http://charlesleifer.com/docs/peewee/peewee/models.html#model-api-smells-like-django\nclass BaseModel(Model):\n class Meta:\n database = database\n\nclass User(BaseModel):\n name = CharField()\n uid = CharField(unique=True)\n bio = CharField()\n birth = DateTimeField()\n gender=IntegerField()\n s_number=IntegerField()\n location = CharField()\n\n def getUsers():\n select=User.select()\n return (select)\n\n def insertUser(user_elem):\n try:\n result = (User\n .create(name=user_elem['name'],uid=user_elem['uid'], bio=user_elem['bio'], birth=user_elem['birth'],gender=user_elem['gender'],s_number=user_elem['s_number'],location=user_elem['location']))\n Photos.insertPhotos(result,user_elem['photos'])\n InstagramPhotos.insertPhotosIg(result,user_elem['instagram'])\n Schools.insertSchools(result,user_elem['schools'])\n Jobs.insertJobs(result,user_elem['jobs'])\n\n except IntegrityError:\n print('Couldnt insert user it might be duplicated')\n\n def getIds():\n result=User.select()\n response=[]\n for u in result.iterator():\n response.append(u.uid)\n return(response)\n\nclass Photos(BaseModel):\n user=ForeignKeyField(User, backref='photos')\n photo=CharField()\n\n def insertPhotos(user,photos_elems):\n for p in photos_elems:\n res=Photos.insert(user=user, photo=p).execute()\n\n\n\nclass InstagramPhotos(BaseModel):\n user=ForeignKeyField(User, backref='inst_photos')\n photo=CharField()\n def insertPhotosIg(user,photos_elems):\n for p in photos_elems:\n InstagramPhotos.insert(user=user, photo=p).execute()\n\nclass Schools(BaseModel):\n user=ForeignKeyField(User, backref='schools')\n school=CharField()\n def insertSchools(user,schools):\n for s in schools:\n res=Schools.insert(user=user, school=s).execute()\n\n\nclass Jobs(BaseModel):\n user=ForeignKeyField(User, backref='jobs')\n job=CharField()\n def insertJobs(user,jobs):\n for j in jobs:\n res=Jobs.insert(user=user, job=j).execute()\n\n\nclass GoogleUrls():\n folder = CharField()\n photo = CharField()\n url = CharField()\n\ndef create_tables():\n with database:\n database.create_tables([User,Photos,InstagramPhotos,Schools,Jobs])\n\n\n\n\n" }, { "path": "src/osint_sources/recognition.py", "content": "import face_recognition\nfrom os import listdir,remove\nfrom os.path import isfile, join\nimport os\nimport cv2\nimport numpy as np\n\ndef face_identification(known_image,folder):\n\t#print('****** Image recognition *******')\n\tonlyfiles = [join(folder, f) for f in listdir(folder) if isfile(join(folder, f))]\n\n\ttry:\n\t\tknown_image_recon = face_recognition.load_image_file(known_image)\n\t\tknown_encoding = face_recognition.face_encodings(known_image_recon)[0]\n\n\texcept:\n\t\tprint('Not valid known image')\n\t\treturn\n\n\tos.mkdir( folder+'recognized/');\n\n\n\tfor image in onlyfiles:\n\t\tif 'data.json' not in image:\n\n\t\t\tunknown_image = face_recognition.load_image_file(image)\n\n\t\t\tif len(face_recognition.face_encodings(unknown_image))>0:\n\t\t\t\tunknown_encoding = face_recognition.face_encodings(unknown_image)[0]\n\t\t\t\tresults = face_recognition.compare_faces([known_encoding], unknown_encoding)\n\t\t\t\tif results[0]==True:\n\t\t\t\t\timg_name=image.split('/')\n\t\t\t\t\timg_name=img_name[len(img_name)-1]\n\t\t\t\t\tnew_image=folder+'/recognized/'+img_name\n\t\t\t\t\tos.rename(image,new_image)\n\t\t\t\telse:\n\t\t\t\t\tpass\n\t\t\telse:\n\t\t\t\tpass\n\t\t\t\t#print('Not face found')\n\t\telse:\n\t\t\tpass\n\t#print()\n" }, { "path": "src/osint_sources/scraper.py", "content": "import csv\nfrom osint_sources.tinder import *\nfrom osint_sources.model import *\nfrom osint_sources.google import *\nfrom osint_sources.twitter import *\nfrom osint_sources.facebook import *\nfrom osint_sources.instagram import *\nfrom osint_sources.boe import *\nfrom osint_sources.yandex import *\n\n\ndef tinder(token):\n\n\t#start_tinder_scrap\n\tscan=Tinder()\n\tauthtk=scan.get_auth_token(token)\n\n\tprint(authtk)\n\tunique_list_ids=[]\n\t#load existent ids from database\n\tunique_list_ids=User.getIds()\n\tprint(len(unique_list_ids))\n\n\twhile True:\n\t\tresponse=scan.getUserInfo()\n\t\tif 'error' in response:\n\t\t\tif response['msg']=='limit rate':\n\t\t\t\tbreak\n\t\t\telif response['msg']=='no data':\n\t\t\t\tprint('no data')\n\t\t\t\tbreak\n\t\telse:\n\t\t\tif response == \"Error\":\n\t\t\t\tbreak\n\t\t\tids=response['ids']\n\t\t\tdata=response['data']\n\t\t\tdifferents= list(set(ids) - set(unique_list_ids))\n\t\t\tunique_list_ids.extend(differents)\n\t\t\tfor d in differents:\n\t\t\t\tuserInfo = [usr['user_info'] for usr in data if usr['id']==d]\n\t\t\t\tfor user in userInfo:\n\t\t\t\t\tUser.insertUser(user)\n\t\t\t\tscan.diskike_users(userInfo)\n\t\t\tprint(len(unique_list_ids))\n\n\n\ndef yandex_scrapper(img,token,verbose):\n\tyandex(img,token,verbose)\n\n\ndef linkedin():\n\tpass\n\n\ndef google_scrapper(toSearch,place,knownImage,number,verbose):\n\tgoogle(toSearch,place,knownImage,number,verbose)\n\n\ndef twitter_scrapper(name,size,knownimage,verbose):\n\ttwitter(name,size,knownimage,verbose)\n\ndef facebook_scrapper(name,knownImage,size,verbose):\n\tfacebook(name,knownImage,size,verbose)\n\ndef instagram_scrapper(name,knownImage,verbose):\n\tinstagram(name,knownImage,verbose)\n\ndef boe_scrapper(toSearch,initDate,finalDate,size,explicit,verbose):\n\tif explicit==None:\n\t\texplicit=True\n\tboe(toSearch,initDate,finalDate,size,explicit,verbose)\n" }, { "path": "src/osint_sources/tinder.py", "content": "# coding=utf-8\nimport json\nimport config\nimport requests\nimport numpy as np\n\nheaders = {\n 'app_version': '6.9.4',\n 'platform': 'ios',\n \"content-type\": \"application/json\",\n \"User-agent\": \"Tinder/7.5.3 (iPhone; iOS 10.3.2; Scale/2.00)\",\n \"Accept\": \"application/json\"\n}\nconfig.host=\"https://api.gotinder.com\"\n\nclass Tinder:\n def __init__(self):\n self.gender = \"\"\n self.location = \"location\"\n\n def get_auth_token(self,tinderToken):\n\n\n try:\n headers.update({\"X-Auth-Token\": tinderToken})\n #print(tinder_auth_token)\n response = self.get_self()\n print(\"You have been successfully authorized!\")\n profile = self.get_self()\n self.gender = profile['gender']\n self.location = profile['pos_info']['city']['name']\n return tinderToken\n except Exception as e:\n print(e)\n return {\"error\": \"Something went wrong. Sorry, but we could not authorize you.\"}\n\n\n def authverif(self):\n res = self.get_auth_token(config.fb_access_token, config.fb_user_id)\n if \"error\" in res:\n return False\n return True\n\n\n def get_recommendations(self):\n '''\n Returns a list of users that you can swipe on\n '''\n try:\n r = requests.get('https://api.gotinder.com/user/recs', headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong with getting recomendations:\", e)\n\n\n def get_updates(self,last_activity_date=\"\"):\n '''\n Returns all updates since the given activity date.\n The last activity date is defaulted at the beginning of time.\n Format for last_activity_date: \"2017-07-09T10:28:13.392Z\"\n '''\n try:\n url = config.host + '/updates'\n r = requests.post(url,\n headers=headers,\n data=json.dumps({\"last_activity_date\": last_activity_date}))\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong with getting updates:\", e)\n\n\n def get_self(self):\n '''\n Returns your own profile data\n '''\n try:\n url = config.host + '/profile'\n r = requests.get(url, headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not get your data:\", e)\n\n\n def change_preferences(self,**kwargs):\n '''\n ex: change_preferences(age_filter_min=30, gender=0)\n kwargs: a dictionary - whose keys become separate keyword arguments and the values become values of these arguments\n age_filter_min: 18..46\n age_filter_max: 22..55\n age_filter_min <= age_filter_max - 4\n gender: 0 == seeking males, 1 == seeking females\n distance_filter: 1..100\n discoverable: true | false\n {\"photo_optimizer_enabled\":false}\n '''\n try:\n url = config.host + '/profile'\n r = requests.post(url, headers=headers, data=json.dumps(kwargs))\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not change your preferences:\", e)\n\n\n def get_meta(self):\n '''\n Returns meta data on yourself. Including the following keys:\n ['globals', 'client_resources', 'versions', 'purchases',\n 'status', 'groups', 'products', 'rating', 'tutorials',\n 'travel', 'notifications', 'user']\n '''\n try:\n url = config.host + '/meta'\n r = requests.get(url, headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not get your metadata:\", e)\n\n def get_meta_v2(self):\n '''\n Returns meta data on yourself from V2 API. Including the following keys:\n ['account', 'client_resources', 'plus_screen', 'boost',\n 'fast_match', 'top_picks', 'paywall', 'merchandising', 'places',\n 'typing_indicator', 'profile', 'recs']\n '''\n try:\n url = config.host + '/v2/meta'\n r = requests.get(url, headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not get your metadata:\", e)\n\n def update_location(self,lat, lon):\n '''\n Updates your location to the given float inputs\n Note: Requires a passport / Tinder Plus\n '''\n try:\n url = config.host + '/passport/user/travel'\n r = requests.post(url, headers=headers, data=json.dumps({\"lat\": lat, \"lon\": lon}))\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not update your location:\", e)\n\n def reset_real_location(self):\n try:\n url = config.host + '/passport/user/reset'\n r = requests.post(url, headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not update your location:\", e)\n\n\n def get_recs_v2(self):\n '''\n This works more consistently then the normal get_recommendations becuase it seeems to check new location\n '''\n try:\n url = config.host + '/v2/recs/core?locale=en-US'\n r = requests.get(url, headers=headers)\n return r.json()\n except Exception as e:\n print('excepted')\n\n def set_webprofileusername(self,username):\n '''\n Sets the username for the webprofile: https://www.gotinder.com/@YOURUSERNAME\n '''\n try:\n url = config.host + '/profile/username'\n r = requests.put(url, headers=headers,\n data=json.dumps({\"username\": username}))\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not set webprofile username:\", e)\n\n def reset_webprofileusername(self,username):\n '''\n Resets the username for the webprofile\n '''\n try:\n url = config.host + '/profile/username'\n r = requests.delete(url, headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not delete webprofile username:\", e)\n\n def get_person(self,id):\n '''\n Gets a user's profile via their id\n '''\n try:\n url = config.host + '/user/%s' % id\n r = requests.get(url, headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not get that person:\", e)\n\n\n def send_msg(self,match_id, msg):\n try:\n url = config.host + '/user/matches/%s' % match_id\n r = requests.post(url, headers=headers,\n data=json.dumps({\"message\": msg}))\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not send your message:\", e)\n\n def unmatch(self,match_id):\n try:\n url = config.host + '/user/matches/%s' % match_id\n r = requests.delete(url, headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not unmatch person:\", e)\n\n def superlike(self,person_id):\n try:\n url = config.host + '/like/%s/super' % person_id\n r = requests.post(url, headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not superlike:\", e)\n\n\n def like(self,person_id):\n try:\n url = config.host + '/like/%s' % person_id\n r = requests.get(url, headers=headers)\n print(r)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not like:\", e)\n\n\n def dislike(self,person_id,s_number):\n try:\n url = config.host + '/pass/%s' % person_id+'?s_number='+str(s_number)\n headers2=headers\n headers2.pop('content-type', None)\n r = requests.get(url, headers=headers2, timeout=0.7).json()\n return r\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not dislike:\", e)\n\n\n def report(self,person_id, cause, explanation=''):\n '''\n There are three options for cause:\n 0 : Other and requires an explanation\n 1 : Feels like spam and no explanation\n 4 : Inappropriate Photos and no explanation\n '''\n try:\n url = config.host + '/report/%s' % person_id\n r = requests.post(url, headers=headers, data={\n \"cause\": cause, \"text\": explanation})\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not report:\", e)\n\n\n def match_info(self,match_id):\n try:\n url = config.host + '/matches/%s' % match_id\n r = requests.get(url, headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not get your match info:\", e)\n\n def all_matches(self):\n try:\n url = config.host + '/v2/matches'\n r = requests.get(url, headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not get your match info:\", e)\n\n def fast_match_info(self):\n try:\n url = config.host + '/v2/fast-match/preview'\n r = requests.get(url, headers=headers)\n count = r.headers['fast-match-count']\n # image is in the response but its in hex..\n return count\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not get your fast-match count:\", e)\n\n def trending_gifs(self,limit=3):\n try:\n url = config.host + '/giphy/trending?limit=%s' % limit\n r = requests.get(url, headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not get the trending gifs:\", e)\n\n def gif_query(self,query, limit=3):\n try:\n url = config.host + '/giphy/search?limit=%s&query=%s' % (limit, query)\n r = requests.get(url, headers=headers)\n return r.json()\n except requests.exceptions.RequestException as e:\n print(\"Something went wrong. Could not get your gifs:\", e)\n\n\n # def see_friends(self):\n # try:\n # url = config.host + '/group/friends'\n # r = requests.get(url, headers=headers)\n # return r.json()['results']\n # except requests.exceptions.RequestException as e:\n # print(\"Something went wrong. Could not get your Facebook friends:\", e)\n\n\n\n def getUserInfo(self):\n recs=self.get_recs_v2()\n if recs == None:\n return \"Error\"\n if 'data' in recs:\n \n data=[]\n ids=[]\n try:\n \tresults=recs['data']['results']\n except:\n \treturn {'error':True,'msg':'no data'}\n\n for r in results:\n user_info={}\n user_info['name']=r['user']['name']\n user_info['uid']=r['user']['_id']\n ids.append(r['user']['_id'])\n try:\n user_info['bio']=r['user']['bio']\n except:\n user_info['bio']=''\n user_info['birth']=r['user']['birth_date']\n photos=[]\n for p in r['user']['photos']:\n photos.append(p['url'])\n user_info['photos']=photos\n try:\n user_info['jobs']=r['user']['jobs']\n except:\n user_info['jobs']=''\n try:\n user_info['schools']=r['user']['schools']\n except:\n user_info['schools']=''\n\n user_info['gender']=r['user']['gender']\n user_info['s_number']=r['s_number']\n try:\n inst=r['instagram']\n photos=[]\n for p in inst['photos']:\n photos.append(p['image'])\n user_info['instagram']=photos\n except:\n user_info['instagram']=[]\n\n\n user_info['location']=self.location\n\n data.append({'user_info':user_info,'id':r['user']['_id']})\n return {'data':data,'ids':ids}\n elif 'error' in recs:\n if recs['error']['code']==42901:\n print('Limit Rate')\n return {'error':True,'msg':'limit rate'}\n\n\n def diskike_users(self,user_info):\n for uinf in user_info:\n resp=self.dislike(uinf['uid'],uinf['s_number'])\n\n\n" }, { "path": "src/osint_sources/twitter.py", "content": "#!/usr/bin/python\n# coding=utf-8\n# encoding=utf8\nimport sys\nimport datetime\nfrom selenium.webdriver.common.keys import Keys\nimport time\nfrom selenium import webdriver\nimport os\nfrom parsel import Selector\nimport urllib.parse\nfrom selenium.common.exceptions import NoSuchElementException\nimport json\nfrom selenium.webdriver.chrome.options import Options\nfrom difflib import SequenceMatcher\nfrom osint_sources.recognition import *\n\ndef twitter (name_to_search,page_number,knownimage,verbose):\n\n placeToSearch='twitter.com'\n chrome_options = Options()\n chrome_options.add_argument(\"--headless\")\n chrome_options.add_argument(\"--no-sandbox\")\n chrome_options.add_argument(\"--disable-dev-shm-usage\")\n\n chrome_path = './chromedriver'\n driver = webdriver.Chrome(chrome_path,chrome_options=chrome_options)\n\n people_list=[]\n for i in range(int(page_number)):\n driver.get(\"https://www.google.com/search?q=site:\"+placeToSearch+\"+AND+\"+name_to_search + \"&start=\" + str(10 * i))\n search=driver.find_elements_by_tag_name('a')\n time.sleep(10)\n\n for s in search:\n href=s.get_attribute('href')\n if href != None:\n if \"https://twitter.com/\" in href:\n if \"/status/\" not in href and \"/media\" not in href and \"/hashtag/\" not in href and \"webcache.googleusercontent.com\" not in href and \"google.com\" not in href:\n people_list.append(href)\n elif \"/hashtag/\" not in href and \"webcache.googleusercontent.com\" not in href and \"google.com\" not in href:\n if \"/status/\" in href:\n people_list.append(href.split(\"/status/\")[0])\n elif \"/media\" not in s.text:\n people_list.append(href.split(\"/media\")[0])\n\n\n people_list=set(people_list)\n now = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n if not os.path.isdir(\"data/twitter\"):\n os.mkdir(\"data/twitter\");\n\n path=os.path.join('data/twitter',str(now)+'_twitter_data.json')\n jsonData=[]\n userLink = set()\n for p in people_list:\n if verbose:\n print(\"*******************************************************************************************************\")\n print(p)\n driver.get(p)\n driver.implicitly_wait(50)\n time.sleep(2)\n\n sel = Selector(text=driver.page_source)\n\n name = sel.xpath('//*[@id=\"react-root\"]/div/div/div[2]/main/div/div/div/div/div/div[2]/div/div/div[1]/div/div[2]/div/div/div[1]/div/span[1]/span/text()').extract_first()\n link = p\n description = sel.xpath('//*[@id=\"react-root\"]/div/div/div[2]/main/div/div/div/div/div/div[2]/div/div/div[1]/div/div[3]/div/div/span[1]/text()').extract_first()\n location = sel.xpath('//*[@id=\"react-root\"]/div/div/div[2]/main/div/div/div/div/div/div[2]/div/div/div[1]/div/div[4]/div/span[1]/span/span/text()').extract_first()\n member_since = sel.xpath('//*[@id=\"react-root\"]/div/div/div[2]/main/div/div/div/div/div/div[2]/div/div/div[1]/div/div[4]/div/span[2]/svg/text()').extract_first()\n born=sel.xpath('//*[@id=\"react-root\"]/div/div/div[2]/main/div/div/div/div/div/div[2]/div/div/div[1]/div/div[4]/div/span[2]/svg/text()').extract_first()\n webpage=sel.xpath('//*[@id=\"react-root\"]/div/div/div[2]/main/div/div/div/div/div/div[2]/div/div/div[1]/div/div[4]/div/a/text()').extract_first()\n image_url=sel.xpath('//*[@id=\"react-root\"]/div/div/div[2]/main/div/div/div/div/div/div[2]/div/div/div[1]/div/div[1]/a/div[1]/div[2]/div/img/@src').extract_first()\n if name==None:\n name=\"\"\n if str(link) not in userLink:\n userLink.add(link)\n nameParts = name_to_search.split(' ')\n isMatcher = False\n for n in nameParts:\n if SequenceMatcher(None,n, name).ratio()>0.4 or SequenceMatcher(None,n,str(link)).ratio()>0.4 or n in str(description).lower():\n isMatcher=True\n\n if SequenceMatcher(None,name_to_search, name).ratio()>0.4 or SequenceMatcher(None,name_to_search,str(link)).ratio()>0.4 or name_to_search in str(description).lower():\n isMatcher=True\n\n if isMatcher:\n userData = {}\n if verbose:\n print(\"Name: \"+str(name))\n print(\"Link: \"+str(link))\n print(\"Description: \"+str(description))\n print(\"Location: \"+ str(location))\n print(\"Member since: \"+str(member_since))\n print(\"Born: \"+str(born))\n print(\"Web: \"+str(webpage))\n print (\"Profile image url: \"+str(image_url))\n print('\\n')\n print('\\n')\n\n if knownimage:\n if not os.path.isdir(\"data/twitter/\"+str(now)+\"_images\"):\n os.mkdir(\"data/twitter/\"+str(now)+\"_images\");\n image=os.path.join(\"data/twitter/\"+str(now)+\"_images/\"+str(link.split('.com/')[1])+\".jpg\")\n try:\n urllib.request.urlretrieve(image_url, image)\n userData={'storedImage':image,'name':str(name),'link':str(link),'description':str(description),'location':str(location),'member_since':str(member_since),'born':str(born),'web':str(webpage),'image':str(image_url)}\n jsonData.append(userData)\n except:\n pass\n else:\n userData={'name':str(name),'link':str(link),'description':str(description),'location':str(location),'member_since':str(member_since),'born':str(born),'web':str(webpage),'image':str(image_url)}\n jsonData.append(userData)\n\n with open(path, 'w+') as outfile:\n json.dump(jsonData, outfile)\n\n print(\"Results Twitter in: \" + str(path))\n response={'results':str(path)}\n\n if len(people_list)>0:\n if knownimage:\n print(\"Compare similarity images.\")\n face_identification(knownimage,'./data/twitter/'+str(now)+'_images/')\n response['images']='./data/twitter/'+str(now)+'_images/'\n response['recognized']='./data/twitter/'+str(now)+'_images/recognized/'\n driver.quit()\n return response\n" }, { "path": "src/osint_sources/yandex.py", "content": "#!/usr/bin/python\n# coding: utf-8\n# encoding=utf8\nfrom selenium.webdriver.common.keys import Keys\nfrom selenium.webdriver.chrome.options import Options\nfrom selenium import webdriver\nfrom urllib.parse import unquote\nfrom os.path import isfile, join\nfrom os import listdir,remove\nimport time\nimport json\nimport os\nimport requests\nimport re\nimport datetime\nimport base64\nimport urllib.parse\nimport urllib.request\nfrom urllib.request import Request, urlopen\nimport random\nfrom bs4 import BeautifulSoup\nfrom fake_useragent import UserAgent\nfrom osint_sources.recognition import *\nimport sys\n\ndef isCaptcha(driver):\n\theaders=driver.find_elements_by_tag_name('h1')\n\tfor h in headers:\n\t\tif h.get_attribute('innerHTML')==\"oops…\":\n\t\t\tprint('captcha detected')\n\t\t\treturn(True)\n\treturn(False)\n\ndef checkProxy(proxy):\n\n\tproxies = {\n\t\"http\": proxy,\n\t\"https\": proxy,\n\t}\n\n\ttry:\n\t\tresp= requests.get(\"https://google.com\", proxies=proxies, timeout=20)\n\t\tprint(\"Google responses \"+str(resp.status_code)+\" in \"+str(resp.elapsed.total_seconds())+ \" seconds.\")\n\t\tif((resp.status_code!=200) or resp.elapsed.total_seconds()>10):\n\t\t\treturn 0\n\t\telse:\n\t\t\treturn 1\n\texcept:\n\t\tprint(\"Error while checking the proxy: \"+ str(proxy))\n\t\treturn 0\n\ndef crawlProxy():\n\t# Retrieve latest proxies\n\tua = UserAgent()\n\tproxies = []\n\tproxies_req = Request('https://www.sslproxies.org/')\n\tproxies_req.add_header('User-Agent', ua.random)\n\tproxies_doc = urlopen(proxies_req).read().decode('utf8')\n\n\tsoup = BeautifulSoup(proxies_doc, 'html.parser')\n\tproxies_table = soup.find(id='proxylisttable')\n\n\t# Save proxies in the array\n\tfor row in proxies_table.tbody.find_all('tr'):\n\t\tproxies.append({ 'ip': row.find_all('td')[0].string, 'port': row.find_all('td')[1].string})\n\n\tinvalidProxy=True\n\tproxy = None\n\twhile invalidProxy:\n\t\trandom_index = random.randint(0, len(proxies) - 1)\n\t\trandom_proxy = proxies[random_index]\n\t\tproxy = str(random_proxy['ip'])+\":\"+str(random_proxy['port'])\n\t\tchk=checkProxy(proxy)\n\t\tif chk:\n\t\t\tinvalidProxy=False\n\treturn proxy\n\ndef searchImages(driver,now,verbose):\n\tos.mkdir(\"data/yandex/\"+str(now)+\"_images\")\n\tsearch=driver.find_elements_by_class_name('other-sites__item')\n\tj=0\n\tprint('Retrieving images')\n\tout = []\n\tfor s in search:\n\t\ttry:\n\t\t\ta = s.find_elements_by_tag_name('a')\n\n\t\t\tfor i,al in enumerate(a):\n\t\t\t\taclass = al.get_attribute('class')\n\t\t\t\tif aclass == 'other-sites__preview-link':\n\t\t\t\t\tlink=al.get_attribute('href')\n\t\t\t\t\tif link != None and link != \"\":\n\t\t\t\t\t\tname=os.path.join('data/yandex/'+str(now)+'_images',str(j)+\"-yandex.jpg\")\n\t\t\t\t\t\tj=j+1\n\t\t\t\t\t\ttitle = s.find_elements_by_class_name('other-sites__snippet-title')[0]\n\t\t\t\t\t\tatittle= title.find_elements_by_tag_name('a')[0]\n\t\t\t\t\t\ttitle = atittle.get_attribute('innerText')\n\t\t\t\t\t\turl = atittle.get_attribute('href')\n\t\t\t\t\t\tdomain =s.find_elements_by_class_name('other-sites__snippet-site')[0]\n\t\t\t\t\t\tdomain= domain.find_elements_by_tag_name('a')[0].get_attribute('innerText')\n\t\t\t\t\t\tinfo ={}\n\t\t\t\t\t\tinfo[\"originUrl\"] = link\n\t\t\t\t\t\tinfo[\"title\"] = title\n\t\t\t\t\t\tinfo[\"url\"] = url\n\t\t\t\t\t\tinfo[\"domain\"] = domain\n\t\t\t\t\t\tif verbose:\n\t\t\t\t\t\t\tprint(\"-----------------\")\n\t\t\t\t\t\t\tprint(info)\n\t\t\t\t\t\tout.append(info)\n\t\t\t\t\t\ttry:\n\t\t\t\t\t\t\turllib.request.urlretrieve(link, name)\n\t\t\t\t\t\texcept:\n\t\t\t\t\t\t\tprint(\"Failed when downloading photo \" + str(j))\n\n\n\t\texcept Exception as e:\n\t\t\tprint(e)\n\n\treturn out\n\ndef deletedImage(hashimage, token):\n headers = {'Authorization': 'Client-ID ' + token}\n req = requests.delete(url= \"https://api.imgur.com/3/image/\" + hashimage, headers= headers)\n if req.status_code == requests.codes.ok:\n return True\n else:\n return False\n\n\ndef yandex(image,token,verbose):\n\tif not os.path.isdir(\"data/yandex\"):\n\t\tos.mkdir(\"data/yandex\");\n\timage_url = image\n\timage_delete = \"\"\n\tresults={}\n\turl = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+] |[!*\\(\\), ]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', image)\n\tif not url:\n\t\ttry:\n\t\t\tf = open(image, \"rb\")\n\t\texcept FileNotFoundError as e:\n\t\t\tprint (\"Image not found: \" + image)\n\t\t\treturn []\n\t\t\t#sys.exit(-1)\n\t\timage_data = f.read()\n\t\tb64_image = base64.standard_b64encode(image_data)\n\t\tclient_id = token\n\t\theaders = {'Authorization': 'Client-ID ' + token}\n\t\tdata = {'image': b64_image, 'title': 'test'}\n\t\ttry:\n\t\t\trequest = requests.post(url=\"https://api.imgur.com/3/upload.json\", data=data,headers=headers)\n\t\t\tif request.status_code == requests.codes.ok:\n\t\t\t\timage_url = request.json()['data']['link']\n\t\t\t\timage_delete = request.json()['data']['deletehash']\n\t\t\t\tprint (\"Image upload to imgur: \" + image_url)\n\t\texcept Exception as e:\n\t\t\tprint(e)\n\t\t\tsys.exit(-1)\n\n\n\tproxy=crawlProxy()\n\tif proxy is not None:\n\t\tprint (proxy)\n\t\tchrome_options = webdriver.ChromeOptions()\n\t\tchrome_options.add_argument(\"--headless\")\n\t\tchrome_options.add_argument('--proxy-server=%s' % proxy)\n\t\tchrome_options.add_argument(\"--no-sandbox\")\n\t\tchrome_options.add_argument(\"--disable-dev-shm-usage\")\n\t\tchrome_path = './chromedriver'\n\t\tdriver = webdriver.Chrome(chrome_path,options=chrome_options)\n\t\turl_final = \"https://yandex.ru/images/search?rpt=imageview&url=\"+image_url+\"&rpt=imagelike\"\n\t\tdriver.get(url_final)\n\t\tdriver.implicitly_wait(50)\n\t\ttime.sleep(3)\n\t\tcaptcha=isCaptcha(driver)\n\t\tif captcha == True:\n\t\t\tdriver.close()\n\t\telse:\n\t\t\tnow = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n\t\t\timages=searchImages(driver,now,verbose)\n\t\t\tif not images:\n\t\t\t\tprint('No images.')\n\t\t\t\tdriver.close()\n\t\t\tpath=os.path.join('data/yandex',str(now)+'_yandex_data.json')\n\t\t\twith open(path, 'w+') as outfile:\n\t\t\t\tjson.dump(images, outfile)\n\n\t\tif not token == None:\n\t\t\tif deletedImage(image_delete,token):\n\t\t\t\tprint (\"Image deleted\")\n\t\t\telse:\n\t\t\t\tprint(\"Problem when deleted image from imgur\")\n\n\t\tprint(\"Results Yandex in: \" + str(path))\n\t\tresults['results']=str(path)\n\telse:\n\t\tprint('Yandex is blocked')\n\n\treturn results\n" }, { "path": "src/requirements.txt", "content": "selenium == 3.141.0\nbs4 == 0.0.1\nrequests == 2.25.0\npeewee == 3.8.0\nrobobrowser == 0.5.3\nconfig == 0.4.0\nface-recognition==1.2.3\nopencv-python==4.4.0.46\nspacy==2.1.6\nurllib3==1.26.2\nfake-useragent==0.1.11\nparsel==1.6.0\n" }, { "path": "web/Docker-compose.yaml", "content": "version: \"3.8\"\nservices:\n front:\n build: ./front\n networks:\n - osint\n back:\n build: \"./back\"\n volumes:\n - ./data:/SpyScrap/src/data\n networks:\n - osint\n\n reverse-proxy:\n container_name: reverse-proxy\n build: reverse-proxy\n ports:\n - \"80:80\"\n networks:\n - osint\n depends_on:\n - back\n - front\nnetworks:\n osint:\n" }, { "path": "web/README.md", "content": "# WEB Wraper For SpyScrap\n\nRequirements docker-compose & docker\n\n\n### Installation\n\ndocker-compose up\n\n" }, { "path": "web/back/.gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndist/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# celery beat schedule file\ncelerybeat-schedule\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n\n# Spyder project settings\n.spyderproject\n.spyproject\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\ntinder.json\ndatabase.db\nchromedriver_linux64/\n/openface/\n/openface\n/openface/*\nchromedriver\n/osint_sources/*\n/osint_sources\nmain.py\nSpyScrap.png\nsetup.sh\n/back/README.md\n/build\n/build/*" }, { "path": "web/back/Dockerfile", "content": "FROM ubuntu:18.04\nRUN apt-get update \\\n && apt-get install -y python3-pip python3-dev \\\n && cd /usr/local/bin \\\n && ln -s /usr/bin/python3 python \\\n && pip3 install --upgrade pip\nRUN apt-get install -y --fix-missing \\\n build-essential \\\n cmake \\\n gfortran \\\n git \\\n wget \\\n curl \\\n pkg-config \\\n python3-dev \\\n python3-numpy \\\n software-properties-common \\\n zip \\\n unzip \\\n && apt-get clean && rm -rf /tmp/* /var/tmp/*\n\nRUN cd ~ && \\\n mkdir -p dlib && \\\n git clone -b 'v19.9' --single-branch https://github.com/davisking/dlib.git dlib/ && \\\n cd dlib/ && \\\n python3 setup.py install --yes USE_AVX_INSTRUCTIONS\n\n\n# Install Google Chrome\nRUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb\nRUN dpkg -i google-chrome-stable_current_amd64.deb; apt-get -fy install\n\n# Install Chrome WebDriver\nRUN CHROMEDRIVER_VERSION=`curl -sS chromedriver.storage.googleapis.com/LATEST_RELEASE` && \\\n mkdir -p /opt/chromedriver-$CHROMEDRIVER_VERSION && \\\n curl -sS -o /tmp/chromedriver_linux64.zip http://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip && \\\n unzip -qq /tmp/chromedriver_linux64.zip -d /opt/chromedriver-$CHROMEDRIVER_VERSION && \\\n rm /tmp/chromedriver_linux64.zip && \\\n chmod +x /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver && \\\n ln -fs /opt/chromedriver-$CHROMEDRIVER_VERSION/chromedriver /usr/local/bin/chromedriver\nRUN git clone https://github.com/RuthGnz/SpyScrap.git\nWORKDIR SpyScrap/src\nRUN pip3 install -r ./requirements.txt\nRUN rm requirements.txt\nRUN python -m spacy download es_core_news_sm\nRUN cp /usr/local/bin/chromedriver .\nCOPY ./osint-back/ .\nRUN pip3 install -r ./back_requirements.txt\nENV PYTHONIOENCODING=utf-8\nENTRYPOINT [\"python3\",\"server.py\"]\n" }, { "path": "web/back/osint-back/api.py", "content": "from flask import Flask, jsonify, request,send_from_directory\nimport logging\nfrom controller import *\nfrom flask_cors import CORS\napp = Flask(__name__,static_url_path = \"/data\", static_folder = \"./data\")\napp.config['UPLOAD_FOLDER'] = './uploads'\napp.config['ALLOWED_EXTENSIONS'] = ['png','jpeg','jpg']\nCORS(app)\n\n\nURL_BASE = '/osint/api/v1'\n\n@app.route(\"/osint/api/v1\")\ndef ping():\n return \"It works OSINT!\"\n\n\n@app.route(URL_BASE+\"/tinder\",methods=['POST'])\ndef tinder():\n\n\tname = request.form.get('name')\n\tcompany = request.form.get('company')\n\tfiles = request.files\n\tusers = []\n\tif len(files)>0 and not name and not company:\n\t\tusers=compareImages(files,None,app)\n\tif company and not name and len(files)==0:\n\t\tusers = getUsersByCompany(company)\n\telif name and not company and len(files)==0:\n\t\tusers = getUsersByName(name)\n\telif company and name and len(files)==0:\n\t\tusers = getUsersByCompanyAndName(company,name)\n\telif company and name and len(files)>0:\n\t\tusers=getUsersNameCompanyPhoto(company,name,files,app)\n\telif company and not name and len(files)>0:\n\t\tusers=getUsersByPhotoAndCompany(company,files,app)\n\telif not company and name and len(files)>0:\n\t\tusers=getUsersByNameAndPhoto(name,files,app)\n\n\treturn jsonify({'msg':users})\n\n@app.route(URL_BASE+\"/google\",methods=['POST'])\ndef google():\n\tdata=[]\n\tname = request.form.get('name')\n\tdownload = request.form.get('download')\n\tnumber = request.form.get('number')\n\tif name == None:\n\t\treturn jsonify(\"Name must me provided\")\n\tplace = request.form.get('place')\n\tfiles = request.files\n\tfiles = request.files\n\tif len(files)==0 and download=='true':\n\t\tfiles=None\n\tdata=google_controller(name,place,number,files,app)\n\treturn jsonify({'msg':data['data']})\n\n@app.route(URL_BASE+\"/instagram\",methods=['POST'])\ndef instagram():\n\tdata=[]\n\tname = request.form.get('name')\n\tdownload = request.form.get('download')\n\n\tif name == None:\n\t\treturn jsonify(\"Name must me provided\")\n\tfiles = request.files\n\tif len(files)==0 and download=='true':\n\t\tfiles=None\n\tdata=instagram_controller(name,files,app)\n\treturn jsonify({'msg':data['data']})\n\n\n##todo\n@app.route(URL_BASE+\"/twitter\",methods=['POST'])\ndef twitter():\n\tdata=[]\n\tname = request.form.get('name')\n\tnumber = request.form.get('number')\n\tdownload = request.form.get('download')\n\tif name == None:\n\t\treturn jsonify(\"Name must me provided\")\n\tif number==None:\n\t\tnumber=1\n\tfiles = request.files\n\tif len(files)==0 and download=='true':\n\t\tfiles=None\n\tdata=twitter_controller(name,files,number,app)\n\treturn jsonify({'msg':data['data']})\n\n\n@app.route(URL_BASE+\"/facebook\",methods=['POST'])\ndef facebook():\n\tdata=[]\n\tname = request.form.get('name')\n\tnumber = request.form.get('number')\n\tdownload = request.form.get('download')\n\tif name == None:\n\t\treturn jsonify(\"Name must me provided\")\n\tif number==None:\n\t\tnumber=1\n\tfiles = request.files\n\tif len(files)==0 and download=='true':\n\t\tfiles=None\n\tdata=facebook_controller(name,files,number,app)\n\treturn jsonify({'msg':data['data']})\n\n@app.route(URL_BASE+\"/boe\",methods=['POST'])\ndef boe():\n\tdata=[]\n\ttext = request.form.get('text')\n\tis_explicit = request.form.get('explicit')\n\tpages = request.form.get('pages')\n\tif text == None:\n\t\treturn jsonify(\"Text must me provided\")\n\tinitDate=None\n\toutDate=None\n\tdata=boe_controller(text,is_explicit,initDate,outDate,pages)\n\treturn jsonify({'msg':data['data']})\n\n@app.route(URL_BASE+\"/yandex\",methods=['POST'])\ndef yandex():\n\tdata=[]\n\turl = request.form.get('url')\n\ttoken = request.form.get('token')\n\tfiles = request.files\n\tdata=yandex_controller(url,files,token,app)\n\treturn jsonify({'msg':data['data']})\n\n\n@app.route(URL_BASE+\"/data///\")\ndef download_file(folder,dateFolder,image):\n\n return send_from_directory('./data/'+folder+'/'+dateFolder,image)\n\n@app.route(URL_BASE+\"/scoring\",methods=['POST'])\ndef scoring():\n\t#TODO\n\tname = request.form.get('name')\n\timgurl = request.form.get('imgurl')\n\tnumber = request.form.get('number')\n\tgnumber = request.form.get('gnumber')\n\tfiles = request.files\n\tif len(files)==0:\n\t\treturn jsonify({'msg':'Image must be sent'}),400\n\tdata=scoring_controller(name,imgurl,number,gnumber,files,app)\n\treturn jsonify({'msg':data})\n\nif __name__ == '__main__':\n app.run(debug=True)\n" }, { "path": "web/back/osint-back/back_model.py", "content": "from peewee import *\nimport datetime\nfrom playhouse.shortcuts import model_to_dict, dict_to_model\nimport json\nimport sys\nfrom flask import jsonify\nsys.path.insert(1, './CLI/')\nfrom osint_sources.scraper import *\n\nscriptDirectory = os.path.dirname(os.path.realpath(__file__))\nprint(scriptDirectory)\n\nclass User_Back(User):\n\n def getIds():\n result=User.select()\n response=[]\n for u in result.iterator():\n response.append(u.uid)\n return(response)\n\n def getById(arrayIds):\n userIds=User.select().where(User.id << arrayIds)\n result = []\n for u in userIds.iterator():\n try:\n job=Jobs.get(Jobs.user_id==u.id).job\n json_acceptable_string = job.replace(\"'\", \"\\\"\")\n job = json.loads(json_acceptable_string)\n user= {'name':u.name,'location':u.location,'birth':u.birth,'job':job}\n except:\n print(\"Unexpected error:\", sys.exc_info()[0])\n user= {'name':u.name,'location':u.location,'birth':u.birth,'job':''}\n photos = []\n ph = Photos.select().where(Photos.user == u)\n for i in ph.iterator():\n photos.append(i.photo)\n ig = InstagramPhotos.select().where(InstagramPhotos.user == u)\n for i in ig.iterator():\n photos.append(i.photo)\n userData={'user':user,'photos':photos}\n result.append(userData)\n return result\n\n def getByName(name):\n userIds=User.select().where(User.name.contains(name))\n result = []\n for u in userIds.iterator():\n try:\n job=Jobs.get(Jobs.user_id==u.id).job\n json_acceptable_string = job.replace(\"'\", \"\\\"\")\n job = json.loads(json_acceptable_string)\n user= {'name':u.name,'location':u.location,'birth':u.birth,'job':job}\n except:\n #print(\"Unexpected error:\", sys.exc_info()[0])\n user= {'name':u.name,'location':u.location,'birth':u.birth,'job':''}\n photos = []\n ph = Photos.select().where(Photos.user == u)\n for i in ph.iterator():\n photos.append(i.photo)\n ig = InstagramPhotos.select().where(InstagramPhotos.user == u)\n for i in ig.iterator():\n photos.append(i.photo)\n userData={'user':user,'photos':photos}\n result.append(userData)\n return result\n\n def getUserIdsByName(name):\n userIds=User.select().where(User.name.contains(name))\n result=[]\n for u in userIds:\n result.append(u)\n return result \n\n def getByJob(company):\n userIds=Jobs.select().where(Jobs.job.contains(company))\n result = []\n for u in userIds.iterator():\n user_obj = User.get(User.id==u.user)\n json_acceptable_string = u.job.replace(\"'\", \"\\\"\")\n job = json.loads(json_acceptable_string)\n user= {'name':user_obj.name,'location':user_obj.location,'birth':user_obj.birth,'job':job}\n photos = []\n ph = Photos.select().where(Photos.user == u.user)\n for i in ph.iterator():\n photos.append(i.photo)\n ig = InstagramPhotos.select().where(InstagramPhotos.user == u)\n for i in ig.iterator():\n photos.append(i.photo)\n userData={'user':user,'photos':photos}\n result.append(userData)\n\n return result\n\n def getUserIdsByCompany(company):\n userIds=Jobs.select().where(Jobs.job.contains(company))\n result=[]\n for u in userIds:\n result.append(u.user)\n return result\n \n def getByCompanyAndName(company,name):\n userIds=User.select().join(Jobs, on=(Jobs.user==User.id)).where(Jobs.job.contains(company) & User.name.contains(name))\n result = []\n for u in userIds.iterator():\n job_obj = Jobs.get(Jobs.user==u)\n\n json_acceptable_string = job_obj.job.replace(\"'\", \"\\\"\")\n try:\n job = json.loads(json_acceptable_string)\n except:\n job=''\n user= {'name':u.name,'location':u.location,'birth':u.birth,'job':job}\n photos = []\n ph = Photos.select().where(Photos.user == u)\n for i in ph.iterator():\n photos.append(i.photo)\n ig = InstagramPhotos.select().where(InstagramPhotos.user == u)\n for i in ig.iterator():\n photos.append(i.photo)\n userData={'user':user,'photos':photos}\n result.append(userData)\n\n return result\n\n def getUsersIdsByCompanyAndName(company,name):\n userIds=User.select().join(Jobs, on=(Jobs.user==User.id)).where(Jobs.job.contains(company) & User.name.contains(name))\n result = []\n for u in userIds:\n result.append(u)\n return result\n\nclass Photos_back(Photos):\n \n def getPhotos():\n result = []\n photos=Photos.select().join(User,on=(User.id==Photos.user))\n for p in photos:\n user= {'id':p.user.id,'name':p.user.name,'location':p.user.location,'birth':p.user.birth}\n userData={'user':user,'photo':p.photo}\n result.append(userData)\n return result\n\n def getPhotosByUsers(users):\n result=[]\n photos=Photos.select().join(User,on=(User.id==Photos.user)).where(User.id.in_(users))\n for p in photos:\n user= {'id':p.user.id,'name':p.user.name,'location':p.user.location,'birth':p.user.birth}\n userData={'user':user,'photo':p.photo}\n result.append(userData)\n return result" }, { "path": "web/back/osint-back/back_requirements.txt", "content": "Flask==1.0.2\nrequests==2.25.0\ngunicorn==19.8.1\npyjwt==1.7.1\nFlask-Cors==3.0.8\npeewee==3.8.0\ngevent==1.4.0\nurllib3==1.24.3\nnumpy==1.15.0\n" }, { "path": "web/back/osint-back/controller.py", "content": "from back_model import *\nfrom werkzeug.utils import secure_filename\nimport os\nfrom os import listdir,remove\nfrom PIL import Image\nimport requests\nfrom io import BytesIO\nfrom selenium.webdriver.common.keys import Keys\nimport time\nfrom selenium import webdriver\nimport os\nimport json\n#from api import app\nimport sys\nfrom osint_sources.scraper import *\nfrom osint_sources.recognition import *\nimport datetime\nimport urllib.request\nimport shutil\nimport re\nimport logging\n\n\ndef allowed_file(filename,app):\n return '.' in filename and \\\n filename.rsplit('.', 1)[1].lower() in app.config['ALLOWED_EXTENSIONS']\n\ndef getValidImagePath(knownFiles,app):\n\tlocation0=None\n\tfor f in knownFiles:\n\t\tfile = knownFiles[f]\n\t\tif file and allowed_file(file.filename,app):\n\t\t\tfilename = secure_filename(file.filename)\n\t\t\tlocation0 = os.path.join(app.config['UPLOAD_FOLDER'], filename)\n\t\t\tfile.save(location0)\n\t\t\tknown_image = face_recognition.load_image_file(location0)\n\t\t\tface_locations= face_recognition.face_locations(known_image)\n\t\t\tif len(face_locations)>0:\n\t\t\t\tbreak\n\treturn location0\n\n\ndef compareImages(knownFiles,users,app):\n\tnow = datetime.datetime.now()\n\tif users==None:\n\t\tphotos=Photos_back.getPhotos()\n\telse:\n\t\tphotos=Photos_back.getPhotosByUsers(users)\n\n\n\tfor f in knownFiles:\n\t\tfile = knownFiles[f]\n\t\tif file and allowed_file(file.filename,app):\n\t\t\tfilename = secure_filename(file.filename)\n\t\t\tlocation0 = os.path.join(app.config['UPLOAD_FOLDER'], filename)\n\t\t\tfile.save(location0)\n\t\t\tknown_image = face_recognition.load_image_file(location0)\n\t\t\tface_locations= face_recognition.face_locations(known_image)\n\t\t\tif len(face_locations)>0:\n\t\t\t\tbreak\n\tfolder=app.config['UPLOAD_FOLDER']+'/'+str(now)+'/'\n\t#save fotos in folder\n\tos.mkdir( folder);\n\tfor i,p in enumerate(photos):\n\t\ttry:\n\t\t\tlocation = os.path.join(folder, str(p['user']['id'])+'-'+str(i)+'.jpg')\n\t\t\turllib.request.urlretrieve(p['photo'], location)\n\t\texcept:\n\t\t\tpass\n\tface_identification(location0,folder)\n\t##get userIds from folder\n\t##remove photos\n\tuserIds=[]\n\n\t# r=root, d=directories, f = files\n\tfor r, d, f in os.walk(folder+'recognized'):\n\t\tfor file in f:\n\t\t\tif '.jpg' in file:\n\t\t\t\tfile=file.split('-')[0]\n\t\t\t\tuserIds.append(int(file))\n\n\tusers = User_Back.getById(userIds)\n\tremove(location0)\n\tshutil.rmtree(folder)\n\treturn users\n\ndef getUsersByPhotoAndCompany(company,knownFiles,app):\n\tusers = User_Back.getUserIdsByCompany(company)\n\tresult=compareImages(knownFiles,users,app)\n\treturn result\n\ndef getUsersByNameAndPhoto(name,knownFiles,app):\n\tresult = []\n\tusers = User_Back.getUserIdsByName(name)\n\tresult = compareImages(knownFiles,users,app)\n\treturn result\n\ndef getUsersNameCompanyPhoto(company,name,knownFiles,app):\n\tusers = User_Back.getUsersIdsByCompanyAndName(company,name)\n\tresult = compareImages(knownFiles,users,app)\n\treturn result\n\ndef getUsersByCompany(company):\n\tids = User_Back.getByJob(company)\n\treturn ids\n\ndef getUsersByCompanyAndName(company,name):\n\tusers = User_Back.getByCompanyAndName(company,name)\n\treturn users\n\ndef getUsersByName(name):\n\tusers = User_Back.getByName(name)\n\treturn users\n\n\ndef getUsersByCompanyAndNameAndImage(company,name,knownFiles):\n\tpass\n\n\n\ndef google_controller(toSearch,placeToSearch,number,knownFiles,app):\n\tif number ==\"\" or number==\"undefined\":\n\t\tnumber=None\n\tif placeToSearch == \"\" or placeToSearch==\"undefined\":\n\t\tplaceToSearch=None\n\tif type(knownFiles)== str:\n\t\tknownImage=knownFiles\n\telse:\n\t\tif knownFiles == None:\n\t\t\tknownImage='file'\n\t\telif len(knownFiles)>0:\n\t\t\tknownImage=getValidImagePath(knownFiles,app)\n\t\telse:\n\t\t\tknownImage=None\n\tpaths=google(toSearch,placeToSearch,knownImage,number,False)\n\tjsonPath=paths['results']\n\tresponse={}\n\twith open(jsonPath) as json_file:\n\t\tdata = json.load(json_file)\n\n\tif \"recognized\" in paths and knownImage!='file':\n\t\tfiles=[]\n\t\tfor r, d, f in os.walk(paths['recognized']):\n\t\t\tfor file in f:\n\t\t\t\tfiles.append(file)\n\t\tnewData = []\n\t\tfor a in data:\n\t\t\ttry:\n\t\t\t\timg = a['storedImage'].split('/')\n\t\t\t\timg= img[len(img)-1]\n\t\t\t\tif img in files:\n\t\t\t\t\tpath=a['storedImage'].split(img)[0]\n\t\t\t\t\tpath=path+\"recognized/\"+img\n\t\t\t\t\ta['storedImage']=path\n\t\t\t\t\tnewData.append(a)\n\t\t\texcept:\n\t\t\t\tpass\n\t\tdata=newData\n\tresponse['data']=data\n\treturn response\ndef instagram_controller(toSearch,knownFiles,app):\n\tif type(knownFiles)== str:\n\t\tknownImage=knownFiles\n\telse:\n\t\tif knownFiles == None:\n\t\t\tknownImage='file'\n\t\telif len(knownFiles)>0:\n\t\t\tknownImage=getValidImagePath(knownFiles,app)\n\t\telse:\n\t\t\tknownImage=None\n\tpaths = instagram(toSearch,knownImage,False)\n\tjsonPath=paths['results']\n\n\tresponse={}\n\n\twith open(jsonPath) as json_file:\n\t\tdata = json.load(json_file)\n\n\n\tif \"recognized\" in paths and knownImage!='file':\n\t\tfiles=[]\n\t\tfor r, d, f in os.walk(paths['recognized']):\n\t\t\tfor file in f:\n\t\t\t\tfiles.append(file)\n\t\tnewData = []\n\t\tfor a in data:\n\t\t\ttry:\n\t\t\t\timg = a['image'].split('/')\n\t\t\t\timg= img[len(img)-1]\n\t\t\t\tif img in files:\n\t\t\t\t\tpath=a['image'].split(img)[0]\n\t\t\t\t\tpath=path+\"recognized/\"+img\n\t\t\t\t\ta['image']=path\n\t\t\t\t\tnewData.append(a)\n\t\t\texcept:\n\t\t\t\tpass\n\t\tdata=newData\n\tresponse['data']=data\n\treturn response\n\ndef twitter_controller(name_to_search,knownFiles,page_number,app):\n\tif type(knownFiles)== str:\n\t\tknownImage=knownFiles\n\telse:\n\t\tif knownFiles == None:\n\t\t\tknownImage='file'\n\t\telif len(knownFiles)>0:\n\t\t\tknownImage=getValidImagePath(knownFiles,app)\n\t\telse:\n\t\t\tknownImage=None\n\tpaths=twitter(name_to_search,page_number,knownImage,False)\n\tjsonPath=paths['results']\n\tresponse={}\n\twith open(jsonPath) as json_file:\n\t\tdata = json.load(json_file)\n\n\tif \"recognized\" in paths and knownImage!='file':\n\t\tfiles=[]\n\t\tfor r, d, f in os.walk(paths['recognized']):\n\t\t\tprint(f)\n\t\t\tfor file in f:\n\t\t\t\tfiles.append(file)\n\t\tnewData = []\n\t\tfor a in data:\n\t\t\ttry:\n\t\t\t\timg = a['storedImage'].split('/')\n\t\t\t\timg= img[len(img)-1]\n\t\t\t\tif img in files:\n\t\t\t\t\tpath=a['storedImage'].split(img)[0]\n\t\t\t\t\tpath=path+\"recognized/\"+img\n\t\t\t\t\ta['storedImage']=path\n\t\t\t\t\tnewData.append(a)\n\t\t\texcept:\n\t\t\t\tpass\n\t\tdata=newData\n\tresponse['data']=data\n\treturn response\n\ndef facebook_controller(name_to_search,knownFiles,page_number,app):\n\tif type(knownFiles)== str:\n\t\tknownImage=knownFiles\n\telse:\n\t\tif knownFiles == None:\n\t\t\tknownImage='file'\n\t\telif len(knownFiles)>0:\n\t\t\tknownImage=getValidImagePath(knownFiles,app)\n\t\telse:\n\t\t\tknownImage=None\n\tpaths=facebook(name_to_search,knownImage,page_number,False)\n\tjsonPath=paths['results']\n\tresponse={}\n\twith open(jsonPath) as json_file:\n\t\tdata = json.load(json_file)\n\n\tif \"recognized\" in paths and knownImage!='file':\n\t\tfiles=[]\n\t\tfor r, d, f in os.walk(paths['recognized']):\n\t\t\tfor file in f:\n\t\t\t\tfiles.append(file)\n\t\tnewData = []\n\t\tfor a in data:\n\t\t\ttry:\n\t\t\t\timg = a['image'].split('/')\n\t\t\t\timg= img[len(img)-1]\n\t\t\t\tif img in files:\n\t\t\t\t\tpath=a['image'].split(img)[0]\n\t\t\t\t\tpath=path+\"recognized/\"+img\n\t\t\t\t\ta['image']=path\n\t\t\t\t\tnewData.append(a)\n\t\t\texcept:\n\t\t\t\tpass\n\t\tdata=newData\n\tresponse['data']=data\n\treturn response\n\ndef yandex_controller(url,knownFiles,token,app):\n\tlogging.warning('URL '+url)\n\tif url != None:\n\t\timage = url\n\telif type(knownFiles)== str:\n\t\tknownImage=knownFiles\n\telse:\n\t\tif knownFiles == None:\n\t\t\tknownImage='file'\n\t\telif len(knownFiles)>0:\n\t\t\tknownImage=getValidImagePath(knownFiles,app)\n\t\telse:\n\t\t\tknownImage=None\n\t\timage=\"\"\n\t\tif token== None and knownImage==None:\n\t\t\treturn \"error\"\n\t\telse:\n\t\t\timage=knownImage\n\tprint(image)\n\tpaths=yandex(image,token,False)\n\tresponse={}\n\tjsonPath=paths['results']\n\twith open(jsonPath) as json_file:\n\t\tdata = json.load(json_file)\n\t\tresponse['data']=data\n\treturn response\n\ndef boe_controller(text,is_explicit,initDate,outDate,pages):\n\tpaths=boe (text,initDate,outDate,pages,is_explicit,False)\n\tresponse={}\n\tjsonPath=paths['results']\n\twith open(jsonPath) as json_file:\n\t\tdata = json.load(json_file)\n\t\tresponse['data']=data\n\treturn response\n\ndef scoring_controller(name,url,number,gnumber,files,app):\n\timagePath=getValidImagePath(files,app)\n\tresponse = {}\n\ttry:\n\t\tins = instagram_controller(name,imagePath,app)\n\t\tresponse['instagram']=ins['data']\n\texcept:\n\t\tprint('instagram error')\n\t\tresponse['instagram']=[]\n\n\ttry:\n\t\tfb = facebook_controller(name,imagePath,number,app)\n\t\tresponse['facebook']=fb['data']\n\texcept:\n\t\tprint('facebook error')\n\t\tresponse['facebook']=[]\n\n\ttry:\n\t\tgl = google_controller(name,\"undefined\",gnumber,imagePath,app)\n\t\tresponse['google']=gl['data']\n\texcept:\n\t\tprint('google error')\n\t\tresponse['google']=[]\n\ttry:\n\t\ttw = twitter_controller(name,imagePath,number,app)\n\t\tresponse['twitter']=tw['data']\n\texcept:\n\t\tprint('twitter error')\n\t\tresponse['twitter']=[]\n\ttry:\n\t\tyn = yandex_controller(url,None,None,app)\n\t\tresponse['yandex']=yn['data']\n\texcept:\n\t\tprint('yandex error')\n\t\tresponse['yandex']=[]\n\n\n\tscore=compute_score(response)\n\tresponse['score']=score\n\treturn response\n\ndef compute_score(data):\n\t#TODO\n\t#TOTAL 100\n\t#Twitter 10\n\t#Instagram 10\n\t#Facebook 10\n\t#Yandex 10 ->\n\t#Google 60\n\tscore=0\n\tif len(data['twitter'])>0:\n\t\tscore=score+10\n\tif len(data['facebook'])>0:\n\t\tscore=score+10\n\tif len(data['instagram'])>0:\n\t\tscore=score+10\n\n\tif data['yandex'] != False:\n\t\tif len(data['yandex'])>0:\n\t\t\tyn=data['yandex']\n\t\t\tscore=score+1\n\t\t\tbonus=0\n\t\t\tif len(yn)>10:\n\t\t\t\tbonus=bonus+1\n\t\t\tif len(yn)>20:\n\t\t\t\tbonus=bonus+2\n\t\t\tif len(yn)>30:\n\t\t\t\tbonus=bonus+3;\n\t\t\tif len(yn)>40:\n\t\t\t\tbonus=bonus+3;\n\t\t\tscore=score+bonus\n\n\n\tif len(data['google'])>0:\n\n\t\tgn=data['google']\n\t\tscore=score+10\n\t\tbonus=0\n\t\tif len(gn)>2:\n\t\t\tbonus=bonus+5\n\t\tif len(gn)>5:\n\t\t\tbonus=bonus+5\n\t\tif len(gn)>10:\n\t\t\tbonus=bonus+5;\n\t\tif len(gn)>15:\n\t\t\tbonus=bonus+5;\n\t\tif len(gn)>25:\n\t\t\tbonus=bonus+5;\n\t\tif len(gn)>35:\n\t\t\tbonus=bonus+5;\n\t\tif len(gn)>45:\n\t\t\tbonus=bonus+5;\n\t\tif len(gn)>55:\n\t\t\tbonus=bonus+5;\n\t\tif len(gn)>65:\n\t\t\tbonus=bonus+5;\n\t\tif len(gn)>75:\n\t\t\tbonus=bonus+5;\n\t\tscore=score+bonus\n\treturn score\n" }, { "path": "web/back/osint-back/server.py", "content": "from gevent.pywsgi import WSGIServer\nfrom api import app # if yourapplication imports from views,\n # the sort would happen once; here.\napp.debug = True\nhttp_server = WSGIServer(('', 5000), app)\nhttp_server.serve_forever()" }, { "path": "web/back/osint-back/uploads/.gitignore", "content": "# Ignore everything in this directory\n*\n# Except this file\n!.gitignore" }, { "path": "web/data/.gitignore", "content": "# Ignore everything in this directory\n*\n# Except this file\n!.gitignore\n" }, { "path": "web/front/.gitignore", "content": ".DS_Store\nThumbs.db\ndb.json\n*.log\nnode_modules/\n.deploy*/\nsrc/_drafts\npackage-lock.json\nrequirements.txt" }, { "path": "web/front/Dockerfile", "content": "FROM node:15.2.1-alpine3.10\n\n# instalar un simple servidor http para servir nuestro contenido estático\nRUN yarn global add http-server\n\n# hacer la carpeta 'app' el directorio de trabajo actual\nWORKDIR /app\n\n# copiar 'package.json' y 'package-lock.json' (si están disponibles)\nCOPY ./osint-front/package*.json ./\n\n# instalar dependencias del proyecto\nRUN yarn install\n\n# copiar los archivos y carpetas del proyecto al directorio de trabajo actual (es decir, la carpeta 'app')\nCOPY ./osint-front/ .\n\n# construir aplicación para producción minificada\nRUN yarn run build\n\nEXPOSE 8080\nCMD [ \"http-server\", \"dist\" ]\n" }, { "path": "web/front/osint-front/.gitignore", "content": ".DS_Store\nnode_modules\n/dist\n\n# local env files\n.env.local\n.env.*.local\n\n# Log files\nnpm-debug.log*\nyarn-debug.log*\nyarn-error.log*\n\n# Editor directories and files\n.idea\n.vscode\n*.suo\n*.ntvs*\n*.njsproj\n*.sln\n*.sw?\n" }, { "path": "web/front/osint-front/README.md", "content": "# front\n\n## Project setup\n```\nyarn install\n```\n\n### Compiles and hot-reloads for development\n```\nyarn serve\n```\n\n### Compiles and minifies for production\n```\nyarn build\n```\n\n### Lints and fixes files\n```\nyarn lint\n```\n\n### Customize configuration\nSee [Configuration Reference](https://cli.vuejs.org/config/).\n" }, { "path": "web/front/osint-front/babel.config.js", "content": "module.exports = {\n presets: [\n '@vue/cli-plugin-babel/preset'\n ]\n}\n" }, { "path": "web/front/osint-front/package.json", "content": "{\n \"name\": \"front\",\n \"version\": \"0.1.0\",\n \"private\": true,\n \"scripts\": {\n \"serve\": \"vue-cli-service serve\",\n \"build\": \"vue-cli-service build\",\n \"lint\": \"vue-cli-service lint\"\n },\n \"dependencies\": {\n \"core-js\": \"^3.6.5\",\n \"vue\": \"^2.6.11\",\n \"vue-resource\": \"^1.5.1\",\n \"vue-router\": \"^3.4.9\",\n \"vue-toast-notification\": \"^0.6.0\",\n \"vuetify\": \"^2.2.11\",\n \"vuetify-numeric\": \"^0.1.7\"\n },\n \"devDependencies\": {\n \"@vue/cli-plugin-babel\": \"~4.5.0\",\n \"@vue/cli-plugin-eslint\": \"~4.5.0\",\n \"@vue/cli-service\": \"~4.5.0\",\n \"babel-eslint\": \"^10.1.0\",\n \"eslint\": \"^6.7.2\",\n \"eslint-plugin-vue\": \"^6.2.2\",\n \"sass\": \"^1.19.0\",\n \"sass-loader\": \"^8.0.0\",\n \"vue-cli-plugin-vuetify\": \"~2.0.7\",\n \"vue-template-compiler\": \"^2.6.11\",\n \"vuetify-loader\": \"^1.3.0\"\n },\n \"eslintConfig\": {\n \"root\": true,\n \"env\": {\n \"node\": true\n },\n \"extends\": [\n \"plugin:vue/essential\",\n \"eslint:recommended\"\n ],\n \"parserOptions\": {\n \"parser\": \"babel-eslint\"\n },\n \"rules\": {}\n },\n \"browserslist\": [\n \"> 1%\",\n \"last 2 versions\",\n \"not dead\"\n ]\n}\n" }, { "path": "web/front/osint-front/public/index.html", "content": "\n\n \n \n \n \n \n favicon.ico\">\n SpyScrap\n \n \n \n \n \n
\n \n \n \n\n" }, { "path": "web/front/osint-front/src/App.vue", "content": "\n\n\n" }, { "path": "web/front/osint-front/src/main.js", "content": "import Vue from 'vue'\nimport App from './App.vue'\nimport router from \"./router\";\nimport vuetify from './plugins/vuetify';\nimport VueResource from \"vue-resource\";\n\nVue.config.productionTip = false\n\nnew Vue({\n router,\n vuetify,\n render: h => h(App)\n}).$mount('#app')\nVue.use(VueResource);\n" }, { "path": "web/front/osint-front/src/plugins/vuetify.js", "content": "import Vue from 'vue';\nimport Vuetify from 'vuetify/lib';\n\nVue.use(Vuetify);\n\nexport default new Vuetify({\n});\n" }, { "path": "web/front/osint-front/src/router.js", "content": "import Vue from \"vue\";\nimport Router from \"vue-router\";\nimport Home from \"./views/Home.vue\";\n\nVue.use(Router)\n\nexport default new Router({\n mode: \"hash\",\n base: process.env.BASE_URL,\n routes: [\n {\n path: \"/\",\n name: \"home\",\n component: Home\n },\n {\n path: \"/tinder\",\n name: \"tinder\",\n component: () =>\n import(/* webpackChunkName: \"about\" */ \"./views/Tinder.vue\")\n },\n {\n path: \"/google\",\n name: \"google\",\n component: () =>\n import(/* webpackChunkName: \"about\" */ \"./views/Google.vue\")\n },\n {\n path: \"/yandex\",\n name: \"yandex\",\n component: () =>\n import(/* webpackChunkName: \"about\" */ \"./views/Yandex.vue\")\n },\n {\n path: \"/boe\",\n name: \"boe\",\n component: () => import(/* webpackChunkName: \"about\" */ \"./views/Boe.vue\")\n },\n {\n path: \"/instagram\",\n name: \"instagram\",\n component: () =>\n import(/* webpackChunkName: \"about\" */ \"./views/Instagram.vue\")\n },\n {\n path: \"/facebook\",\n name: \"facebook\",\n component: () =>\n import(/* webpackChunkName: \"about\" */ \"./views/Facebook.vue\")\n },\n {\n path: \"/twitter\",\n name: \"twitter\",\n component: () =>\n import(/* webpackChunkName: \"about\" */ \"./views/Twitter.vue\")\n },\n {\n path: \"/score\",\n name: \"score\",\n component: () =>\n import(/* webpackChunkName: \"about\" */ \"./views/Score.vue\")\n },\n {\n path: \"/about\",\n name: \"about\",\n // route level code-splitting\n // this generates a separate chunk (about.[hash].js) for this route\n // which is lazy-loaded when the route is visited.\n component: () =>\n import(/* webpackChunkName: \"about\" */ \"./views/About.vue\")\n }\n ]\n});\n" }, { "path": "web/front/osint-front/src/views/About.vue", "content": "\n" }, { "path": "web/front/osint-front/src/views/Boe.vue", "content": "\n\n" }, { "path": "web/front/osint-front/src/views/Facebook.vue", "content": "\n\n\n" }, { "path": "web/front/osint-front/src/views/Google.vue", "content": "\n\n\n" }, { "path": "web/front/osint-front/src/views/Home.vue", "content": "\n\n\n" }, { "path": "web/front/osint-front/src/views/Instagram.vue", "content": "\n\n\n" }, { "path": "web/front/osint-front/src/views/Score.vue", "content": "\n\n\n" }, { "path": "web/front/osint-front/src/views/Tinder.vue", "content": "\n\n\n" }, { "path": "web/front/osint-front/src/views/Twitter.vue", "content": "\n\n\n" }, { "path": "web/front/osint-front/src/views/Yandex.vue", "content": "\n\n\n" }, { "path": "web/front/osint-front/vue.config.js", "content": "module.exports = {\n \"transpileDependencies\": [\n \"vuetify\"\n ]\n}" }, { "path": "web/reverse-proxy/Dockerfile", "content": "FROM nginx:1.19-alpine\n\nCOPY nginx.conf /etc/nginx/nginx.conf\n\nEXPOSE 80\n" }, { "path": "web/reverse-proxy/nginx.conf", "content": "worker_processes 4;\n\nevents { worker_connections 1024; }\n\nhttp {\n # Basic Settings\n sendfile on;\n tcp_nopush on;\n tcp_nodelay on;\n keepalive_timeout 65;\n types_hash_max_size 2048;\n proxy_connect_timeout 300000;\n proxy_send_timeout 300000;\n proxy_read_timeout 300000;\n send_timeout 300000;\n\n\n\n upstream backend {\n server back:5000;\n }\n upstream frontend {\n server front:8080;\n }\n server {\n\n listen 80;\n\n server_name localhost;\n\n if ($http_x_forwarded_proto = 'http') {\n return 301 https://$server_name$request_uri;\n }\n\n access_log /var/log/nginx/client.access.log;\n error_log /var/log/nginx/client.error.log;\n\n\n gzip on;\n gzip_http_version 1.1;\n gzip_min_length 1100;\n gzip_vary on;\n gzip_proxied expired no-cache no-store private auth;\n gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript text/x-js;\n gzip_comp_level 9;\n\n location / {\n proxy_pass http://frontend;\n proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;\n proxy_redirect off;\n proxy_buffering off;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n\n }\n\n location /osint {\n\n proxy_pass http://backend;\n proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;\n proxy_redirect off;\n proxy_buffering off;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n\n add_header 'Access-Control-Allow-Credentials' 'true';\n add_header 'Access-Control-Allow-Origin' 'api';\n }\n location /data {\n\n proxy_pass http://backend;\n proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;\n proxy_redirect off;\n proxy_buffering off;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n\n add_header 'Access-Control-Allow-Credentials' 'true';\n add_header 'Access-Control-Allow-Origin' 'api';\n }\n\n }\n}\n" } ]