Repository: Guezone/SECMON
Branch: master
Commit: 04915cc21f0e
Files: 282
Total size: 9.1 MB
Directory structure:
gitextract_dmivgmii/
├── .gitattributes
├── .gitignore
├── .gitlab-ci.yml
├── DOCS.md
├── Dockerfile
├── LICENSE
├── README.md
├── SECURITY.md
├── cve_poller.py
├── cve_template.html
├── cve_updater.py
├── docker/
│ ├── install.py
│ ├── secmon-ci.conf
│ └── secmon.conf
├── requirements.txt
├── rss_poller.py
├── rss_template.html
├── secmon.py
├── secmon.wsgi
├── secmon_lib.py
├── secmon_monitor.py
├── secmon_web.py
├── setup.py
├── static/
│ ├── css/
│ │ ├── dataTables.bootstrap.css
│ │ ├── main.css
│ │ ├── main2.css
│ │ ├── morris.css
│ │ ├── mycss.css
│ │ ├── sb-admin-2.css
│ │ ├── site.css
│ │ ├── timeline.css
│ │ ├── util.css
│ │ └── util2.css
│ ├── fonts/
│ │ ├── font-awesome-4.7.0/
│ │ │ ├── HELP-US-OUT.txt
│ │ │ ├── css/
│ │ │ │ └── font-awesome.css
│ │ │ ├── fonts/
│ │ │ │ └── FontAwesome.otf
│ │ │ ├── less/
│ │ │ │ ├── animated.less
│ │ │ │ ├── bordered-pulled.less
│ │ │ │ ├── core.less
│ │ │ │ ├── fixed-width.less
│ │ │ │ ├── font-awesome.less
│ │ │ │ ├── icons.less
│ │ │ │ ├── larger.less
│ │ │ │ ├── list.less
│ │ │ │ ├── mixins.less
│ │ │ │ ├── path.less
│ │ │ │ ├── rotated-flipped.less
│ │ │ │ ├── screen-reader.less
│ │ │ │ ├── stacked.less
│ │ │ │ └── variables.less
│ │ │ └── scss/
│ │ │ ├── _animated.scss
│ │ │ ├── _bordered-pulled.scss
│ │ │ ├── _core.scss
│ │ │ ├── _fixed-width.scss
│ │ │ ├── _icons.scss
│ │ │ ├── _larger.scss
│ │ │ ├── _list.scss
│ │ │ ├── _mixins.scss
│ │ │ ├── _path.scss
│ │ │ ├── _rotated-flipped.scss
│ │ │ ├── _screen-reader.scss
│ │ │ ├── _stacked.scss
│ │ │ ├── _variables.scss
│ │ │ └── font-awesome.scss
│ │ ├── montserrat/
│ │ │ └── OFL.txt
│ │ └── raleway/
│ │ └── OFL.txt
│ ├── js/
│ │ ├── demo/
│ │ │ ├── chart-area-demo.js
│ │ │ ├── chart-bar-demo.js
│ │ │ ├── chart-pie-demo.js
│ │ │ └── datatables-demo.js
│ │ ├── main.js
│ │ └── sb-admin-2.js
│ ├── scss/
│ │ ├── _buttons.scss
│ │ ├── _cards.scss
│ │ ├── _charts.scss
│ │ ├── _dropdowns.scss
│ │ ├── _error.scss
│ │ ├── _footer.scss
│ │ ├── _global.scss
│ │ ├── _login.scss
│ │ ├── _mixins.scss
│ │ ├── _navs.scss
│ │ ├── _utilities.scss
│ │ ├── _variables.scss
│ │ ├── navs/
│ │ │ ├── _global.scss
│ │ │ ├── _sidebar.scss
│ │ │ └── _topbar.scss
│ │ ├── sb-admin-2.scss
│ │ └── utilities/
│ │ ├── _animation.scss
│ │ ├── _background.scss
│ │ ├── _border.scss
│ │ ├── _display.scss
│ │ ├── _progress.scss
│ │ ├── _rotate.scss
│ │ └── _text.scss
│ └── vendor/
│ ├── animate/
│ │ └── animate.css
│ ├── bootstrap/
│ │ ├── css/
│ │ │ ├── bootstrap-grid.css
│ │ │ ├── bootstrap-reboot.css
│ │ │ └── bootstrap.css
│ │ ├── js/
│ │ │ ├── bootstrap.bundle.js
│ │ │ ├── bootstrap.js
│ │ │ ├── popper.js
│ │ │ └── tooltip.js
│ │ └── scss/
│ │ ├── _alert.scss
│ │ ├── _badge.scss
│ │ ├── _breadcrumb.scss
│ │ ├── _button-group.scss
│ │ ├── _buttons.scss
│ │ ├── _card.scss
│ │ ├── _carousel.scss
│ │ ├── _close.scss
│ │ ├── _code.scss
│ │ ├── _custom-forms.scss
│ │ ├── _dropdown.scss
│ │ ├── _forms.scss
│ │ ├── _functions.scss
│ │ ├── _grid.scss
│ │ ├── _images.scss
│ │ ├── _input-group.scss
│ │ ├── _jumbotron.scss
│ │ ├── _list-group.scss
│ │ ├── _media.scss
│ │ ├── _mixins.scss
│ │ ├── _modal.scss
│ │ ├── _nav.scss
│ │ ├── _navbar.scss
│ │ ├── _pagination.scss
│ │ ├── _popover.scss
│ │ ├── _print.scss
│ │ ├── _progress.scss
│ │ ├── _reboot.scss
│ │ ├── _root.scss
│ │ ├── _spinners.scss
│ │ ├── _tables.scss
│ │ ├── _toasts.scss
│ │ ├── _tooltip.scss
│ │ ├── _transitions.scss
│ │ ├── _type.scss
│ │ ├── _utilities.scss
│ │ ├── _variables.scss
│ │ ├── bootstrap-grid.scss
│ │ ├── bootstrap-reboot.scss
│ │ ├── bootstrap.scss
│ │ ├── mixins/
│ │ │ ├── _alert.scss
│ │ │ ├── _background-variant.scss
│ │ │ ├── _badge.scss
│ │ │ ├── _border-radius.scss
│ │ │ ├── _box-shadow.scss
│ │ │ ├── _breakpoints.scss
│ │ │ ├── _buttons.scss
│ │ │ ├── _caret.scss
│ │ │ ├── _clearfix.scss
│ │ │ ├── _deprecate.scss
│ │ │ ├── _float.scss
│ │ │ ├── _forms.scss
│ │ │ ├── _gradients.scss
│ │ │ ├── _grid-framework.scss
│ │ │ ├── _grid.scss
│ │ │ ├── _hover.scss
│ │ │ ├── _image.scss
│ │ │ ├── _list-group.scss
│ │ │ ├── _lists.scss
│ │ │ ├── _nav-divider.scss
│ │ │ ├── _pagination.scss
│ │ │ ├── _reset-text.scss
│ │ │ ├── _resize.scss
│ │ │ ├── _screen-reader.scss
│ │ │ ├── _size.scss
│ │ │ ├── _table-row.scss
│ │ │ ├── _text-emphasis.scss
│ │ │ ├── _text-hide.scss
│ │ │ ├── _text-truncate.scss
│ │ │ ├── _transition.scss
│ │ │ └── _visibility.scss
│ │ ├── utilities/
│ │ │ ├── _align.scss
│ │ │ ├── _background.scss
│ │ │ ├── _borders.scss
│ │ │ ├── _clearfix.scss
│ │ │ ├── _display.scss
│ │ │ ├── _embed.scss
│ │ │ ├── _flex.scss
│ │ │ ├── _float.scss
│ │ │ ├── _interactions.scss
│ │ │ ├── _overflow.scss
│ │ │ ├── _position.scss
│ │ │ ├── _screenreaders.scss
│ │ │ ├── _shadows.scss
│ │ │ ├── _sizing.scss
│ │ │ ├── _spacing.scss
│ │ │ ├── _stretched-link.scss
│ │ │ ├── _text.scss
│ │ │ └── _visibility.scss
│ │ └── vendor/
│ │ └── _rfs.scss
│ ├── chart.js/
│ │ ├── Chart.bundle.js
│ │ └── Chart.js
│ ├── css-hamburgers/
│ │ └── hamburgers.css
│ ├── datatables/
│ │ ├── dataTables.bootstrap4.css
│ │ ├── dataTables.bootstrap4.js
│ │ └── jquery.dataTables.js
│ ├── fontawesome-free/
│ │ ├── LICENSE.txt
│ │ ├── css/
│ │ │ ├── all.css
│ │ │ ├── brands.css
│ │ │ ├── fontawesome.css
│ │ │ ├── regular.css
│ │ │ ├── solid.css
│ │ │ ├── svg-with-js.css
│ │ │ └── v4-shims.css
│ │ ├── js/
│ │ │ ├── all.js
│ │ │ ├── brands.js
│ │ │ ├── conflict-detection.js
│ │ │ ├── fontawesome.js
│ │ │ ├── regular.js
│ │ │ ├── solid.js
│ │ │ └── v4-shims.js
│ │ ├── less/
│ │ │ ├── _animated.less
│ │ │ ├── _bordered-pulled.less
│ │ │ ├── _core.less
│ │ │ ├── _fixed-width.less
│ │ │ ├── _icons.less
│ │ │ ├── _larger.less
│ │ │ ├── _list.less
│ │ │ ├── _mixins.less
│ │ │ ├── _rotated-flipped.less
│ │ │ ├── _screen-reader.less
│ │ │ ├── _shims.less
│ │ │ ├── _stacked.less
│ │ │ ├── _variables.less
│ │ │ ├── brands.less
│ │ │ ├── fontawesome.less
│ │ │ ├── regular.less
│ │ │ ├── solid.less
│ │ │ └── v4-shims.less
│ │ ├── metadata/
│ │ │ ├── categories.yml
│ │ │ ├── icons.yml
│ │ │ ├── shims.yml
│ │ │ └── sponsors.yml
│ │ ├── package.json
│ │ └── scss/
│ │ ├── _animated.scss
│ │ ├── _bordered-pulled.scss
│ │ ├── _core.scss
│ │ ├── _fixed-width.scss
│ │ ├── _icons.scss
│ │ ├── _larger.scss
│ │ ├── _list.scss
│ │ ├── _mixins.scss
│ │ ├── _rotated-flipped.scss
│ │ ├── _screen-reader.scss
│ │ ├── _shims.scss
│ │ ├── _stacked.scss
│ │ ├── _variables.scss
│ │ ├── brands.scss
│ │ ├── fontawesome.scss
│ │ ├── regular.scss
│ │ ├── solid.scss
│ │ └── v4-shims.scss
│ ├── jquery/
│ │ ├── jquery.js
│ │ └── jquery.slim.js
│ ├── jquery-easing/
│ │ ├── jquery.easing.compatibility.js
│ │ └── jquery.easing.js
│ └── select2/
│ ├── select2.css
│ └── select2.js
└── templates/
├── about.html
├── base.html
├── by-product-vulns.html
├── config.html
├── cve-alerts.html
├── cve-updates.html
├── cyber-threats.html
├── errors.html
├── exploits.html
├── home.html
├── login.html
├── logs.html
├── product-card.html
├── product-mgmt.html
├── reports.html
├── result.html
├── rss-news.html
├── search.html
├── settings.html
├── tasks.html
└── vuln-mgmt.html
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitattributes
================================================
# Auto detect text files and perform LF normalization
* text=auto
================================================
FILE: .gitignore
================================================
SECMON-CVE-Report.xlsx
secmon.db
logs.txt
__pycache__/
stats.py
test.py
certs/
================================================
FILE: .gitlab-ci.yml
================================================
variables:
PRIVATE_REGISTRY_PASSWD: $PRIVATE_REGISTRY_PASSWD
PRIVATE_REGISTRY_USERNAME: $PRIVATE_REGISTRY_USERNAME
PRIVATE_REGISTRY_URL: $PRIVATE_REGISTRY_URL
DOCKER_HUB_PASSWD: $DOCKER_HUB_PASSWD
DOCKER_HUB_USERNAME: $DOCKER_HUB_USERNAME
FQDN: $SECMON_FQDN
SENDER: $SECMON_SENDER
SMTP_PASSWD: $SECMON_SMTP_PASSWD
SMTP_SRV: $SECMON_SMTP_SRV
GITHUB_USERNAME: $SECMON_GITHUB_USERNAME
GITHUB_API_KEY: $SECMON_GITHUB_API_KEY
WEB_USERNAME: $SECMON_WEB_USERNAME
WEB_PASSWD: $SECMON_WEB_PASSWD
stages:
- build
- vuln_scan
- malware_scan
- test
- release_image
Build-Image:
stage: build
script:
- docker login $PRIVATE_REGISTRY_URL -u $PRIVATE_REGISTRY_USERNAME -p $PRIVATE_REGISTRY_PASSWD
- docker pull $SECMON_REGISTRY_IMAGE_WITHOUT_TAG:latest || true
- docker build . -t $SECMON_REGISTRY_IMAGE_WITHOUT_TAG:$CI_COMMIT_REF_SLUG
- docker push $SECMON_REGISTRY_IMAGE_WITHOUT_TAG:$CI_COMMIT_REF_SLUG
allow_failure: false
Vuln-Scan:
stage: vuln_scan
variables:
ANCHORE_CLI_URL: "http://anchore-engine:8228/v1"
GIT_STRATEGY: none
image: docker.io/anchore/inline-scan:latest
services:
- name: docker.io/anchore/inline-scan:latest
alias: anchore-engine
command: ["start"]
script:
- anchore-cli system wait
- anchore-cli registry add $PRIVATE_REGISTRY_URL gitlab-ci-token "$CI_JOB_TOKEN" --skip-validate --insecure
- anchore-cli image add $SECMON_REGISTRY_IMAGE_WITHOUT_TAG:$CI_COMMIT_REF_SLUG
- anchore-cli image wait $SECMON_REGISTRY_IMAGE_WITHOUT_TAG:$CI_COMMIT_REF_SLUG
- anchore-cli image vuln $SECMON_REGISTRY_IMAGE_WITHOUT_TAG:$CI_COMMIT_REF_SLUG all
allow_failure: true
Malware-Scan:
stage: malware_scan
script:
- docker stop secmontest && docker rm secmontest || true
- docker login $PRIVATE_REGISTRY_URL -u $PRIVATE_REGISTRY_USERNAME -p $PRIVATE_REGISTRY_PASSWD
- docker run -i -t --name secmontest -d $SECMON_REGISTRY_IMAGE_WITHOUT_TAG:$CI_COMMIT_REF_SLUG
- docker cp . secmontest:/var/www/secmon
- docker exec secmontest bash -c 'apt update && apt install -y clamav && freshclam'
- docker exec secmontest bash -c 'cd /var/www/secmon ; python3 -c "import secmon_lib" ; python3 -c "from secmon_web import *" 2>/dev/null' || true
- docker exec secmontest clamscan --remove --infected -r /var/www /home /usr /etc /mnt /media /srv | tee scan-results.txt
- docker stop secmontest && docker rm secmontest
allow_failure: false
Test:
stage: test
script:
- docker stop secmontest && docker rm secmontest || true
- docker login $PRIVATE_REGISTRY_URL -u $PRIVATE_REGISTRY_USERNAME -p $PRIVATE_REGISTRY_PASSWD
- docker run -i -t --name secmontest -p 4443 -e SENDER=$SENDER -e GITHUB_USERNAME=$GITHUB_USERNAME -e GITHUB_API_KEY=$GITHUB_API_KEY -e WEB_USERNAME=$WEB_USERNAME -e WEB_PASSWD=$WEB_PASSWD -e FQDN=$FQDN -e SMTP_PASSWD=$SMTP_PASSWD -d $SECMON_REGISTRY_IMAGE_WITHOUT_TAG:$CI_COMMIT_REF_SLUG
- docker cp . secmontest:/var/www/secmon
- docker exec secmontest openssl req -newkey rsa:4096 -x509 -subj "/C=FR/ST=France/L=Paris/O=Test/CN=$FQDN" -sha512 -days 3650 -nodes -out /etc/ssl/secmon/secmon.crt -keyout /etc/ssl/secmon/secmon.key
- docker exec secmontest python3 /var/www/secmon/setup.py -login $SENDER -p $SMTP_PASSWD -server $SMTP_SRV -port '587' -tls yes -lang fr -sender $SENDER -r $SENDER -accept true -github_username $GITHUB_USERNAME -github_api_key $GITHUB_API_KEY -username $WEB_USERNAME -password $WEB_PASSWD
- docker exec secmontest nohup python3 /var/www/secmon/secmon.py >/dev/null 2>&1 &
- docker exec secmontest nohup python3 /var/www/secmon/cve_updater.py >/dev/null 2>&1 &
- docker exec secmontest cp /var/www/secmon/docker/secmon-ci.conf /etc/apache2/sites-enabled/secmon.conf
- docker exec secmontest chown -R www-data:www-data /var/www/secmon/
- docker exec secmontest chmod -R 744 /var/www/secmon/
- docker exec secmontest sed -i "s/{FQDN}/$FQDN/" /etc/apache2/sites-enabled/secmon.conf
- docker exec secmontest sed -i "s/Listen 443/Listen 4443/" /etc/apache2/ports.conf
- docker exec secmontest a2enmod ssl && docker exec secmontest service apache2 restart
- docker exec secmontest apt install -y wget
- docker exec secmontest wget -q -S -O - https://127.0.0.1:4443 --no-check-certificate | grep "SECMON - Login"
- docker stop secmontest && docker rm secmontest || true
allow_failure: false
Release-Image:
stage: release_image
script:
- docker login $PRIVATE_REGISTRY_URL -u $PRIVATE_REGISTRY_USERNAME -p $PRIVATE_REGISTRY_PASSWD
- docker tag $SECMON_REGISTRY_IMAGE_WITHOUT_TAG:$CI_COMMIT_REF_SLUG $SECMON_REGISTRY_IMAGE_WITHOUT_TAG:latest
- docker push $SECMON_REGISTRY_IMAGE_WITHOUT_TAG:latest
- docker login -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWD
- docker tag $SECMON_REGISTRY_IMAGE_WITHOUT_TAG:latest guezone/secmon:latest
- docker push guezone/secmon:latest
allow_failure: false
================================================
FILE: DOCS.md
================================================
# Table of contents
- [Table of contents](#table-of-contents)
- [Installation](#installation)
- [With docker](#with-docker)
- [Autosetup](#autosetup)
- [Automation](#automation)
- [Without docker](#without-docker)
- [Setup](#setup)
- [Automation](#automation-1)
- [Startup](#startup)
- [Configuration of the web service](#configuration-of-the-web-service)
- [Login](#login)
- [Configuration](#configuration)
- [Search products](#search-products)
- [Add your products](#add-your-products)
- [User interface](#user-interface)
- [CVE Module](#cve-module)
- [RSS Module](#rss-module)
- [Settings](#settings)
- [Upgrade](#upgrade)
- [Logging](#logging)
- [Useful logs](#useful-logs)
- [Error logs](#error-logs)
# Installation
Before installation, you must have drawn up :
- The list of products present in your infrastructure by their common name without version (ex: ```Apache;Microsoft Windows;CentOS```)
- The list of CPE product identifiers present on your infrastructure (ex: ```cpe:2.3:o:microsoft:windows_server_2008:r2:sp1:*:*:*:x64:*;cpe:2.3:o:apple:mac_os:-:*:*:*:*:*```)
**Note 1** : You can also upload a CSV file with one line per product via the web interface but I advise you to put the list at the installation.
**Note 2** : Product names (keywords) should not be too succinct (e.g. prefer to use 'Microsoft Office' instead of 'Office'). This could avoid false positives during alerts.
You will be able to search and add new products after installation via the web UI.
## With docker (recommanded)
With the docker installation, everything is automated. The script creates a self-signed certificate if you wish and configures apache. Obviously, you can change the configuration according to your needs.
The installation with Docker allows via a volume (-v option) to easily update SECMON with Git.
To update it, you will have to :
- Go to the folder where SECMON was cloned at installation
- Do a git pull to get the changes
The files will then be modified directly in your SECMON container.
### Autosetup
```bash
sudo git clone https://github.com/Guezone/SECMON && cd SECMON && mkdir certs && PWD=$(pwd)
sudo docker build . -t secmon:latest
sudo docker network create --driver=bridge --subnet=10.10.10.0/24 --gateway=10.10.10.254 SECMON_NET
sudo docker run -i -t --hostname secmonsrv --ip 10.10.10.100 --name secmon-srv -v $PWD/:/var/www/secmon -v $PWD/certs:/etc/ssl/secmon --network SECMON_NET --expose 80 --expose 443 -d secmon:latest
sudo docker exec -it secmon-srv python3 /var/www/secmon/docker/install.py
```
```
| SECMON - DockerAutoInstall |
1. Certificate :
Note 1 : You can delete this self signed cert and put your CA signed cert after install.
Enter the FQDN of the web server :secmon.demo.net
Note 2 : Don't forget to enter a good value for CN (FQDN of your server)
Generating a RSA private key
........................................................................++++
...................................................................................................................++++
writing new private key to '/etc/ssl/secmon/secmon.key'
-----
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [AU]:FR
State or Province Name (full name) [Some-State]:IDF
Locality Name (eg, city) []:Paris
Organization Name (eg, company) [Internet Widgits Pty Ltd]:SecmonDemo
Organizational Unit Name (eg, section) []:X
Common Name (e.g. server FQDN or YOUR name) []:secmon.demo.net
Email Address []:XXXXXXXXXXX@XXXX.XX
Your certificate is available on /etc/ssl/secmon !
----------------------------------------------------------------------------------------------------------------------
2. SECMON Install :
Sender email address : sender@mail.com
Sender email account password :XXXXXXXXXXXXXXXXXXXX
SMTP Server FQDN or IP :XXXX.XXXX.XXXX
SMTP used port :587
Using TLS SMTP auth ? (yes/no) :yes
Language (en/fr) :fr
SECMON email receivers (single or many seperated by ;):receiver1@mail.com;receiver2@mail.com
------------------------------------
SECMON setup script - Version 1.0.0
------------------------------------
SECMON is licensed by CC BY-NC-SA 4.0 license. Do you accept the terms of the license? (y/Y;n/N) : y
Do you want to use a Github API for exploit retrieval (y/Y;n/N) : y
Please enter your Github username : XXXXXXXXXXX
Please enter your Github API Token : XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
Enter the username that can be used on the Web UI : demo_user
Enter the password (please choose a strong password !) : XXXXXXXX
Confirm the password : XXXXXXXX
RSS news database recording in progress...
Successful build database.
Please wait. A test message to receiver1@mail.com will be sent to test your configuration.
Please wait. A test message to receiver2@mail.com will be sent to test your configuration.
SECMON is now ready. Execute secmon.py now and automate it.
Executing secmon in background...
Executing cve_updater in background...
SECMON is successfully installed and configured !
----------------------------------------------------------------------------------------------------------------------
3. Apache configuration :
Considering dependency setenvif for ssl:
Module setenvif already enabled
Considering dependency mime for ssl:
Module mime already enabled
Considering dependency socache_shmcb for ssl:
Enabling module socache_shmcb.
Enabling module ssl.
See /usr/share/doc/apache2/README.Debian.gz on how to configure SSL and create self-signed certificates.
To activate the new configuration, you need to run:
service apache2 restart
[ ok ] Restarting Apache httpd web server: apache2.
Apache is successfully configured !
You can now access the web interface at the following address: https://secmon.demo.net
----------------------------------------------------------------------------------------------------------------------
```
### Automation
Edit the **host crontab** with **vi /etc/crontab** command, for example like this :
```bash
0 * * * * root docker exec secmon-srv python3 /var/www/secmon/secmon.py >> /var/log/secmon 2>&1
0 6 * * * root docker exec secmon-srv python3 /var/www/secmon/cve_updater.py >> /var/log/secmon-updates 2>&1
@reboot root docker start secmon-srv
@reboot root docker exec secmon-srv service apache2 start
@reboot root docker exec secmon-srv service ssh start
```
This configuration automates the retrieval of new CVE (and news) every 20 minutes and an update of the scores and associated products every day at 6am.
Now, you can login to SECMON with a giving URL.
## Without docker
### Setup
```bash
sudo apt update
sudo apt install -y python3 python3-pip apache2 libapache2-mod-wsgi-py3 git
sudo git clone https://github.com/Guezone/SECMON
cd SECMON
sudo pip3 install -r requirements.txt
sudo rm -Rf /var/www/html
sudo mkdir /var/www/secmon && sudo cp -r * /var/www/secmon/ && cd /var/www/secmon/
sudo python3 setup.py -sender sender@mail.com -p 'XXXXXXXXXXXXXXXX' -login [your SMTP login, often email address] -server srv.mail.com -port 587 -tls yes -lang fr -r 'receiver1@mail.com;receiver2@mail.com'
```
**Note :** Since a few time, the product list entry is only on web UI after install.
**Output :**
```bash
------------------------------------
SECMON setup script - Version 1.0.0
------------------------------------
SECMON is licensed by CC BY-NC-SA 4.0 license. Do you accept the terms of the license? (y/Y;n/N) : y
Let's go to add your products list for which you want to check the new CVEs.
Of course, you can add more with the web interface after installation !
Do you want to use a Github API for exploit retrieval (y/Y;n/N) : y
Please enter your Github username : XXXXXXXXXXX
Please enter your Github API Token : XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
Enter the username that can be used on the Web UI : demo_user
Enter the password (please choose a strong password !) : XXXXXXXX
Confirm the password : XXXXXXXX
RSS news database recording in progress...
Successful build database.
Please wait. A test message to receiver1@mail.com will be sent to test your configuration.
Please wait. A test message to receiver2@mail.com will be sent to test your configuration.
SECMON is now ready. Execute secmon.py now and automate it.
```
### Automation
Edit your crontab with **sudo vi /etc/crontab** command, for example like this :
```bash
0 * * * * root python3 /var/www/secmon/secmon.py >> /var/log/secmon 2>&1
0 6 * * * root python3 /var/www/secmon/cve_updater.py >> /var/log/secmon-updates 2>&1
```
This configuration automates the retrieval of new CVE (and news) every 60 minutes and an update of the scores and associated products every day at 6am.
### Startup
Launch the main script of the application :
```bash
sudo python3 secmon.py
Cloning exploit-database repository <-- At the first launch only, there is a download of the exploit-db.com exploit database.
Clonage dans 'exploit-database'...
remote: Enumerating objects: 194, done.
remote: Counting objects: 100% (194/194), done.
remote: Compressing objects: 100% (163/163), done.
remote: Total 135299 (delta 72), reused 139 (delta 31), pack-reused 135105
Réception d'objets: 100% (135299/135299), 151.14 MiB | 3.94 MiB/s, fait.
Résolution des deltas: 100% (85326/85326), fait.
Extraction des fichiers: 100% (44380/44380), fait.
------------ CVE Module ------------
Starting at : 30/11/2020 23:09:30
------------------------------------
Configuration is good.
------------------------------------
Polling NVD RSS feed (keyword based search).
No new CVE matched with your keyword list.
------------------------------------
Polling NVD related to your product list (CPE based search).
No new CVE related to your (CPE) product list
------------------------------------
Finished at : 30/11/2020 23:09:38
------------------------------------
------------ RSS Module ------------
Starting at : 30/11/2020 23:09:38
------------------------------------
Configuration is good.
------------------------------------
No news. Goodbye.
------------------------------------
Finished at : 30/11/2020 23:09:44
------------------------------------
```
At this step, SECMON is properly configured and you can receive alerts about new CVE that affect your products as well as new "cyber" news.
You now need to update the list of available exploits for each CVE (exploit-db and Github) as well as your high security risk products.
**Note :** "high security risk" products are those that have :
- Critical and high vulnerabilities (based on the CVSS 3.0 score)
- Vulnerabilities where there are known exploits (Github or Exploit-DB)
To make this update :
```bash
sudo python3 cve_updater.py
------------ CVE Module - Update ------------
Starting at : 30/11/2020 23:34:06
---------------------------------------------
Refreshing exploit-database repo with latest exploits
From https://github.com/offensive-security/exploit-database
* branch master -> FETCH_HEAD
Already up to date.
Refreshing EDBID-CVE mapping
100% (43828 of 43828) |###############################################################################################| Elapsed Time: 0:00:00 Time: 0:00:00
Exploit found for CVE-2021-3156 from Exploit-DB
Exploit found for CVE-2014-8380 from Exploit-DB
Search Github Exploit for registered CVE...
Exploit found for CVE-2020-1590 from Github
Updating high risk product list....
New critical product : Microsoft Windows Server 2008 R2
Updating affected products and CVSS score related to your CVE list...
No CVE have been updated.
---------------------------------------------
Finished at : 30/11/2020 23:52:24
---------------------------------------------
```
**Note :** This operation can take a few minutes to a few hours.
### Configuration of the web service
You must then delete the Apache sites configured by default :
```bash
sudo rm /etc/apache2/sites-enabled/*
sudo rm /etc/apache2/sites-available/*
sudo a2enmod wsgi
sudo a2enmod ssl
```
Then, you have to create a new configuration file dedicated to SECMON (in ```/etc/apache2/sites-available/secmon.conf```) :
Here is an example of configuration for a "secure" basic configuration :
```bash
ServerName secmon.corp.net
ServerAdmin admin@mail.com
ServerName secmon.corp.net
ServerAlias www.secmon.corp.net
DocumentRoot /var/www/secmon
Redirect permanent / https://secmon.corp.net
WSGIProcessGroup secmon_web
WSGIPassAuthorization On
WSGIApplicationGroup %{GLOBAL}
Order deny,allow
Allow from all
ServerSignature off
ErrorLog ${APACHE_LOG_DIR}/error_secmon.log
CustomLog ${APACHE_LOG_DIR}/access_secmon.log combined
WSGIScriptAlias / /var/www/secmon/secmon.wsgi
WSGIDaemonProcess secmon_web
DocumentRoot /var/www/secmon
ServerName secmon.corp.net
ServerAlias www.secmon.corp.net
WSGIProcessGroup secmon_web
WSGIPassAuthorization On
WSGIApplicationGroup %{GLOBAL}
Order deny,allow
Allow from all
ServerSignature Off
LogLevel info
ErrorLog ${APACHE_LOG_DIR}/error_secmon.log
CustomLog ${APACHE_LOG_DIR}/access_secmon.log combined
SSLProtocol -all +TLSv1.2 +TLSv1.3
SSLEngine on
SSLCertificateFile PATH/YOUR_CERT.crt
SSLCertificateKeyFile PATH/YOUR_PRIVATE_KEY.key
```
**Note :** You can use a self-signed certificate or a certificate signed by a known CA. This configuration must be changed according to your settings and is an example.
```bash
sudo a2ensite secmon
Enabling site secmon.
To activate the new configuration, you need to run:
systemctl reload apache2
sudo systemctl reload apache2
cd /var/www/secmon && sudo chown -R www-data:www-data .
sudo chmod -R 744 .
```
# Login
Now you will be able to connect to the Web UI with the credentials entered during installation :
SECMON is now fully ready to use ! It's time for a ☕.
# Configuration
## Search products
SECMON accepts the "products" to be supervised in keywords or in CPE (Common Platform Enumeration).
The keywords are used when the CVE do not have a CPE filled in and this is particularly the case when it has just been published.
In SECMON's WEB UI, go to the **Settings -> Product management** tab. You have at your disposal a search section in order to find the CPE references by their common name (example for Windows Server 2019) :
CPEs allow greater accuracy down to the product version level. CPEs can only be found when they have been provided by the publisher, which is not always the case. You can still use previous versions to determine this.
Example :
```
The CPE search for Apache 2.4.46 does not find the correct CPE code but the CPE code for version 2.4.44 is "cpe: 2.3: a: apache: http_server: 2.4.44: *: *: *: *: *: *: * "
All you have to do is replace the correct fields with the correct version: "cpe: 2.3: a: apache: http_server: 2.4.46: *: *: *: *: *: *: *"
```
## Add your products
To add new products to your list, go to the **Settings -> Product management** tab then to the ** Manage product ** section. You then have the choice to add / delete a CPE or a keyword :
You can also import a list of products (CPE and keywords) into a CSV in the same tab. It must consist of a single command. Example file :
```csv
cpe:2.3:a:keepass:password_safe:*:*:*:*:*:*:*:*
cpe:2.3:o:microsoft:windows_10:2004:*:*:*:*:*:*:*
cpe:2.3:a:docker:docker:0.2.1:*:*:*:*:*:*:*
cpe:2.3:a:anydesk:anydesk:*:*:*:*:*:*:*:*
cpe:2.3:o:vmware:esxi:7.0:*:*:*:*:*:*:*
Microsoft Exchange
Microsoft Office
HPE
F5
BigIP
Extreme
```
**Note**: when you add a CPE reference, it may take several minutes or even several hours for all the CVEs to be recorded in the database. To follow the progress, go to the tab **Settings -> Tasks**. A good practice might be to set the keywords to match your CPE references ("Docker" as a keyword and "cpe:2.3:a:docker:docker:0.2.1:*:*:*:*:*:*"as a CPE).
## User interface
**Home**: dashboard allowing you to see the state of health of your fleet regarding vulnerabilities. Products with high security risks appear as well as the rate of corrected / read / unread vulnerabilities.
### CVE Module
**Search**: this page allows you to search for details about a CVE
**Vulnerability management**: this page allows you to put a stamp in order to know the status of the correction of your vulnerability. For better management, I encourage you to indicate when you became aware of a vulnerability and also when it is fixed. Once the alert has been sent by email, I also encourage you to check that the CVE concerns your product with the exact version. There may be false positives in particular on the search for CVE by keyword, you can also note the false on this page.
**Last CVE alerts**: this page allows you to view vulnerabilities by product and also to see the list of high risk products, ie those with exploitable vulnerabilities.
**Exploits**: allows you to search for exploits linked to a CVE and also to see the list of exploitable CVEs. The source of the information is based on Github and Exploit-DB.
**Last CVE updates**: Allows you to see if recent changes have been made to CVSS scores or affected products regarding your vulnerabilities.
### RSS Module
**Cybersecurity news**: this page allows you to view the latest information and articles identified by the tool.
### Settings
**Logs**: allows you to view the logs produced by the tool.
**Product management**: this page allows you to search for CPE references, add / remove products (keywords or CPE) to your list and display the list.
**SMTP Configuration**: allows you to change the SMTP configuration linked to sending CVE alerts or news.
**Tasks**: allows you to view the status of recent tasks which are heavy for SECMON. Today, only the addition of products appears here as it is a process that can be very long.
You can update your web UI password and also add users. To do this, click on the menu at the top right and then go to **Account settings**.
**Note** : It is currently impossible to reset passwords via a token and email as is usually the case.
# Upgrade
The installation of SECMON with Docker allows the use of a volume in order to have a "synchronisation" between the host and the container.
If you want to have the latest version of SECMON with the latest changes, go to the folder where SECMON was cloned (on the host) and run the following commands:
```
git pull origin
sudo chown -R www-data:www-data .
sudo chmod -R 744 .
sudo docker exec -it secmon-srv service apache2 restart
```
The files will be updated on the host side and in the container and SECMON will be updated.
If you are not using docker, you can do the same in the folder where the repository was cloned. However, you will need to copy the modified files to **/var/www/secmon** and also assign the correct rights. You will also have to restart the Apache service.
# Logging
## Useful logs
All useful SECMON logs are written to your **/var/www/secmon/logs.txt**. The format is not really standard but they are parsed automatically by Splunk for example.
Obviously, I encourage you to create alerts if you have a SIEM to manage critical vulnerabilities and high-risk products in your regular processes.
Here is an example of field extractions (Graylog compatible file):
**Note**: you can obviously reuse the **Grok patterns** contained in the file to extract fields on your log collection and analysis tools.
```json
{
"extractors": [
{
"title": "SECMON-Authentication-Extract",
"extractor_type": "grok",
"converters": [],
"order": 0,
"cursor_strategy": "cut",
"source_field": "message",
"target_field": "",
"extractor_config": {
"grok_pattern": "%{TIMESTAMP_ISO8601:timestamp} source_script=%{GREEDYDATA:script} server=\"%{IP:dst_ip}\" src_ip=\"%{IPV4:src_ip}\" username=\"%{USERNAME:username}\" auth_status=\"%{WORD:status}\" message=\"%{GREEDYDATA:msg}\""
},
"condition_type": "regex",
"condition_value": "auth_status"
},
{
"title": "SECMON-CVE-Extract",
"extractor_type": "grok",
"converters": [],
"order": 0,
"cursor_strategy": "cut",
"source_field": "message",
"target_field": "",
"extractor_config": {
"grok_pattern": "%{TIMESTAMP_ISO8601:timestamp} source_script=\"%{GREEDYDATA:script}\" server=\"%{IP:dst_ip}\" cve_id=\"%{GREEDYDATA:cve_id}\" type=\"%{WORD:type}\" matched_product=\"%{GREEDYDATA:matched_product}\" cvss_score=\"%{GREEDYDATA:cvss_score}\" severity=\"%{GREEDYDATA:severity}\" cve_date=\"%{GREEDYDATA:cve_date}\" cpes=\"%{GREEDYDATA:cpes}\" report=\"%{WORD:report}\" alert=\"%{WORD:alert}\" message=\"%{GREEDYDATA:msg}\""
},
"condition_type": "string",
"condition_value": "cve_id"
},
{
"title": "SECMON-Exploits-Extract",
"extractor_type": "grok",
"converters": [],
"order": 0,
"cursor_strategy": "cut",
"source_field": "message",
"target_field": "",
"extractor_config": {
"grok_pattern": "%{TIMESTAMP_ISO8601:timestamp} source_script=%{GREEDYDATA:script} server=\"%{IP:dst_ip}\" cve=\"%{GREEDYDATA:cve_id}\" exploit_source=\"%{GREEDYDATA:exploit_source}\" message=\"%{GREEDYDATA:msg}\""
},
"condition_type": "string",
"condition_value": "exploit"
},
{
"title": "SECMON-Tasks-Extract",
"extractor_type": "grok",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "",
"extractor_config": {
"grok_pattern": "%{TIMESTAMP_ISO8601:timestamp} source_script=%{GREEDYDATA:script} server=\"%{IP:dst_ip}\" task_id=\"%{INT:task_id}\" task_status=\"%{WORD:task_status}\" message=\"%{GREEDYDATA:msg}\""
},
"condition_type": "string",
"condition_value": "task_id"
},
{
"title": "SECMON-News-Extract",
"extractor_type": "grok",
"converters": [],
"order": 0,
"cursor_strategy": "cut",
"source_field": "message",
"target_field": "",
"extractor_config": {
"grok_pattern": "%{TIMESTAMP_ISO8601:timestamp} source_script=%{GREEDYDATA:script} server=\"%{IP:dst_ip}\" feed=\"%{GREEDYDATA:feed}\" feed_url=\"%{URI:feed_url}\" news_title=\"%{GREEDYDATA:news_title}\" news_url=\"%{URI:news_url}\" mail=\"%{WORD:mail}\""
},
"condition_type": "string",
"condition_value": "rss_poller"
},
{
"title": "SECMON-HighRiskProducts-Extract",
"extractor_type": "grok",
"converters": [],
"order": 0,
"cursor_strategy": "cut",
"source_field": "message",
"target_field": "",
"extractor_config": {
"grok_pattern": "%{TIMESTAMP_ISO8601:timestamp} source_script=%{GREEDYDATA:script} server=\"%{IP:dst_ip}\" cpe=\"%{GREEDYDATA:cpe}\" hname=\"%{GREEDYDATA:hname}\" message=\"%{GREEDYDATA:msg}\""
},
"condition_type": "string",
"condition_value": "A new critical product"
}
],
"version": "4.0.5"
}
```
## Error logs
The error logs generated by the pollers will be located where you redirect the standard output in your cron configuration.
Error logs related to your web interface will be available in the Apache error file (or other depending on your usage). The errors are normally formatted.
Feel free to file an issue on Github if you see an error.
================================================
FILE: Dockerfile
================================================
FROM debian:latest
MAINTAINER Aubin Custodio (Guezone)
RUN apt update && apt upgrade -y && apt install -y vim apache2 nano python3 git python3-pip libapache2-mod-wsgi-py3 net-tools dnsutils openssh-server openssl python3-dev libffi-dev
RUN service apache2 stop
RUN rm /etc/apache2/sites-enabled/* && rm /etc/apache2/sites-available/*
RUN mkdir /etc/ssl/secmon/ && mkdir /var/www/secmon/
COPY ./requirements.txt /tmp
RUN cd /tmp && pip3 install -r requirements.txt && rm -Rf /var/www/html
EXPOSE 80
EXPOSE 443
EXPOSE 22
================================================
FILE: LICENSE
================================================
Attribution-NonCommercial-ShareAlike 4.0 International
=======================================================================
Creative Commons Corporation ("Creative Commons") is not a law firm and
does not provide legal services or legal advice. Distribution of
Creative Commons public licenses does not create a lawyer-client or
other relationship. Creative Commons makes its licenses and related
information available on an "as-is" basis. Creative Commons gives no
warranties regarding its licenses, any material licensed under their
terms and conditions, or any related information. Creative Commons
disclaims all liability for damages resulting from their use to the
fullest extent possible.
Using Creative Commons Public Licenses
Creative Commons public licenses provide a standard set of terms and
conditions that creators and other rights holders may use to share
original works of authorship and other material subject to copyright
and certain other rights specified in the public license below. The
following considerations are for informational purposes only, are not
exhaustive, and do not form part of our licenses.
Considerations for licensors: Our public licenses are
intended for use by those authorized to give the public
permission to use material in ways otherwise restricted by
copyright and certain other rights. Our licenses are
irrevocable. Licensors should read and understand the terms
and conditions of the license they choose before applying it.
Licensors should also secure all rights necessary before
applying our licenses so that the public can reuse the
material as expected. Licensors should clearly mark any
material not subject to the license. This includes other CC-
licensed material, or material used under an exception or
limitation to copyright. More considerations for licensors:
wiki.creativecommons.org/Considerations_for_licensors
Considerations for the public: By using one of our public
licenses, a licensor grants the public permission to use the
licensed material under specified terms and conditions. If
the licensor's permission is not necessary for any reason--for
example, because of any applicable exception or limitation to
copyright--then that use is not regulated by the license. Our
licenses grant only permissions under copyright and certain
other rights that a licensor has authority to grant. Use of
the licensed material may still be restricted for other
reasons, including because others have copyright or other
rights in the material. A licensor may make special requests,
such as asking that all changes be marked or described.
Although not required by our licenses, you are encouraged to
respect those requests where reasonable. More considerations
for the public:
wiki.creativecommons.org/Considerations_for_licensees
=======================================================================
Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International
Public License
By exercising the Licensed Rights (defined below), You accept and agree
to be bound by the terms and conditions of this Creative Commons
Attribution-NonCommercial-ShareAlike 4.0 International Public License
("Public License"). To the extent this Public License may be
interpreted as a contract, You are granted the Licensed Rights in
consideration of Your acceptance of these terms and conditions, and the
Licensor grants You such rights in consideration of benefits the
Licensor receives from making the Licensed Material available under
these terms and conditions.
Section 1 -- Definitions.
a. Adapted Material means material subject to Copyright and Similar
Rights that is derived from or based upon the Licensed Material
and in which the Licensed Material is translated, altered,
arranged, transformed, or otherwise modified in a manner requiring
permission under the Copyright and Similar Rights held by the
Licensor. For purposes of this Public License, where the Licensed
Material is a musical work, performance, or sound recording,
Adapted Material is always produced where the Licensed Material is
synched in timed relation with a moving image.
b. Adapter's License means the license You apply to Your Copyright
and Similar Rights in Your contributions to Adapted Material in
accordance with the terms and conditions of this Public License.
c. BY-NC-SA Compatible License means a license listed at
creativecommons.org/compatiblelicenses, approved by Creative
Commons as essentially the equivalent of this Public License.
d. Copyright and Similar Rights means copyright and/or similar rights
closely related to copyright including, without limitation,
performance, broadcast, sound recording, and Sui Generis Database
Rights, without regard to how the rights are labeled or
categorized. For purposes of this Public License, the rights
specified in Section 2(b)(1)-(2) are not Copyright and Similar
Rights.
e. Effective Technological Measures means those measures that, in the
absence of proper authority, may not be circumvented under laws
fulfilling obligations under Article 11 of the WIPO Copyright
Treaty adopted on December 20, 1996, and/or similar international
agreements.
f. Exceptions and Limitations means fair use, fair dealing, and/or
any other exception or limitation to Copyright and Similar Rights
that applies to Your use of the Licensed Material.
g. License Elements means the license attributes listed in the name
of a Creative Commons Public License. The License Elements of this
Public License are Attribution, NonCommercial, and ShareAlike.
h. Licensed Material means the artistic or literary work, database,
or other material to which the Licensor applied this Public
License.
i. Licensed Rights means the rights granted to You subject to the
terms and conditions of this Public License, which are limited to
all Copyright and Similar Rights that apply to Your use of the
Licensed Material and that the Licensor has authority to license.
j. Licensor means the individual(s) or entity(ies) granting rights
under this Public License.
k. NonCommercial means not primarily intended for or directed towards
commercial advantage or monetary compensation. For purposes of
this Public License, the exchange of the Licensed Material for
other material subject to Copyright and Similar Rights by digital
file-sharing or similar means is NonCommercial provided there is
no payment of monetary compensation in connection with the
exchange.
l. Share means to provide material to the public by any means or
process that requires permission under the Licensed Rights, such
as reproduction, public display, public performance, distribution,
dissemination, communication, or importation, and to make material
available to the public including in ways that members of the
public may access the material from a place and at a time
individually chosen by them.
m. Sui Generis Database Rights means rights other than copyright
resulting from Directive 96/9/EC of the European Parliament and of
the Council of 11 March 1996 on the legal protection of databases,
as amended and/or succeeded, as well as other essentially
equivalent rights anywhere in the world.
n. You means the individual or entity exercising the Licensed Rights
under this Public License. Your has a corresponding meaning.
Section 2 -- Scope.
a. License grant.
1. Subject to the terms and conditions of this Public License,
the Licensor hereby grants You a worldwide, royalty-free,
non-sublicensable, non-exclusive, irrevocable license to
exercise the Licensed Rights in the Licensed Material to:
a. reproduce and Share the Licensed Material, in whole or
in part, for NonCommercial purposes only; and
b. produce, reproduce, and Share Adapted Material for
NonCommercial purposes only.
2. Exceptions and Limitations. For the avoidance of doubt, where
Exceptions and Limitations apply to Your use, this Public
License does not apply, and You do not need to comply with
its terms and conditions.
3. Term. The term of this Public License is specified in Section
6(a).
4. Media and formats; technical modifications allowed. The
Licensor authorizes You to exercise the Licensed Rights in
all media and formats whether now known or hereafter created,
and to make technical modifications necessary to do so. The
Licensor waives and/or agrees not to assert any right or
authority to forbid You from making technical modifications
necessary to exercise the Licensed Rights, including
technical modifications necessary to circumvent Effective
Technological Measures. For purposes of this Public License,
simply making modifications authorized by this Section 2(a)
(4) never produces Adapted Material.
5. Downstream recipients.
a. Offer from the Licensor -- Licensed Material. Every
recipient of the Licensed Material automatically
receives an offer from the Licensor to exercise the
Licensed Rights under the terms and conditions of this
Public License.
b. Additional offer from the Licensor -- Adapted Material.
Every recipient of Adapted Material from You
automatically receives an offer from the Licensor to
exercise the Licensed Rights in the Adapted Material
under the conditions of the Adapter's License You apply.
c. No downstream restrictions. You may not offer or impose
any additional or different terms or conditions on, or
apply any Effective Technological Measures to, the
Licensed Material if doing so restricts exercise of the
Licensed Rights by any recipient of the Licensed
Material.
6. No endorsement. Nothing in this Public License constitutes or
may be construed as permission to assert or imply that You
are, or that Your use of the Licensed Material is, connected
with, or sponsored, endorsed, or granted official status by,
the Licensor or others designated to receive attribution as
provided in Section 3(a)(1)(A)(i).
b. Other rights.
1. Moral rights, such as the right of integrity, are not
licensed under this Public License, nor are publicity,
privacy, and/or other similar personality rights; however, to
the extent possible, the Licensor waives and/or agrees not to
assert any such rights held by the Licensor to the limited
extent necessary to allow You to exercise the Licensed
Rights, but not otherwise.
2. Patent and trademark rights are not licensed under this
Public License.
3. To the extent possible, the Licensor waives any right to
collect royalties from You for the exercise of the Licensed
Rights, whether directly or through a collecting society
under any voluntary or waivable statutory or compulsory
licensing scheme. In all other cases the Licensor expressly
reserves any right to collect such royalties, including when
the Licensed Material is used other than for NonCommercial
purposes.
Section 3 -- License Conditions.
Your exercise of the Licensed Rights is expressly made subject to the
following conditions.
a. Attribution.
1. If You Share the Licensed Material (including in modified
form), You must:
a. retain the following if it is supplied by the Licensor
with the Licensed Material:
i. identification of the creator(s) of the Licensed
Material and any others designated to receive
attribution, in any reasonable manner requested by
the Licensor (including by pseudonym if
designated);
ii. a copyright notice;
iii. a notice that refers to this Public License;
iv. a notice that refers to the disclaimer of
warranties;
v. a URI or hyperlink to the Licensed Material to the
extent reasonably practicable;
b. indicate if You modified the Licensed Material and
retain an indication of any previous modifications; and
c. indicate the Licensed Material is licensed under this
Public License, and include the text of, or the URI or
hyperlink to, this Public License.
2. You may satisfy the conditions in Section 3(a)(1) in any
reasonable manner based on the medium, means, and context in
which You Share the Licensed Material. For example, it may be
reasonable to satisfy the conditions by providing a URI or
hyperlink to a resource that includes the required
information.
3. If requested by the Licensor, You must remove any of the
information required by Section 3(a)(1)(A) to the extent
reasonably practicable.
b. ShareAlike.
In addition to the conditions in Section 3(a), if You Share
Adapted Material You produce, the following conditions also apply.
1. The Adapter's License You apply must be a Creative Commons
license with the same License Elements, this version or
later, or a BY-NC-SA Compatible License.
2. You must include the text of, or the URI or hyperlink to, the
Adapter's License You apply. You may satisfy this condition
in any reasonable manner based on the medium, means, and
context in which You Share Adapted Material.
3. You may not offer or impose any additional or different terms
or conditions on, or apply any Effective Technological
Measures to, Adapted Material that restrict exercise of the
rights granted under the Adapter's License You apply.
Section 4 -- Sui Generis Database Rights.
Where the Licensed Rights include Sui Generis Database Rights that
apply to Your use of the Licensed Material:
a. for the avoidance of doubt, Section 2(a)(1) grants You the right
to extract, reuse, reproduce, and Share all or a substantial
portion of the contents of the database for NonCommercial purposes
only;
b. if You include all or a substantial portion of the database
contents in a database in which You have Sui Generis Database
Rights, then the database in which You have Sui Generis Database
Rights (but not its individual contents) is Adapted Material,
including for purposes of Section 3(b); and
c. You must comply with the conditions in Section 3(a) if You Share
all or a substantial portion of the contents of the database.
For the avoidance of doubt, this Section 4 supplements and does not
replace Your obligations under this Public License where the Licensed
Rights include other Copyright and Similar Rights.
Section 5 -- Disclaimer of Warranties and Limitation of Liability.
a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE
EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS
AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF
ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS,
IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION,
WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS,
ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT
KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT
ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU.
b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE
TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION,
NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT,
INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES,
COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR
USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN
ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR
DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR
IN PART, THIS LIMITATION MAY NOT APPLY TO YOU.
c. The disclaimer of warranties and limitation of liability provided
above shall be interpreted in a manner that, to the extent
possible, most closely approximates an absolute disclaimer and
waiver of all liability.
Section 6 -- Term and Termination.
a. This Public License applies for the term of the Copyright and
Similar Rights licensed here. However, if You fail to comply with
this Public License, then Your rights under this Public License
terminate automatically.
b. Where Your right to use the Licensed Material has terminated under
Section 6(a), it reinstates:
1. automatically as of the date the violation is cured, provided
it is cured within 30 days of Your discovery of the
violation; or
2. upon express reinstatement by the Licensor.
For the avoidance of doubt, this Section 6(b) does not affect any
right the Licensor may have to seek remedies for Your violations
of this Public License.
c. For the avoidance of doubt, the Licensor may also offer the
Licensed Material under separate terms or conditions or stop
distributing the Licensed Material at any time; however, doing so
will not terminate this Public License.
d. Sections 1, 5, 6, 7, and 8 survive termination of this Public
License.
Section 7 -- Other Terms and Conditions.
a. The Licensor shall not be bound by any additional or different
terms or conditions communicated by You unless expressly agreed.
b. Any arrangements, understandings, or agreements regarding the
Licensed Material not stated herein are separate from and
independent of the terms and conditions of this Public License.
Section 8 -- Interpretation.
a. For the avoidance of doubt, this Public License does not, and
shall not be interpreted to, reduce, limit, restrict, or impose
conditions on any use of the Licensed Material that could lawfully
be made without permission under this Public License.
b. To the extent possible, if any provision of this Public License is
deemed unenforceable, it shall be automatically reformed to the
minimum extent necessary to make it enforceable. If the provision
cannot be reformed, it shall be severed from this Public License
without affecting the enforceability of the remaining terms and
conditions.
c. No term or condition of this Public License will be waived and no
failure to comply consented to unless expressly agreed to by the
Licensor.
d. Nothing in this Public License constitutes or may be interpreted
as a limitation upon, or waiver of, any privileges and immunities
that apply to the Licensor or You, including from the legal
processes of any jurisdiction or authority.
=======================================================================
Creative Commons is not a party to its public
licenses. Notwithstanding, Creative Commons may elect to apply one of
its public licenses to material it publishes and in those instances
will be considered the “Licensor.” The text of the Creative Commons
public licenses is dedicated to the public domain under the CC0 Public
Domain Dedication. Except for the limited purpose of indicating that
material is shared under a Creative Commons public license or as
otherwise permitted by the Creative Commons policies published at
creativecommons.org/policies, Creative Commons does not authorize the
use of the trademark "Creative Commons" or any other trademark or logo
of Creative Commons without its prior written consent including,
without limitation, in connection with any unauthorized modifications
to any of its public licenses or any other arrangements,
understandings, or agreements concerning use of licensed material. For
the avoidance of doubt, this paragraph does not form part of the
public licenses.
Creative Commons may be contacted at creativecommons.org.
================================================
FILE: README.md
================================================
# SECMON - Infosec Watching Tool
[](https://www.python.org/)    -red) 
# Description
SECMON is a **web-based tool** for the automation of **infosec watching** and **vulnerability management** with a web interface.
A demo is available [here](https://secmon.guezone.info/). A Discord channel is available [here](https://discord.gg/cXzvb4fKzE)
## Features
- **Mail alerting** when a **new CVE** is published and which concerns your product list
- **Mail alerting** when a "cyber-security" news are published: new threats, recent attacks, events, etc.
- Visualize the **high security risk products** present on your IT infrastructure
- Download CVE **Excel report** by date range
- Display **top cybersecurity subject** (**Light cyber landscape**)
- Logs **easy to integrate in a SIEM** (verified on **Splunk** and **Graylog**)
- View the **latest CVE** and **latest news related to cyber security are published**
- Assign a buffer of **management status of a CVE**
- Search all the **details of a CVE**
- Check if there is an **exploit** on **Github or Exploit-DB** concerning a CVE
- Search for **vulnerabilities for a specified product**
- **Manage your product list**: search/add/delete a product, display your referenced product list
- **Monitor the sources** used by pollers
Email alerts can be sent in English or French. SECMON web UI now support multi user account.
CVE are polled using two methods of collection/correspondence:
- **Keyword-based** : allows you to be proactive on the retrieval of CVE by leaving a little bit of precision (no version check, just word matching) on the affected products (ex: "VMWare", "Apache").
- **CPE based** (Common Platform Enumeration) : allows the retrieval of CVE that only concern the product version entered (e.g. "Windows 10 1909", "Apache 2.4.38")
## Requirements
SECMON requires registration on Github API for exploits retrieval. It also requires :
- OS : Linux-based system (tested on Debian 10)
- Environnement : Python 3 (tested on Python 3.9 and Python 3.8)
- A valid Github API key
**WARNING** : Web UI credentials are hashed (SHA512 with salt), on the other hand, the Github API connection credentials and the application session key are neither encrypted nor hashed. All data is stored in an unencrypted sqlite database. A few advices :
- Allow access to this machine only to persons who are authorized to do so.
- Isolate the host machine from the rest.
- Use a dedicated server/VM or Docker container.
## Some screenshots
# Installation
[Read the docs](DOCS.md)
# Thanks
Thanks to @andreafioraldi for cve_searchsploit Python module.
Thanks to @konsav for HTML/CSS email template.
Thanks to @rodolfo-mendes for the sbadmin2 (Bootstrap) template adapted to Flask.
Thanks to Florent Chatain for the first web security auditing.
# License
**SECMON** (by Aubin Custodio - Guezone) is licensed under **CC BY-NC-SA 4.0**.
This license allows you to **use** SECMON, to **improve it** and to make it live **by mentioning the author** but **without using it for commercial purposes**. As the infosec watching process is quite difficult and time consuming, it should only be used to help companies and/or users **to secure their IT infrastructure** and to know the current cyber security world.
# Changelog
**2.0 :**
- Modification of the log format
- Reporting method (generation via dates)
- Web UI - new boostrap template
- Work on the Docker automation part
**2.1 :**
- Add a multi-user support on Web UI
- Improved auto docker configuration (to improve updates with git and volume)
- Added a **Cyber Threats** section that highlights the top cyber topics reported in the RSS module
- Update of README and DOCS (docker, update & screenshots part)
- Prioritisation of the CPE polling method over the keyword method
# Roadmap
## Features
- [x] Automate the deployment with docker
- [x] First security auditing (front-end only)
- [x] Write the logs in a standard format and plan to send them to a third party of SIEM type.
- [X] Write user documentation for the Web UI
- [ ] Create script to allow CPE scanning on Windows and Linux based system
- [ ] Add new sources of cyber-news
- [ ] Create a REST API for calling in other applications
- [ ] Send CVE daily update report (new high risk product, CVSS changes, affected product changes, new exploitable CVE)
- [ ] SECMON-Open-API (free and open API to get 0-day, viral vulns & data breach news)
- [ ] Others Vuln Database polling
## Reliability improvement
- [X] NIST API rate complete management (partial correction)
- [ ] Product unification (product = CPE + Keyword)
- [X] Keyword management to be case unsensitive
================================================
FILE: SECURITY.md
================================================
## Reporting a Vulnerability
Please report (suspected) security vulnerabilities via the dedicated "bugs-vulnérabilités" Discord channel ([Discord server link](https://discord.gg/cXzvb4fKzE)). You will receive a response from
us within few days. If the issue is confirmed, we will release a patch as soon
as possible depending on complexity but historically within a few days.
================================================
FILE: cve_poller.py
================================================
# -*- coding: utf-8 -*-
"""
SECMON - Source code of the SECMON CVE poller script.
"""
__author__ = "Aubin Custodio"
__copyright__ = "Copyright 2021, SECMON"
__credits__ = ["Aubin Custodio","Guezone"]
__license__ = "CC BY-NC-SA 4.0"
__version__ = "2.1"
__maintainer__ = "Aubin Custodio"
__email__ = "custodio.aubin@outlook.com"
import base64, os, smtplib, re, requests, time, sqlite3, feedparser, colorama, warnings
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email import encoders
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from secmon_lib import getParsedCpe, translateText, getUserLanguage, getCpeList, pollCveIdFromCpe, writeCveTypeLog, writeMgmtTasksLog,handleException
warnings.filterwarnings("ignore", category=DeprecationWarning)
sender, receiver, smtp_login, smtp_password, smtpsrv, port, tls, keywords = '','','','','','','',''
script_path = os.path.abspath(__file__)
dir_path = script_path.replace("cve_poller.py","")
log_file = dir_path+"logs.txt"
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
def checkConfig(sender, receiver, smtp_login, smtp_password, smtpsrv, port, tls,keywords):
script_path = os.path.abspath(__file__)
dir_path = script_path.replace("cve_poller.py","")
con = sqlite3.connect(dir_path+'secmon.db')
cur = con.cursor()
cur.execute("SELECT sender FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
sender = value
cur.execute("SELECT smtp_login FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
smtp_login = value
cur.execute("SELECT smtp_password FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
b = value.encode("UTF-8")
bytes_password = base64.b64decode(b)
smtp_password = bytes_password.decode("UTF-8")
cur.execute("SELECT smtpsrv FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
smtpsrv = value
cur.execute("SELECT port FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
port = value
cur.execute("SELECT receiver FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
receivers = value.split(";")
cur.execute("SELECT tls FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
tls = value
cur.execute("SELECT language FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
language = value
cur.execute("SELECT receiver FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
receivers = value.split(";")
cur.execute("SELECT keyword FROM keyword_list")
db_result_list = cur.fetchall()
keywords = []
for db_result_tuple in db_result_list:
for result in db_result_tuple:
keywords.append(result)
if all(value != '' for value in [sender, receivers, smtp_login, smtp_password, smtpsrv, str(port), tls, language]):
print(bcolors.OKGREEN+"Configuration is good."+bcolors.ENDC)
cvePoller(sender, receivers, smtp_login, smtp_password, smtpsrv, port, tls, keywords, language)
else:
print(bcolors.FAIL+"Error in the config."+bcolors.ENDC)
exit()
def cvePoller(sender, receivers, smtp_login, smtp_password, smtpsrv, port, tls, keywords, language):
print("------------------------------------")
summary, url = "",""
cve_rss = []
current_feed = feedparser.parse("https://nvd.nist.gov/feeds/xml/cve/misc/nvd-rss.xml")
for entries in current_feed.entries:
title = entries.title
summary = entries.summary
cve_id = re.findall('CVE-\d{4}-\d{4,7}',title)
cve_id = cve_id[0]
cve_rss.append(cve_id+"(=)"+summary)
summary, url = "",""
con = sqlite3.connect(dir_path+'secmon.db')
cur = con.cursor()
dest_language = getUserLanguage()
now_date = str(datetime.now()).split(" ")[0].split("-")
idx_date = now_date[2]+"/"+now_date[1]+"/"+now_date[0]
time.sleep(10)
# CPE
new_cve_list = []
cur.execute("SELECT cpe FROM cpe_list")
db_result_list = cur.fetchall()
cpes = getCpeList()
if cpes != []:
print(bcolors.HEADER+"Polling NVD related to your product list (CPE based search)."+bcolors.ENDC+"\n")
for cpe in cpes:
time.sleep(30)
cve_ids = pollCveIdFromCpe(cpe)
cpe = cpe.replace("\n","").replace(" ","")
for cve_id in cve_ids:
cur.execute("SELECT CVE_ID FROM CVE_DATA WHERE CVE_ID = (?);", (cve_id,))
db_result_list = cur.fetchall()
db_result_str = ""
for db_result_tuple in db_result_list:
for result in db_result_tuple:
db_result_str+=result
if cve_id not in db_result_str:
time.sleep(10)
nvd_base_url = "https://services.nvd.nist.gov/rest/json/cve/1.0/"
nvd_query = nvd_base_url+cve_id
nvd_response = requests.get(url=nvd_query)
nvd_data = nvd_response.json()
if 'result' in nvd_data.keys():
if 'reference_data' in nvd_data['result']['CVE_Items'][0]['cve']['references']:
nvd_links = nvd_data['result']['CVE_Items'][0]['cve']['references']['reference_data']
cve_sources = ""
for link in nvd_links:
cve_sources += (link['url']+"\n")
else:
cve_sources = "N/A"
key_match = cpe
cve_score = ""
fr_society = " (constructeur/éditeur)"
en_society = " (builder/publisher)"
if not "impact" in nvd_data['result']['CVE_Items'][0].keys():
cve_score = "N/A"
try:
webpage = requests.get("https://nvd.nist.gov/vuln/detail/{}".format(cve_id))
soup = BeautifulSoup(webpage.content, 'html.parser')
cve_cna_score = soup.find(id='Cvss3CnaCalculatorAnchor')
cve_cna_score = cve_cna_score.get_text()
if cve_cna_score != "":
if language == "fr" or language=="FR":
cve_score = cve_cna_score + fr_society
else:
cve_score = cve_cna_score + en_society
else:
cve_score = "N/A"
except:
cve_score = "N/A"
else:
try:
cve_score = nvd_data['result']['CVE_Items'][0]['impact']['baseMetricV3']['cvssV3']['baseScore']
except:
cve_score = "N/A"
published_date = nvd_data['result']['CVE_Items'][0]['publishedDate']
formatted_date = published_date.split("T")[0]
formatted_date = datetime.strptime(formatted_date,"%Y-%m-%d")
date_2_days_ago = datetime.now() - timedelta(days=7)
if (date_2_days_ago<=formatted_date) and (formatted_date<=datetime.now()):
cve_date_status = "Potentially new CVE"
else:
cve_date_status = "Potentially an update"
cve_date = published_date.split("T")[0]
if language == "fr" or language == "FR":
fr_date = cve_date.split("-")
cve_date = fr_date[2]+"/"+fr_date[1]+"/"+fr_date[0]
if (cve_score=="N/A"):
cve_status = cve_date_status+" - "+"Not yet rated (No score, no CPE)"
else:
cve_status = cve_date_status+" - "+"Valid evaluation"
cve_cpe = ""
if "configurations" in nvd_data['result']['CVE_Items'][0].keys():
if nvd_data['result']['CVE_Items'][0]['configurations']['nodes']:
for node in nvd_data['result']['CVE_Items'][0]['configurations']['nodes']:
if 'cpe_match' in node.keys():
for cpe_node in node['cpe_match']:
cve_cpe += (cpe_node['cpe23Uri']+'\n')
if cve_cpe == "":
cve_cpe = "N/A"
cve_description = nvd_data['result']['CVE_Items'][0]['cve']['description']['description_data'][0]['value']
if language == "fr" or language=="FR":
try:
cve_description = translateText(dest_language,cve_description)
except:
pass
if not "/" in cve_date:
cve_date = cve_date.split("-")[2]+"/"+cve_date.split("-")[1]+"/"+cve_date.split("-")[0]
try:
cve_status = translateText(dest_language,cve_status).replace("nouveau", "une nouvelle").replace("évalué","évaluée")
except:
pass
nvd_link = 'https://nvd.nist.gov/vuln/detail/'+cve_id
else:
cve_description = "N/A"
date = str(datetime.now()).split(" ")[0].split("-")
cve_date = date[2]+"/"+date[1]+"/"+date[0]
cve_score = "N/A"
cve_sources = "N/A"
cve_status = "N/A"
cve_cpe = "N/A"
nvd_link = 'https://nvd.nist.gov/vuln/detail/'+cve_id
status = "Unread"
if cve_id not in new_cve_list:
cur.execute("INSERT INTO CVE_DATA (CVE_ID,KEYWORD,STATUS,CVE_SCORE,CVE_DATE,CVE_DESCRIPTION,CVE_EVAL,CVE_CPE,CVE_SOURCES,EXPLOIT_FIND,INDEXING_DATE) VALUES (?,?,?,?,?,?,?,?,?,?,?);", (cve_id,key_match,status,str(cve_score).split(" ")[0],cve_date,cve_description,cve_status,cve_cpe,cve_sources,"False",idx_date))
con.commit()
report="updated"
print("New CVE detected -> "+cve_id)
try:
sendAlert(smtp_login, smtp_password, smtpsrv, port, tls, sender, receivers, cve_id, cve_sources, str(cve_score).split(" ")[0], cve_status, cve_cpe, cve_date, cve_description, language,key_match)
alert = "sent"
except Exception as e:
handleException(e)
alert = "unsent"
new_cve_list.append(cve_id)
writeCveTypeLog("cve_poller",cve_id,"new",key_match,str(cve_score).split(" ")[0],cve_date,cve_cpe,report,alert)
if not new_cve_list:
print(bcolors.HEADER+"No new CVE related to your (CPE) product list."+bcolors.ENDC)
print("------------------------------------")
else:
timestamp = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
print(bcolors.OKGREEN+"Database was updated with new CVE."+bcolors.ENDC)
print("------------------------------------")
# RSS
new_cve_list = []
print(bcolors.HEADER+"Polling NVD RSS feed (keyword based search)."+bcolors.ENDC+"\n")
for cve in cve_rss:
cve_id = cve.split("(=)")[0]
summary = cve.split("(=)")[1]
for key in keywords:
if bool(re.fullmatch(r'[A-Z0-9-._]{4,}',key)) == True or bool(re.fullmatch(r'\w{1,3}',key)) == True:
if key in summary:
key_match = key
cur.execute("SELECT CVE_ID FROM CVE_DATA WHERE CVE_ID = (?);", (cve_id,))
db_result_list = cur.fetchall()
db_result_str = ""
for db_result_tuple in db_result_list:
for result in db_result_tuple:
db_result_str+=result
if cve_id not in db_result_str:
time.sleep(15)
nvd_base_url = "https://services.nvd.nist.gov/rest/json/cve/1.0/"
nvd_query = nvd_base_url+cve_id
nvd_response = requests.get(url=nvd_query)
nvd_data = nvd_response.json()
if 'result' in nvd_data.keys():
if 'reference_data' in nvd_data['result']['CVE_Items'][0]['cve']['references']:
nvd_links = nvd_data['result']['CVE_Items'][0]['cve']['references']['reference_data']
cve_sources = ""
for link in nvd_links:
cve_sources += (link['url']+"\n")
else:
cve_sources = "N/A"
cve_score = ""
fr_society = " (constructeur/éditeur)"
en_society = " (builder/publisher)"
if not "impact" in nvd_data['result']['CVE_Items'][0].keys():
cve_score = "N/A"
try:
webpage = requests.get("https://nvd.nist.gov/vuln/detail/{}".format(cve_id))
soup = BeautifulSoup(webpage.content, 'html.parser')
cve_cna_score = soup.find(id='Cvss3CnaCalculatorAnchor')
cve_cna_score = cve_cna_score.get_text()
if cve_cna_score != "":
if language == "fr" or language=="FR":
cve_score = cve_cna_score + fr_society
else:
cve_score = cve_cna_score + en_society
else:
cve_score = "N/A"
except:
cve_score = "N/A"
else:
try:
cve_score = nvd_data['result']['CVE_Items'][0]['impact']['baseMetricV3']['cvssV3']['baseScore']
except:
cve_score = "N/A"
published_date = nvd_data['result']['CVE_Items'][0]['publishedDate']
formatted_date = published_date.split("T")[0]
formatted_date = datetime.strptime(formatted_date,"%Y-%m-%d")
date_2_days_ago = datetime.now() - timedelta(days=7)
if (date_2_days_ago<=formatted_date) and (formatted_date<=datetime.now()):
cve_date_status = "Potentially new CVE"
else:
cve_date_status = "Potentially an update"
cve_date = published_date.split("T")[0]
if language == "fr" or language == "FR":
fr_date = cve_date.split("-")
cve_date = fr_date[2]+"/"+fr_date[1]+"/"+fr_date[0]
if (cve_score=="N/A"):
cve_status = cve_date_status+" - "+"Not yet rated (No score, no CPE)"
else:
cve_status = cve_date_status+" - "+"Valid evaluation"
cve_cpe = ""
if "configurations" in nvd_data['result']['CVE_Items'][0].keys():
if nvd_data['result']['CVE_Items'][0]['configurations']['nodes']:
for node in nvd_data['result']['CVE_Items'][0]['configurations']['nodes']:
if 'cpe_match' in node.keys():
for cpe_node in node['cpe_match']:
cve_cpe += (cpe_node['cpe23Uri']+'\n')
if cve_cpe == "":
cve_cpe = "N/A"
cve_description = nvd_data['result']['CVE_Items'][0]['cve']['description']['description_data'][0]['value']
nvd_link = 'https://nvd.nist.gov/vuln/detail/'+cve_id
if language == "fr" or language=="FR":
try:
cve_description = translateText(dest_language,cve_description)
except:
pass
if not "/" in cve_date:
cve_date = cve_date.split("-")[2]+"/"+cve_date.split("-")[1]+"/"+cve_date.split("-")[0]
try:
cve_status = translateText(dest_language,cve_status).replace("nouveau", "une nouvelle").replace("évalué","évaluée")
except:
pass
else:
cve_description = summary
date = str(datetime.now()).split(" ")[0].split("-")
cve_date = date[2]+"/"+date[1]+"/"+date[0]
cve_score = "N/A"
cve_sources = "N/A"
cve_status = "N/A"
cve_cpe = "N/A"
nvd_link = 'https://nvd.nist.gov/vuln/detail/'+cve_id
status = "Unread"
if cve_id not in new_cve_list:
cur.execute("INSERT INTO CVE_DATA (CVE_ID,KEYWORD,STATUS,CVE_SCORE,CVE_DATE,CVE_DESCRIPTION,CVE_EVAL,CVE_CPE,CVE_SOURCES,EXPLOIT_FIND,INDEXING_DATE) VALUES (?,?,?,?,?,?,?,?,?,?,?);", (cve_id,key_match,status,str(cve_score).split(" ")[0],cve_date,cve_description,cve_status,cve_cpe,cve_sources,"False",idx_date))
con.commit()
print("New CVE detected -> "+cve_id)
report="updated"
try:
sendAlert(smtp_login, smtp_password, smtpsrv, port, tls, sender, receivers, cve_id, cve_sources, str(cve_score).split(" ")[0], cve_status, cve_cpe, cve_date, cve_description, language,key_match)
alert = "sent"
except Exception as e:
handleException(e)
alert = "unsent"
new_cve_list.append(cve_id)
writeCveTypeLog("cve_poller",cve_id,"new",key_match,str(cve_score).split(" ")[0],cve_date,cve_cpe,report,alert)
else:
if bool(re.search(key,summary,re.IGNORECASE)) == True:
key_match = key
cur.execute("SELECT CVE_ID FROM CVE_DATA WHERE CVE_ID = (?);", (cve_id,))
db_result_list = cur.fetchall()
db_result_str = ""
for db_result_tuple in db_result_list:
for result in db_result_tuple:
db_result_str+=result
if cve_id not in db_result_str:
time.sleep(15)
nvd_base_url = "https://services.nvd.nist.gov/rest/json/cve/1.0/"
nvd_query = nvd_base_url+cve_id
nvd_response = requests.get(url=nvd_query)
nvd_data = nvd_response.json()
if 'result' in nvd_data.keys():
if 'reference_data' in nvd_data['result']['CVE_Items'][0]['cve']['references']:
nvd_links = nvd_data['result']['CVE_Items'][0]['cve']['references']['reference_data']
cve_sources = ""
for link in nvd_links:
cve_sources += (link['url']+"\n")
else:
cve_sources = "N/A"
cve_score = ""
fr_society = " (constructeur/éditeur)"
en_society = " (builder/publisher)"
if not "impact" in nvd_data['result']['CVE_Items'][0].keys():
cve_score = "N/A"
try:
webpage = requests.get("https://nvd.nist.gov/vuln/detail/{}".format(cve_id))
soup = BeautifulSoup(webpage.content, 'html.parser')
cve_cna_score = soup.find(id='Cvss3CnaCalculatorAnchor')
cve_cna_score = cve_cna_score.get_text()
if cve_cna_score != "":
if language == "fr" or language=="FR":
cve_score = cve_cna_score + fr_society
else:
cve_score = cve_cna_score + en_society
else:
cve_score = "N/A"
except:
cve_score = "N/A"
else:
try:
cve_score = nvd_data['result']['CVE_Items'][0]['impact']['baseMetricV3']['cvssV3']['baseScore']
except:
cve_score = "N/A"
published_date = nvd_data['result']['CVE_Items'][0]['publishedDate']
formatted_date = published_date.split("T")[0]
formatted_date = datetime.strptime(formatted_date,"%Y-%m-%d")
date_2_days_ago = datetime.now() - timedelta(days=7)
if (date_2_days_ago<=formatted_date) and (formatted_date<=datetime.now()):
cve_date_status = "Potentially new CVE"
else:
cve_date_status = "Potentially an update"
cve_date = published_date.split("T")[0]
if language == "fr" or language == "FR":
fr_date = cve_date.split("-")
cve_date = fr_date[2]+"/"+fr_date[1]+"/"+fr_date[0]
if (cve_score=="N/A"):
cve_status = cve_date_status+" - "+"Not yet rated (No score, no CPE)"
else:
cve_status = cve_date_status+" - "+"Valid evaluation"
cve_cpe = ""
if "configurations" in nvd_data['result']['CVE_Items'][0].keys():
if nvd_data['result']['CVE_Items'][0]['configurations']['nodes']:
for node in nvd_data['result']['CVE_Items'][0]['configurations']['nodes']:
if 'cpe_match' in node.keys():
for cpe_node in node['cpe_match']:
cve_cpe += (cpe_node['cpe23Uri']+'\n')
if cve_cpe == "":
cve_cpe = "N/A"
cve_description = nvd_data['result']['CVE_Items'][0]['cve']['description']['description_data'][0]['value']
nvd_link = 'https://nvd.nist.gov/vuln/detail/'+cve_id
if language == "fr" or language=="FR":
try:
cve_description = translateText(dest_language,cve_description)
except:
pass
if not "/" in cve_date:
cve_date = cve_date.split("-")[2]+"/"+cve_date.split("-")[1]+"/"+cve_date.split("-")[0]
try:
cve_status = translateText(dest_language,cve_status).replace("nouveau", "une nouvelle").replace("évalué","évaluée")
except:
pass
else:
cve_description = summary
date = str(datetime.now()).split(" ")[0].split("-")
cve_date = date[2]+"/"+date[1]+"/"+date[0]
cve_score = "N/A"
cve_sources = "N/A"
cve_status = "N/A"
cve_cpe = "N/A"
nvd_link = 'https://nvd.nist.gov/vuln/detail/'+cve_id
status = "Unread"
if cve_id not in new_cve_list:
cur.execute("INSERT INTO CVE_DATA (CVE_ID,KEYWORD,STATUS,CVE_SCORE,CVE_DATE,CVE_DESCRIPTION,CVE_EVAL,CVE_CPE,CVE_SOURCES,EXPLOIT_FIND,INDEXING_DATE) VALUES (?,?,?,?,?,?,?,?,?,?,?);", (cve_id,key_match,status,str(cve_score).split(" ")[0],cve_date,cve_description,cve_status,cve_cpe,cve_sources,"False",idx_date))
con.commit()
print("New CVE detected -> "+cve_id)
report="updated"
try:
sendAlert(smtp_login, smtp_password, smtpsrv, port, tls, sender, receivers, cve_id, cve_sources, str(cve_score).split(" ")[0], cve_status, cve_cpe, cve_date, cve_description, language,key_match)
alert = "sent"
except Exception as e:
handleException(e)
alert = "unsent"
new_cve_list.append(cve_id)
writeCveTypeLog("cve_poller",cve_id,"new",key_match,str(cve_score).split(" ")[0],cve_date,cve_cpe,report,alert)
if not new_cve_list:
print(bcolors.HEADER+"No new CVE matched with your keyword list."+bcolors.ENDC)
print("------------------------------------")
timestamp = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
print(bcolors.OKBLUE+"Finished at : "+timestamp+bcolors.ENDC)
print("------------------------------------")
else:
print(bcolors.OKGREEN+"Database was updated with new CVE."+bcolors.ENDC)
timestamp = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
print("------------------------------------")
print(bcolors.OKBLUE+"Finished at : "+timestamp+bcolors.ENDC)
print("------------------------------------")
def sendAlert(smtp_login, smtp_password, smtpsrv, port, tls, sender, receivers, cve_id, cve_sources, cve_score, cve_status, cve_cpe, cve_date, cve_description, language,key_match):
body = ""
nvd_url = "https://nvd.nist.gov/vuln/detail/"+cve_id
for receiver in receivers:
with open(dir_path+'cve_template.html', 'r') as template:
html_code = template.read()
if "cpe" in key_match:
key_match = getParsedCpe(key_match)
if language == "fr" or language=="FR":
html_code = html_code.replace("Responsive HTML email templates","Alerte")
html_code = html_code.replace("$CVE",cve_id+" (Produit : {})".format(key_match))
try:
cve_description = translateText(dest_language,cve_description)
except:
pass
html_code = html_code.replace("$DESCRIPTION", cve_description.replace("u'","").replace("u ``",""))
if not "/" in cve_date:
cve_date = cve_date.split("-")[2]+"/"+cve_date.split("-")[1]+"/"+cve_date.split("-")[0]
html_code = html_code.replace("$DATE", cve_date)
html_code = html_code.replace("$SCORE", str(cve_score))
try:
cve_status = translateText(dest_language,cve_status).replace("nouveau", "une nouvelle").replace("évalué","évaluée")
except:
pass
html_code = html_code.replace("$STATUS", cve_status)
html_code = html_code.replace("$CPE", cve_cpe)
html_code = html_code.replace("$SOURCES", cve_sources)
html_code = html_code.replace("See more details", "Voir plus de détails")
html_code = html_code.replace("Publication date", "Date de publication")
html_code = html_code.replace("Status", "Statut")
html_code = html_code.replace("CPE/Product", "CPE/Produit")
html_code = html_code.replace("CVSS Score (V3)", "Score CVSS (V3)")
html_code = html_code.replace("This email was sent by", "Cet email a été envoyé par")
html_code = html_code.replace("CVE module", "Module CVE")
body = html_code.replace("URL OF THE NEWS",nvd_url)
else:
html_code = html_code.replace("Responsive HTML email templates","Alert")
html_code = html_code.replace("$CVE",cve_id+" (Product : {})".format(key_match))
html_code = html_code.replace("$DESCRIPTION", cve_description.replace("u'","").replace("u ``",""))
html_code = html_code.replace("$DATE", cve_date)
html_code = html_code.replace("$SCORE", str(cve_score))
html_code = html_code.replace("$STATUS", cve_status)
html_code = html_code.replace("$CPE", cve_cpe)
html_code = html_code.replace("$SOURCES", cve_sources)
body = html_code.replace("URL OF THE NEWS",nvd_url)
print(bcolors.HEADER+"Sending alert at {}...".format(receiver)+bcolors.ENDC)
try:
smtpserver = smtplib.SMTP(smtpsrv,port)
msg = MIMEMultipart()
msg['Subject'] = 'SECMON - CVE'
msg['From'] = sender
msg['To'] = receiver
msg.attach(MIMEText(body, 'html'))
except Exception as e:
handleException(e)
print(bcolors.FAIL+"Failed to send alert at {}.".format(receiver)+bcolors.ENDC)
exit()
try:
if tls == "yes":
smtpserver.ehlo()
smtpserver.starttls()
smtpserver.login(smtp_login, smtp_password)
smtpserver.sendmail(sender, receiver, msg.as_string())
print(bcolors.HEADER+"Alert was sent at {}\n".format(receiver)+bcolors.ENDC)
elif tls == "no":
smtpserver.login(smtp_login, smtp_password)
smtpserver.sendmail(sender, receiver, msg.as_string())
print(bcolors.HEADER+"Alert was sent at {}\n".format(receiver)+bcolors.ENDC)
except Exception as e:
handleException(e)
print(bcolors.FAIL+"An error occurred during authentication with the SMTP server. Check the configuration and try again."+bcolors.ENDC)
exit()
def main():
colorama.init()
print("------------ CVE Module ------------")
timestamp = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
print(bcolors.OKBLUE+"Starting at : "+timestamp+bcolors.ENDC)
print("------------------------------------")
checkConfig(sender, receiver, smtp_login, smtp_password, smtpsrv, port, tls,keywords)
main()
================================================
FILE: cve_template.html
================================================
SECMON - CVE$CVE : $DESCRIPTION
================================================
FILE: cve_updater.py
================================================
# -*- coding: utf-8 -*-
"""
SECMON - Source code of the SECMON updater script.
"""
__author__ = "Aubin Custodio"
__copyright__ = "Copyright 2021, SECMON"
__credits__ = ["Aubin Custodio","Guezone"]
__license__ = "CC BY-NC-SA 4.0"
__version__ = "2.1"
__maintainer__ = "Aubin Custodio"
__email__ = "custodio.aubin@outlook.com"
from secmon_lib import writeNewExploitFoundLog, getUnexploitableCveIdList,getGithubAPISettings,getRegisteredCveInfos, get_db_connection, getUnregisteredCveInfos, getFormatedProductList, getCveByProduct, writeNewHighRiskProductLog, writeCveTypeLog
from datetime import datetime
import os,requests,time
import cve_searchsploit as CS
script_path = os.path.abspath(__file__)
dir_path = script_path.replace("cve_updater.py","")
log_file = dir_path+"logs.txt"
print("------------ CVE Module - Update ------------")
timestamp = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
print("Starting at : "+timestamp)
print("---------------------------------------------")
CS.update_db()
con = get_db_connection()
cur = con.cursor()
cve_list = getUnexploitableCveIdList()
for cve in cve_list:
exploitdb_exploits = CS.edbid_from_cve(cve)
if len(exploitdb_exploits) > 0:
cur.execute("UPDATE CVE_DATA SET EXPLOIT_FIND = (?) WHERE CVE_ID = (?)", ("True",cve))
con.commit()
print("Exploit found for "+cve+" from Exploit-DB")
writeNewExploitFoundLog("cve_updater","exploit-db",cve,f"New(s) exploit found for {cve} from Exploit-DB !")
print("\nSearch Github Exploit for registered CVE...")
cve_list = getUnexploitableCveIdList()
user,key = getGithubAPISettings()
con = get_db_connection()
cur = con.cursor()
if user == "None" or key == "None":
print("No Github API configuration found.")
else:
for cve in cve_list:
time.sleep(30)
github_query = "https://api.github.com/search/repositories?q=exploit+"+cve
github_response = requests.get(url=github_query,auth=(user,key))
github_data = github_response.json()
if "items" in github_data.keys():
if len(github_data['items']) > 0:
cur.execute("UPDATE CVE_DATA SET EXPLOIT_FIND = (?) WHERE CVE_ID = (?)", ("True",cve))
con.commit()
print("Exploit found for "+cve+" from Github")
writeNewExploitFoundLog("cve_updater","github",cve,f"New exploit found for {cve} from Github !")
print("\nUpdating high risk product list....")
plist = getFormatedProductList()
con = get_db_connection()
cur = con.cursor()
critical_product = []
old_critical_product = []
cur.execute("SELECT hname FROM high_risk_products;")
db_result_list = cur.fetchall()
for db_result_tuple in db_result_list:
for result in db_result_tuple:
if result not in old_critical_product:
old_critical_product.append(result)
for product in plist:
if "cpe" in product[1]:
target_product = product[1]
else:
target_product = product[0]
critical_cve, high_cve, medium_cve, low_cve, na_cve, exploitable_cve = getCveByProduct(target_product, True)
if exploitable_cve != []:
if product[0] not in old_critical_product:
critical_product.append(product)
print("New critical product : "+product[0])
cur.execute("INSERT INTO high_risk_products (cpe,hname) VALUES (?,?);", (product[1],product[0]))
writeNewHighRiskProductLog("cve_updater",product[1],f"A new critical product was discovered. ({product[1]})")
con.commit()
print("\nUpdating affected products and CVSS score related to your CVE list...")
con = get_db_connection()
cur = con.cursor()
req = cur.execute("SELECT CVE_ID FROM CVE_DATA")
indb_cve_list = []
changelog = []
for tup in req:
for cve in tup:
indb_cve_list.append(cve)
for cve in indb_cve_list:
try:
try:
before_update = getRegisteredCveInfos(cve, full=True)
time.sleep(5)
after_update = getUnregisteredCveInfos(cve)
except:
continue
for cpe in before_update['cve_cpe']:
if cpe == '':
before_update['cve_cpe'].remove(cpe)
for item in before_update.keys():
if item == "cve_score":
if str(after_update[item]) != str(before_update[item]):
writeCveTypeLog("cve_updater",cve,"update","N/A",str(after_update[item]),"N/A","N/A","N/A","N/A","The CVSS Score was changed from {} to {}.".format(str(before_update[item]), str(after_update[item])))
changelog.append(cve+" - There has been a change in the CVSS score : {} TO {}\n".format(str(before_update[item]), str(after_update[item])))
print(cve+" - There has been a change in the CVSS score : {} TO {}\n".format(str(before_update[item]), str(after_update[item])))
cur.execute("UPDATE CVE_DATA SET CVE_SCORE = (?) WHERE CVE_ID = (?)", (str(after_update[item]),cve))
con.commit()
elif item == "cve_cpe":
if after_update[item] != before_update[item]:
change = ""
diff = list(set(after_update[item])-set(before_update[item]))
if diff == []:
change = "removed"
diff = list(set(before_update[item])-set(after_update[item]))
cpe_diff = ""
for elem in diff:
if len(diff) == 1 or elem == diff[-1]:
cpe_diff += elem
else:
cpe_diff = cpe_diff+elem+" AND "
else:
change = "added"
cpe_diff = ""
for elem in diff:
if len(diff) == 1 or elem == diff[-1]:
cpe_diff += elem
else:
cpe_diff = cpe_diff+elem+" AND "
writeCveTypeLog("cve_updater",cve,"update","N/A",str(after_update[item]),"N/A","N/A","N/A","N/A","The following products have been {} : {} \n".format(change,cpe_diff))
changelog.append(cve+" - The following products have been {} : {} \n".format(change,cpe_diff))
print(cve+" - The following products have been {} : {} \n".format(change,cpe_diff))
new_cpe = ""
for cpe in after_update[item]:
new_cpe+=(cpe+"\n")
cur.execute("UPDATE CVE_DATA SET CVE_CPE = (?) WHERE CVE_ID = (?)", (new_cpe,cve))
con.commit()
except:
continue
if changelog == []:
print("No CVE have been updated. \n")
timestamp = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
print("---------------------------------------------")
print("Finished at : "+timestamp)
print("---------------------------------------------")
================================================
FILE: docker/install.py
================================================
from time import sleep
import os
from getpass import getpass
print("| SECMON - DockerAutoInstall |")
print("1. Certificate : ")
print("Note 1 : You can delete this self signed cert and put your CA signed cert after install. ")
sleep(2)
fqdn = input("Enter the FQDN of the web server :")
print("Note 2 : Don't forget to enter a good value for CN (FQDN of your server)")
sleep(2)
os.system("openssl req -newkey rsa:4096 -x509 -sha512 -days 3650 -nodes -out /etc/ssl/secmon/secmon.crt -keyout /etc/ssl/secmon/secmon.key")
print("Your certificate is available on /etc/ssl/secmon ! ")
print("----------------------------------------------------------------------------------------------------------------------")
print("2. SECMON Install : ")
sender = input("Sender email address :")
smtp_login = input("SMTP account login :")
smtp_password = getpass("SMTP email account password :")
smtpsrv = input("SMTP Server FQDN or IP :")
smtpport = input("SMTP used port :")
tls = input("Using TLS SMTP auth ? (yes/no) :")
lang = input("Language (en/fr) :")
receivers = input("SECMON email receivers (single or many seperated by ;):")
os.system(f"chown -R www-data:www-data /var/www/secmon && chmod -R 744 /var/www/secmon")
os.system(f"python3 /var/www/secmon/setup.py -login '{smtp_login}' -p '{smtp_password}' -server '{smtpsrv}' -port '{smtpport}' -tls '{tls}' -lang '{lang}' -sender '{sender}' -r '{receivers}'")
print("Executing secmon in background...")
os.system("nohup python3 /var/www/secmon/secmon.py >/dev/null 2>&1 &")
print("Executing cve_updater in background...")
os.system("nohup python3 /var/www/secmon/cve_updater.py >/dev/null 2>&1 &")
print("SECMON is successfully installed and configured !")
print("----------------------------------------------------------------------------------------------------------------------")
print("3. Apache configuration : ")
f = open("/var/www/secmon/docker/secmon.conf").read()
os.system("chown -R www-data:www-data /var/www/secmon/ && chmod -R 744 /var/www/secmon/")
content = f.replace("{FQDN}",fqdn)
f = open("/etc/apache2/sites-enabled/secmon.conf","w")
f.write(content)
f.close()
os.system("a2enmod ssl && service apache2 restart")
print("Apache is successfully configured !")
print(f"You can now access the web interface at the following address: https://{fqdn}")
print("----------------------------------------------------------------------------------------------------------------------")
================================================
FILE: docker/secmon-ci.conf
================================================
ServerName {FQDN}
WSGIScriptAlias / /var/www/secmon/secmon.wsgi
WSGIDaemonProcess secmon_web
DocumentRoot /var/www/secmon
ServerName {FQDN}
ServerAlias www.{FQDN}
WSGIProcessGroup secmon_web
WSGIPassAuthorization On
WSGIApplicationGroup %{GLOBAL}
Order deny,allow
Allow from all
ServerSignature Off
LogLevel info
ErrorLog ${APACHE_LOG_DIR}/error_secmon.log
CustomLog ${APACHE_LOG_DIR}/access_secmon.log combined
SSLProtocol -all +TLSv1.2 +TLSv1.3
SSLEngine on
SSLCertificateFile /etc/ssl/secmon/secmon.crt
SSLCertificateKeyFile /etc/ssl/secmon/secmon.key
================================================
FILE: docker/secmon.conf
================================================
ServerName {FQDN}
ServerAdmin admin@mail.com
ServerName {FQDN}
ServerAlias www.{FQDN}
DocumentRoot /var/www/secmon
Redirect permanent / https://{FQDN}
WSGIProcessGroup secmon_web
WSGIPassAuthorization On
WSGIApplicationGroup %{GLOBAL}
Order deny,allow
Allow from all
ServerSignature off
ErrorLog ${APACHE_LOG_DIR}/error_secmon.log
CustomLog ${APACHE_LOG_DIR}/access_secmon.log combined
WSGIScriptAlias / /var/www/secmon/secmon.wsgi
WSGIDaemonProcess secmon_web
DocumentRoot /var/www/secmon
ServerName {FQDN}
ServerAlias www.{FQDN}
WSGIProcessGroup secmon_web
WSGIPassAuthorization On
WSGIApplicationGroup %{GLOBAL}
Order deny,allow
Allow from all
ServerSignature Off
LogLevel info
ErrorLog ${APACHE_LOG_DIR}/error_secmon.log
CustomLog ${APACHE_LOG_DIR}/access_secmon.log combined
SSLProtocol -all +TLSv1.2 +TLSv1.3
SSLEngine on
SSLCertificateFile /etc/ssl/secmon/secmon.crt
SSLCertificateKeyFile /etc/ssl/secmon/secmon.key
================================================
FILE: requirements.txt
================================================
Jinja2==3.0.1
Werkzeug==2.0
Flask_SQLAlchemy==2.4.4
requests==2.22.0
colorama==0.4.3
tweepy==3.8.0
openpyxl==3.0.5
git+https://github.com/Guezone/cve_searchsploit
Flask==2.0.3
deep_translator==1.3.2
feedparser==6.0.8
beautifulsoup4==4.9.3
flask_simplelogin==0.0.7
python-dateutil
PyGithub
traceback-with-variables==2.0.4
================================================
FILE: rss_poller.py
================================================
# -*- coding: utf-8 -*-
"""
SECMON - Source code of the SECMON rss poller script.
"""
__author__ = "Aubin Custodio"
__copyright__ = "Copyright 2021, SECMON"
__credits__ = ["Aubin Custodio","Guezone"]
__license__ = "CC BY-NC-SA 4.0"
__version__ = "2.1"
__maintainer__ = "Aubin Custodio"
__email__ = "custodio.aubin@outlook.com"
import base64, os, smtplib, re, requests, sqlite3, feedparser, colorama
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from datetime import datetime, timedelta
from secmon_lib import translateText, getUserLanguage, writeMgmtTasksLog, writeNewRssNewLog,handleException
rss_feeds = [
'https://cyware.com/allnews/feed',
'https://www.cshub.com/rss/categories/attacks',
'https://www.schneier.com/feed/atom/',
'https://www.cshub.com/rss/categories/threat-defense',
'https://www.cshub.com/rss/categories/malware',
'https://www.fortiguard.com/rss/ir.xml',
'https://nakedsecurity.sophos.com/feed/',
'https://www.techrepublic.com/rssfeeds/topic/security/?feedType=rssfeeds',
'https://www.cert.ssi.gouv.fr/feed/',
'https://us-cert.cisa.gov/ncas/all.xml',
'https://www.zataz.com/feed/',
]
sender, receiver, smtp_login, smtp_password, smtpsrv, port, tls, keywords = '','','','','','','',''
script_path = os.path.abspath(__file__)
dir_path = script_path.replace("rss_poller.py","")
log_file = dir_path+"logs.txt"
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
def parseRSSSource(source):
if "us-cert.cisa" in source or "cyware.com" in source:
new_source = source.replace("https://","").split(".")[0]
else:
new_source = source.replace("https://","").replace("www.","").split(".")[0]
return new_source
def checkConfig(sender, receivers, smtp_login, smtp_password, smtpsrv, port, tls,rss_feeds):
script_path = os.path.abspath(__file__)
dir_path = script_path.replace("rss_poller.py","")
con = sqlite3.connect(dir_path+'secmon.db')
cur = con.cursor()
cur.execute("SELECT sender FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
sender = value
cur.execute("SELECT smtp_login FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
smtp_login = value
cur.execute("SELECT smtp_password FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
b = value.encode("UTF-8")
bytes_password = base64.b64decode(b)
smtp_password = bytes_password.decode("UTF-8")
cur.execute("SELECT smtpsrv FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
smtpsrv = value
cur.execute("SELECT port FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
port = value
cur.execute("SELECT receiver FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
receivers = value.split(";")
cur.execute("SELECT tls FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
tls = value
cur.execute("SELECT language FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
language = value
cur.execute("SELECT receiver FROM config")
db_result_tuple = cur.fetchone()
for value in db_result_tuple:
receivers = value.split(";")
cur.execute("SELECT keyword FROM keyword_list")
db_result_list = cur.fetchall()
keywords = []
for db_result_tuple in db_result_list:
for result in db_result_tuple:
keywords.append(result)
if all(value != '' for value in [sender, receivers, smtp_login, smtp_password, smtpsrv, str(port), tls, language]):
print(bcolors.OKGREEN+"Configuration is good."+bcolors.ENDC)
rssPoller(sender, receivers, smtp_login, smtp_password, smtpsrv, port, tls, language, rss_feeds)
else:
print(bcolors.FAIL+"Error in the config."+bcolors.ENDC)
def rssPoller(sender, receivers, smtp_login, smtp_password, smtpsrv, port, tls, language, rss_feeds):
print("------------------------------------")
summary, url = "",""
new = False
con = sqlite3.connect(dir_path+'secmon.db')
cur = con.cursor()
for rss_feed in rss_feeds:
current_feed = feedparser.parse(rss_feed)
for entries in current_feed.entries:
url = entries.link
cur.execute("SELECT RSS_URL FROM RSS_DATA WHERE RSS_URL = (?);", (url,))
db_result_list = cur.fetchall()
db_result_str = ""
for db_result_tuple in db_result_list:
for result in db_result_tuple:
db_result_str+=result
if url not in db_result_str:
new = True
title = entries.title
summary = entries.summary
if summary == "":
summary+="Empty summary."
dest_language = getUserLanguage()
try:
title = translateText(dest_language,title)
summary = translateText(dest_language,summary)
except:
pass
cur.execute("INSERT INTO RSS_DATA (RSS_URL, title, rss_f, summary) VALUES (?,?,?,?);", (url,title,rss_feed,summary))
con.commit()
try:
sendAlert(smtp_login, smtp_password, smtpsrv, port, tls, sender, receivers, title, summary, url, language, rss_feed)
mail = "sent"
except Exception as e:
handleException(e)
mail = "unsent"
writeNewRssNewLog("rss_poller",parseRSSSource(rss_feed),rss_feed,title,url,mail)
if new == True:
print(bcolors.OKGREEN+"Database was updated. Goodbye."+bcolors.ENDC)
else:
print(bcolors.OKGREEN+"No news. Goodbye."+bcolors.ENDC)
print("------------------------------------")
timestamp = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
print(bcolors.OKBLUE+"Finished at : "+timestamp+bcolors.ENDC)
print("------------------------------------")
def sendAlert(smtp_login, smtp_password, smtpsrv, port, tls, sender, receivers, title, summary, url, language, rss_feed):
body = ""
for receiver in receivers:
with open(dir_path+'rss_template.html', 'r') as template:
html_code = template.read()
html_code = html_code.replace("$SOURCE", parseRSSSource(rss_feed))
if language == "fr" or language=="FR":
html_code = html_code.replace("See more details", "Voir plus de détails")
html_code = html_code.replace("This email was sent by", "Cet email a été envoyé par")
html_code = html_code.replace("Notification", "Information")
html_code = html_code.replace("RSS module", "Module RSS")
html_code = html_code.replace("$TITLE", title)
html_code = html_code.replace("$SUMMARY", summary)
html_code = html_code.replace("Title", "Titre")
html_code = html_code.replace("Summary", "Résumé")
body = html_code.replace("URL OF THE NEWS",url)
else:
html_code = html_code.replace("$TITLE", title)
html_code = html_code.replace("Notification", "News")
html_code = html_code.replace("$SUMMARY", summary)
body = html_code.replace("URL OF THE NEWS",url)
print(bcolors.HEADER+"Sending news at {}...".format(receiver)+bcolors.ENDC)
try:
smtpserver = smtplib.SMTP(smtpsrv,port)
msg = MIMEMultipart()
msg['Subject'] = 'SECMON - RSS'
msg['From'] = sender
msg['To'] = receiver
msg.attach(MIMEText(body, 'html'))
except Exception as e:
handleException(e)
print(bcolors.FAIL+"Failed to send news at {}.".format(receiver)+bcolors.ENDC)
exit()
try:
if tls == "yes":
smtpserver.ehlo()
smtpserver.starttls()
smtpserver.login(smtp_login, smtp_password)
smtpserver.sendmail(sender, receiver, msg.as_string())
print(bcolors.HEADER+"News was sent at {}\n".format(receiver)+bcolors.ENDC)
elif tls == "no":
smtpserver.login(smtp_login, smtp_password)
smtpserver.sendmail(sender, receiver, msg.as_string())
print(bcolors.HEADER+"News was sent at {}\n".format(receiver)+bcolors.ENDC)
except Exception as e:
handleException(e)
print(bcolors.FAIL+"An error occurred during authentication with the SMTP server. Check the configuration and try again."+bcolors.ENDC)
exit()
def main():
print()
colorama.init()
print("------------ RSS Module ------------")
timestamp = datetime.now().strftime("%d/%m/%Y %H:%M:%S")
print(bcolors.OKBLUE+"Starting at : "+timestamp+bcolors.ENDC)
print("------------------------------------")
checkConfig(sender, receiver, smtp_login, smtp_password, smtpsrv, port, tls,rss_feeds)
main()
================================================
FILE: rss_template.html
================================================
SECMON - RSS$SUMMARY
