Repository: terdia/mqttui Branch: main Commit: ec08e99df1a3 Files: 96 Total size: 440.5 KB Directory structure: gitextract_wv3k4jzs/ ├── .env_example ├── .github/ │ └── workflows/ │ └── docker-publish.yml ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── Dockerfile.multiarch ├── LICENSE.md ├── README.md ├── app.py ├── database.py ├── debug_bar.py ├── demo.sh ├── docker-compose.yml ├── entrypoint.sh ├── mosquitto.conf ├── mqttui/ │ ├── __init__.py │ ├── analytics.py │ ├── app.py │ ├── auth.py │ ├── broker_manager.py │ ├── database.py │ ├── events.py │ ├── extensions.py │ ├── helpers.py │ ├── logging_config.py │ ├── models.py │ ├── mqtt_client.py │ ├── plugins/ │ │ ├── __init__.py │ │ ├── examples/ │ │ │ ├── __init__.py │ │ │ ├── json_formatter.py │ │ │ └── topic_logger.py │ │ ├── hookspec.py │ │ ├── models.py │ │ ├── registry.py │ │ └── runner.py │ ├── routes/ │ │ ├── __init__.py │ │ ├── alerts.py │ │ ├── analytics.py │ │ ├── api.py │ │ ├── api_v1.py │ │ ├── brokers.py │ │ ├── debug.py │ │ ├── main.py │ │ ├── metrics.py │ │ ├── plugins.py │ │ └── rules.py │ ├── rules/ │ │ ├── __init__.py │ │ ├── actions.py │ │ ├── cooldown.py │ │ ├── engine.py │ │ ├── evaluator.py │ │ ├── models.py │ │ └── ssrf.py │ ├── socketio_batch.py │ └── state.py ├── pytest.ini ├── requirements-dev.txt ├── requirements.txt ├── static/ │ ├── css/ │ │ ├── input.css │ │ └── output.css │ ├── script.js │ └── styles.css ├── templates/ │ ├── base.html │ ├── index.html │ ├── login.html │ └── partials/ │ ├── alert_row.html │ ├── alerts_list.html │ ├── analytics.html │ ├── brokers.html │ ├── dry_run_result.html │ ├── plugins.html │ ├── rule_form.html │ ├── rule_row.html │ └── rules_list.html ├── tests/ │ ├── __init__.py │ ├── conftest.py │ ├── test_alerts_api.py │ ├── test_analytics.py │ ├── test_api_v1.py │ ├── test_app_factory.py │ ├── test_auth.py │ ├── test_cooldown.py │ ├── test_database.py │ ├── test_frontend.py │ ├── test_observability.py │ ├── test_plugin_registry.py │ ├── test_plugin_runner.py │ ├── test_plugins_api.py │ ├── test_routes.py │ ├── test_rules_api.py │ ├── test_rules_engine.py │ ├── test_rules_evaluator.py │ ├── test_rules_models.py │ ├── test_ux_features.py │ └── test_webhook.py └── wsgi.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .env_example ================================================ DEBUG=False HOST=0.0.0.0 PORT=5000 MQTT_BROKER=your_mqtt_broker MQTT_PORT=1883 MQTT_USERNAME=your_username MQTT_PASSWORD=your_password MQTT_KEEPALIVE=60 MQTT_VERSION=3.1.1 MQTT_TOPICS=# SECRET_KEY=your-secret-key LOG_LEVEL=INFO DB_ENABLED=True DB_PATH=mqtt_messages.db DB_MAX_MESSAGES=10000 DB_CLEANUP_DAYS=30 # TLS/SSL (for MQTTS connections) MQTT_TLS=false MQTT_TLS_CA_CERTS= MQTT_TLS_CERTFILE= MQTT_TLS_KEYFILE= MQTT_TLS_INSECURE=false # v2.0 Authentication MQTTUI_ADMIN_USER=admin MQTTUI_ADMIN_PASSWORD=changeme MQTTUI_RATE_LIMIT=30/minute ================================================ FILE: .github/workflows/docker-publish.yml ================================================ name: Docker on: push: tags: [ 'v*.*.*' ] env: DOCKER_HUB_REPO: terdia07/mqttui jobs: push_to_registry: name: Push Docker image to Docker Hub runs-on: ubuntu-latest steps: - name: Check out the repo uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Log in to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v5 with: images: ${{ env.DOCKER_HUB_REPO }} - name: Build and push Docker image uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile.multiarch platforms: linux/amd64,linux/arm64,linux/arm/v7 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} ================================================ FILE: .gitignore ================================================ # Environment variables .env # Python __pycache__ *.pyc *.pyo # Database files data/ *.db *.sqlite *.sqlite3 # Logs *.log *.log.* # macOS .DS_Store # Static assets (if needed) static/mqttui-logo-svg.svg static/mqttui-logo.png # SQLite WAL/SHM temp files *.db-shm *.db-wal # GSD planning artifacts .planning/ ================================================ FILE: CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [2.1.0] - 2026-03-24 ### Added - **Multi-broker support**: Connect to multiple MQTT brokers simultaneously from the Brokers tab - **Broker management API**: Full CRUD at `/api/v1/brokers/` with connect/disconnect endpoints - **Broker management UI**: Add, edit, remove, connect, disconnect brokers with live status indicators - **Broker filtering**: Filter messages by source broker in Advanced Search dropdown - **Broker name badges**: Each message shows which broker it came from - **Demo script updated**: `./demo.sh` populates both brokers with different data profiles (home vs warehouse) - **Backward compatible**: Existing env var config auto-seeds a "Default" broker on first run ### Fixed - Debug bar performance tab now uses modern Performance API (deprecated `window.performance.timing` replaced) ## [2.0.0] - 2026-03-24 ### Added #### Automation Rules Engine - **IF/THEN automation rules**: Create rules that evaluate conditions against incoming MQTT messages and fire actions automatically - **11-operator condition evaluator**: eq, ne, gt, lt, gte, lte, contains, not_contains, regex, exists, not_exists - **Compound conditions**: `all` (AND) and `any` (OR) for complex logic - **JSON path support**: Dot-notation paths like `sensors.outdoor.temp` for nested payload fields - **Action types**: Publish to topic, webhook HTTP POST, log to alert history - **Loop detection**: `__source` marker prevents feedback loops, per-rule rate limiting (10/min default), global circuit breaker (100/min) - **Time-based rules**: APScheduler with cron expressions (e.g., "every 5 minutes") - **Hot-reload**: Rule changes take effect immediately without app restart - **Dry-run testing**: Test rules against sample payloads before activating - **Rules REST API**: Full CRUD at `/api/v1/rules/` with enable/disable and dry-run endpoints #### Alerting & Notifications - **Telegram alerts**: First-class action type — enter bot token + chat ID, get alerts in Telegram with Markdown formatting - **Slack alerts**: First-class action type — enter incoming webhook URL, get alerts in Slack with mrkdwn formatting - **HTTP webhook delivery**: Generic webhook with customizable payload templates using `{{topic}}`, `{{payload}}`, `{{timestamp}}` variables - **Retry with exponential backoff**: 3 retries at 1s, 5s, 25s on server errors - **SSRF protection**: Blocks webhook URLs pointing to private/reserved addresses (RFC-1918, localhost, link-local) - **Alert deduplication**: Configurable cooldown window (5 min default) prevents alert storms - **Alert history**: Persistent record of all alerts with delivery status, viewable via API and UI #### Modern Frontend - **Alpine.js component architecture**: Reactive UI components replacing vanilla JS - **htmx interactions**: Form submissions, CRUD operations, and pagination without full page reloads - **Socket.IO batching**: Server-side 100ms batch window handles 1000+ msg/sec without browser flooding - **Rules Editor UI**: Create, edit, delete, enable/disable, and dry-run test rules inline - **Alert History panel**: Filterable by rule, severity, and time range with pagination - **Tab navigation**: Dashboard / Rules / Alerts / Analytics / Plugins tabs - **Favorites toggle**: Filter message list to show only bookmarked topics - **Advanced Search**: Filter messages by topic, content, regex, JSON path, and time range - **Redesigned message rate chart**: Gradient area chart with 30-second rolling window and live msg/s counter - **Demo script**: `./demo.sh` populates realistic test data across all features #### REST API & Authentication - **Versioned API**: All endpoints under `/api/v1/` prefix with consistent JSON envelope responses - **OpenAPI documentation**: Swagger UI at `/api/v1/docs` with full endpoint documentation - **User authentication**: Flask-Login with username/password, session cookies - **API tokens**: `X-API-Key` header for programmatic access, token CRUD endpoints - **Rate limiting**: Configurable per-IP limit on publish endpoint (30/min default) - **CORS support**: Cross-origin API access for external consumers - **SECRET_KEY guard**: Application refuses to start with insecure key in production #### Analytics & Observability - **Per-topic analytics**: Message rate counters and numeric payload histograms - **Analytics dashboard**: Real-time top-topics-by-rate widget with histogram drill-down - **Structured logging**: structlog with JSON output (production) and colored console (development) - **Prometheus metrics**: `/metrics` endpoint with message counters, rule firing rates, connection gauges - **Topic favorites**: Star/bookmark topics for quick access - **Retained message indicator**: Visual "R" badge on retained messages #### Plugin Architecture - **Plugin hook specification**: `MQTTUIPlugin` base class with `on_message`, `on_connect`, `on_rule_trigger` hooks via pluggy - **Subprocess isolation**: Plugins run in separate processes with empty environment — no access to app, database, or MQTT client - **Entry-point discovery**: Standard Python packaging via `importlib.metadata` entry_points - **Plugin management UI**: View installed plugins, enable/disable via toggle - **Bundled examples**: JSON Formatter and Topic Logger plugins included ### Changed #### Architecture Overhaul - **Flask application factory**: Monolithic `app.py` refactored into `mqttui/` package with blueprints - **gevent async mode**: Replaced unmaintained eventlet with gevent for WebSocket support - **paho-mqtt 2.x**: Upgraded to modern `CallbackAPIVersion.VERSION2` callback API - **Flask 3.1.x**: Upgraded from Flask 2.0.1 with compatible Werkzeug - **SQLite WAL mode**: Write-ahead logging with `busy_timeout` on all connections for concurrent access - **Blinker event bus**: Internal signals (`mqtt_message_received`, `rule_fired`, `alert_triggered`) decouple all components - **Python 3.11**: Docker base image upgraded from 3.9 - **Tailwind CSS v4**: Standalone CLI replacing CDN v2.x (no Node.js dependency) #### Testing - **218+ automated tests**: pytest infrastructure with shared fixtures across all 7 phases - **Test categories**: App factory, database, routes, rules evaluator, rules engine, rules API, auth, API v1, webhook, cooldown, alerts API, frontend partials, analytics, observability, UX features, plugin registry, plugin runner, plugins API ### New Environment Variables - `MQTTUI_ADMIN_USER`: Admin username (default: admin) - `MQTTUI_ADMIN_PASSWORD`: Admin password (required for first run) - `MQTTUI_RATE_LIMIT`: Publish rate limit (default: 30/minute) - `SECRET_KEY`: Flask secret key (required in production, must not be "dev" or "change-me") ## [1.3.2] - 2025-08-24 ### Added - **Complete UI Redesign**: Collapsible sidebar layout for maximum screen real estate utilization - **Advanced Search Panel**: Comprehensive message filtering with regex patterns, JSON path queries, content search, and time-based filters - **Message Persistence**: SQLite database storage with automatic cleanup and configurable message limits - **Filter Presets**: Save and load frequently used search filter combinations - **Network Visualization Enhancements**: - Node pinning functionality (double-click to pin/unpin nodes, red color indicates pinned) - Improved physics engine with overlap prevention - Fullscreen support for Message Flow diagram - Right-click context menu for node management - **Collapsible Sidebar**: Hide/show controls panel to maximize Message Flow and Messages viewing area - **Enhanced API Endpoints**: - `/api/messages` with advanced filtering support - `/api/topics` with statistics - `/api/filter-presets` for preset management ### Changed - **Layout Architecture**: Complete redesign from grid-based to flexbox sidebar layout - **Message Flow Area**: Now takes up significantly more screen space (up to 100% width when sidebar hidden) - **Message Rate Chart**: Moved to compact sidebar position for better space utilization - **Topic Filtering**: Now actually filters displayed messages instead of just clearing the list - **Advanced Search**: Collapsible panel, closed by default to reduce visual clutter - **Screen Space Optimization**: Removed unnecessary padding, maximized content area utilization ### Fixed - **Topic Dropdown Functionality**: Fixed issue where topic selection only cleared messages instead of filtering - **Message Stacking**: Resolved overlapping div issue in the main content area - **Layout Responsiveness**: Proper flexbox implementation ensures consistent 50/50 split between Message Flow and Messages - **Database Thread Safety**: Implemented thread-local connections for concurrent message storage ### Technical Improvements - **Database Schema**: Enhanced with indexes for better query performance - **Message Filtering**: Support for regex topic patterns, JSON path queries, and content search - **Network Physics**: Improved node positioning with `avoidOverlap` and better stabilization - **JavaScript Architecture**: Modular functions for filter management and UI interactions - **CSS Layout**: Modern flexbox implementation with smooth transitions and animations ## [1.3.1] - 2024-08-24 ### Added - LOG_LEVEL environment variable support for controlling application logging verbosity (#9) - Topic subscription filtering via MQTT_TOPICS environment variable (#6) - Multi-architecture Docker support (AMD64, ARM64, ARMv7) with new Dockerfile.multiarch (#3) - GitHub Actions workflow for automated multi-platform Docker builds - Enhanced documentation for MQTT broker address format (#7) - Proper Flask-SocketIO server startup when running app.py directly (#12) ### Fixed - MQTT v5 connection error by adding properties parameter to on_connect callback (#8) - Application no longer exits prematurely when run outside Docker (#12) - Improved LOG_LEVEL validation in entrypoint.sh with gunicorn support ### Changed - Enhanced logging configuration with better formatting and level control - Updated README with comprehensive configuration documentation - Improved error handling for MQTT v5 connections ## [1.3.0] - 2024-08-27 ### Added - Improved logging for MQTT connection attempts and status - Client ID specification for MQTT connection - Support for different MQTT protocol versions ### Changed - Modified app.py to ensure MQTT connection is attempted when run in a container - Refactored server startup process for better compatibility with both development and production environments ### Fixed - Resolved issues with MQTT connection not being established - Fixed problems related to username/password authentication for MQTT brokers - Improved error handling for non-UTF-8 encoded MQTT messages ## [1.2.0] - 2024-08-24 ### Added - Debug Bar feature for enhanced developer insights - Real-time websocket connection/disconnect status - MQTT connection status and last message details - Request duration tracking - Toggle functionality to show/hide the Debug Bar ## [1.0.0] - 2024-08-19 ### Added - Initial release of MQTT Web Interface - Real-time visualization of MQTT topic hierarchy and message flow - Ability to publish messages to MQTT topics - Display of message statistics (connection count, topic count, message count) - Interactive network graph showing topic relationships - Docker support for easy deployment ================================================ FILE: Dockerfile ================================================ FROM python:3.11-slim WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY . . COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] ================================================ FILE: Dockerfile.multiarch ================================================ # Multi-architecture Dockerfile supporting both AMD64 and ARM64 (fixes issue #3) FROM --platform=$TARGETPLATFORM python:3.11-slim # Set build arguments for multi-platform builds ARG TARGETPLATFORM ARG BUILDPLATFORM RUN echo "Building on $BUILDPLATFORM for $TARGETPLATFORM" WORKDIR /app # Install build dependencies for compiling packages on ARM platforms RUN apt-get update && apt-get install -y \ build-essential \ gcc \ g++ \ libc6-dev \ libffi-dev \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt . RUN pip install --no-cache-dir --upgrade pip setuptools wheel && \ pip install --no-cache-dir -r requirements.txt COPY . . # Use an entrypoint script to allow for variable substitution COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] ================================================ FILE: LICENSE.md ================================================ MIT License Copyright (c) [2024] [Terry Osayawe] Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # MQTTUI — Intelligent MQTT Web Interface An open-source web application that monitors, visualizes, and **automates** your MQTT infrastructure. Create automation rules, get webhook alerts, track per-topic analytics, and extend with plugins — all from a single interface. ![Message Flow Screenshot](static/screenshot.png) ## What's New in v2.1 - **Multi-Broker Support** — Connect to multiple MQTT brokers simultaneously, manage from the Brokers tab, filter messages by broker - **Telegram & Slack Alerts** — First-class action types in the rules editor, no raw webhook URLs needed ## What's New in v2.0 - **Automation Rules Engine** — Create IF/THEN rules: when a topic matches a pattern and payload meets a condition, automatically publish, fire a webhook, or log an alert - **Alerting** — Telegram, Slack, and HTTP webhook notifications with retry/backoff, SSRF protection, and alert deduplication - **Modern UI** — Alpine.js + htmx component architecture with tabbed navigation (Dashboard / Rules / Alerts / Analytics / Plugins / Brokers) - **REST API** — Versioned `/api/v1/` endpoints with OpenAPI docs at `/api/v1/docs` - **User Authentication** — Login with username/password, API tokens for programmatic access - **Analytics** — Per-topic message rates, payload histograms, Prometheus `/metrics` endpoint - **Plugin Architecture** — Extend with custom handlers running in subprocess isolation - **218+ Automated Tests** — Full pytest suite across all features ## Features ### Core - **Multi-broker** — connect to multiple MQTT brokers simultaneously, filter messages by broker - Real-time MQTT message streaming via Socket.IO (handles 1000+ msg/sec with server-side batching) - Interactive topic hierarchy network graph (Vis.js) - Publish messages to any topic - SQLite message persistence with advanced search (regex, JSON path, time range) - Filter presets — save and load search combinations - MQTT v3.1.1 and v5 protocol support - Docker deployment with multi-arch support (AMD64, ARM64, ARMv7) ### Automation - **Rules Engine**: 11 condition operators (eq, gt, contains, regex, exists, etc.) with compound all/any logic - **JSON Path Conditions**: Match nested payload fields like `sensors.outdoor.temp > 30` - **Actions**: Publish to topic, HTTP webhook, log alert - **Loop Detection**: `__source` marker + per-rule rate limiting + global circuit breaker - **Time-Based Rules**: Cron schedules via APScheduler (e.g., publish heartbeat every 5 minutes) - **Dry-Run Testing**: Test rules against sample payloads before activating - **Hot-Reload**: Rule changes take effect immediately, no restart needed ### Alerting & Notifications - **Telegram**: Enter bot token + chat ID — get alerts directly in Telegram - **Slack**: Enter incoming webhook URL — get alerts in your Slack channel - **HTTP Webhooks**: Generic webhook with customizable payload templates (`{{topic}}`, `{{payload}}`, `{{timestamp}}`) - Retry with exponential backoff (1s / 5s / 25s) on server errors - SSRF protection — blocks private/reserved addresses - Alert deduplication with configurable cooldown (5 min default) - Persistent alert history with delivery status tracking ### Analytics & Observability - Per-topic message rate counters and numeric payload histograms - Structured JSON logging (structlog) for production - Prometheus-compatible `/metrics` endpoint - Topic favorites/bookmarks for quick access - Retained message indicator ### Plugin System - `MQTTUIPlugin` base class with `on_message`, `on_connect`, `on_rule_trigger` hooks - Subprocess isolation — plugins cannot access app internals - Python entry-point discovery (`pip install` your plugin) - Management UI with enable/disable toggles - Bundled examples: JSON Formatter, Topic Logger ## Quick Start ### Docker Compose (Recommended) ```bash git clone https://github.com/terdia/mqttui.git cd mqttui docker compose up -d ``` This starts: - Mosquitto MQTT broker on port 1883 - MQTTUI on port 8088 Open `http://localhost:8088` and log in with the admin credentials you set. ### Docker ```bash docker pull terdia07/mqttui:v2.0 docker run -p 8088:5000 \ -e MQTT_BROKER=your_broker \ -e MQTTUI_ADMIN_USER=admin \ -e MQTTUI_ADMIN_PASSWORD=changeme \ -e SECRET_KEY=your-secret-key \ terdia07/mqttui:v2.0 ``` ### Manual Installation ```bash git clone https://github.com/terdia/mqttui.git cd mqttui pip install -r requirements.txt # Set required environment variables export MQTTUI_ADMIN_USER=admin export MQTTUI_ADMIN_PASSWORD=changeme export SECRET_KEY=your-secret-key export MQTT_BROKER=localhost python wsgi.py ``` ### Running Tests ```bash pip install -r requirements.txt pytest tests/ -q ``` ### Demo Data Populate realistic test data across all features (requires Docker Compose running): ```bash ./demo.sh ``` This creates 200+ messages, 4 automation rules, triggers alerts, sets up filter presets, and bookmarks topics. Great for evaluating all tabs and features. ## Configuration ### Required Environment Variables | Variable | Description | Default | |----------|-------------|---------| | `MQTTUI_ADMIN_USER` | Admin username | `admin` | | `MQTTUI_ADMIN_PASSWORD` | Admin password | *(required)* | | `SECRET_KEY` | Flask secret key (must not be "dev" in production) | *(required)* | ### MQTT Connection | Variable | Description | Default | |----------|-------------|---------| | `MQTT_BROKER` | Broker address (IP or hostname, no protocol prefix) | `localhost` | | `MQTT_PORT` | Broker port | `1883` | | `MQTT_USERNAME` | Broker username | *(optional)* | | `MQTT_PASSWORD` | Broker password | *(optional)* | | `MQTT_KEEPALIVE` | Keep-alive interval (seconds) | `60` | | `MQTT_VERSION` | Protocol version (`3.1.1` or `5`) | `3.1.1` | | `MQTT_TOPICS` | Topics to subscribe (comma-separated) | `#` | ### TLS/SSL (MQTTS) | Variable | Description | Default | |----------|-------------|---------| | `MQTT_TLS` | Enable TLS connection | `false` | | `MQTT_TLS_CA_CERTS` | Path to CA certificate file | *(optional)* | | `MQTT_TLS_CERTFILE` | Path to client certificate | *(optional)* | | `MQTT_TLS_KEYFILE` | Path to client private key | *(optional)* | | `MQTT_TLS_INSECURE` | Skip certificate verification (not recommended) | `false` | **Example — connect to a TLS broker:** ```bash docker run -p 8088:5000 \ -e MQTT_BROKER=broker.hivemq.com \ -e MQTT_PORT=8883 \ -e MQTT_TLS=true \ terdia07/mqttui:v2.0.0 ``` ### Application | Variable | Description | Default | |----------|-------------|---------| | `DEBUG` | Enable development mode | `False` | | `PORT` | Application port | `5000` | | `LOG_LEVEL` | Logging level (DEBUG/INFO/WARNING/ERROR) | `INFO` | | `MQTTUI_RATE_LIMIT` | Publish endpoint rate limit | `30/minute` | ### Database | Variable | Description | Default | |----------|-------------|---------| | `DB_ENABLED` | Enable message persistence | `True` | | `DB_PATH` | SQLite database path | `./data/mqtt_messages.db` | | `DB_MAX_MESSAGES` | Max stored messages | `10000` | | `DB_CLEANUP_DAYS` | Auto-delete messages older than X days | `30` | ## API Documentation All API endpoints are under `/api/v1/` with OpenAPI documentation at `/api/v1/docs`. ### Key Endpoints | Method | Endpoint | Description | |--------|----------|-------------| | `GET` | `/api/v1/messages` | Message history with filtering | | `POST` | `/api/v1/publish` | Publish MQTT message | | `GET` | `/api/v1/topics` | Topic list with stats | | `GET/POST` | `/api/v1/rules/` | List / create automation rules | | `PUT/DELETE` | `/api/v1/rules/` | Update / delete rule | | `POST` | `/api/v1/rules//test` | Dry-run test a rule | | `POST` | `/api/v1/rules//enable` | Enable rule | | `GET` | `/api/v1/alerts/` | Alert history | | `GET` | `/api/v1/analytics/topics` | Per-topic analytics | | `GET` | `/api/v1/plugins/` | List plugins | | `GET` | `/metrics` | Prometheus metrics | All data endpoints require authentication (session cookie or `X-API-Key` header). ## Tech Stack - **Backend**: Python 3.11, Flask 3.1.x, Flask-SocketIO + gevent, paho-mqtt 2.1.0 - **Database**: SQLite with WAL mode - **Frontend**: Alpine.js 3.x, htmx 2.x, Tailwind CSS v4, Vis.js, Chart.js - **Real-time**: Socket.IO with server-side 100ms batching - **Scheduling**: APScheduler with GeventScheduler - **Plugins**: pluggy + subprocess isolation - **Observability**: structlog + prometheus_client ## Contributing 1. Fork the repository 2. Create a feature branch (`git checkout -b feature/amazing-feature`) 3. Run tests (`pytest tests/ -q`) 4. Commit your changes 5. Push and open a Pull Request ## License MIT License — see [LICENSE.md](LICENSE.md) for details. ## Links - [GitHub Repository](https://github.com/terdia/mqttui) - [Docker Hub](https://hub.docker.com/r/terdia07/mqttui) - [Changelog](CHANGELOG.md) ================================================ FILE: app.py ================================================ __version__ = "1.3.0" from flask import Flask, render_template, request, jsonify, send_from_directory from flask_socketio import SocketIO, emit import paho.mqtt.client as mqtt from datetime import datetime, timedelta import os from debug_bar import debug_bar, debug_bar_middleware import logging import time from dotenv import load_dotenv from logging.handlers import RotatingFileHandler from werkzeug.serving import run_simple from database import MessageDatabase # Load environment variables load_dotenv() # Configuration DEBUG = os.getenv('DEBUG', 'False').lower() in ('true', '1', 't') HOST = os.getenv('HOST', '0.0.0.0') PORT = int(os.getenv('PORT', 5000)) MQTT_BROKER = os.getenv('MQTT_BROKER', 'localhost') MQTT_PORT = int(os.getenv('MQTT_PORT', 1883)) MQTT_USERNAME = os.getenv('MQTT_USERNAME') MQTT_PASSWORD = os.getenv('MQTT_PASSWORD') MQTT_KEEPALIVE = int(os.getenv('MQTT_KEEPALIVE', 60)) MQTT_VERSION = os.getenv('MQTT_VERSION', '3.1.1') # Support for topic filtering (issue #6) MQTT_TOPICS = os.getenv('MQTT_TOPICS', '#') # Comma-separated list of topics to subscribe to # Database configuration for message persistence DB_ENABLED = os.getenv('DB_ENABLED', 'True').lower() in ('true', '1', 't') DB_PATH = os.getenv('DB_PATH', 'mqtt_messages.db') DB_MAX_MESSAGES = int(os.getenv('DB_MAX_MESSAGES', 10000)) DB_CLEANUP_DAYS = int(os.getenv('DB_CLEANUP_DAYS', 30)) # Set up logging with LOG_LEVEL environment variable support (fixes issue #9) LOG_LEVEL = os.getenv('LOG_LEVEL', 'DEBUG' if DEBUG else 'INFO').upper() log_levels = { 'DEBUG': logging.DEBUG, 'INFO': logging.INFO, 'WARNING': logging.WARNING, 'WARN': logging.WARNING, # Support both WARN and WARNING 'ERROR': logging.ERROR, 'CRITICAL': logging.CRITICAL } log_level = log_levels.get(LOG_LEVEL, logging.INFO) if LOG_LEVEL not in log_levels: print(f"Invalid LOG_LEVEL: {LOG_LEVEL}. Using INFO level.") logging.basicConfig(level=log_level, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') if not DEBUG: handler = RotatingFileHandler('mqttui.log', maxBytes=10000, backupCount=1) handler.setLevel(log_level) handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) logging.getLogger('').addHandler(handler) app = Flask(__name__, static_url_path='/static') app.config['SECRET_KEY'] = os.getenv('SECRET_KEY', 'your-secret-key') socketio = SocketIO(app, async_mode='threading') # Initialize database for message persistence db = None if DB_ENABLED: try: db = MessageDatabase(DB_PATH, DB_MAX_MESSAGES) logging.info(f"Database initialized: {DB_PATH}") except Exception as e: logging.error(f"Failed to initialize database: {e}") db = None MQTT_RC_CODES = { 0: "Connection successful", 1: "Connection refused - incorrect protocol version", 2: "Connection refused - invalid client identifier", 3: "Connection refused - server unavailable", 4: "Connection refused - bad username or password", 5: "Connection refused - not authorised", # MQTT v5 specific codes 16: "Connection refused - no matching subscribers", 17: "Connection refused - no subscription existed", 128: "Connection refused - unspecified error", 129: "Connection refused - malformed packet", 130: "Connection refused - protocol error", 131: "Connection refused - implementation specific error", 132: "Connection refused - unsupported protocol version", 133: "Connection refused - client identifier not valid", 134: "Connection refused - bad user name or password", 135: "Connection refused - not authorized", 136: "Connection refused - server unavailable", 137: "Connection refused - server busy", 138: "Connection refused - banned", 139: "Connection refused - server shutting down", 140: "Connection refused - bad authentication method", 141: "Connection refused - topic name invalid", 142: "Connection refused - packet too large", 143: "Connection refused - quota exceeded", 144: "Connection refused - payload format invalid", 145: "Connection refused - retain not supported", 146: "Connection refused - QoS not supported", 147: "Connection refused - use another server", 148: "Connection refused - server moved", 149: "Connection refused - connection rate exceeded" } app.before_request(debug_bar_middleware) @app.after_request def after_request(response): debug_bar.record('request', 'status_code', response.status_code) debug_bar.end_request() return response # MQTT setup mqtt_version = os.getenv('MQTT_VERSION', '3.1.1') if mqtt_version == '5': mqtt_client = mqtt.Client(client_id=f"mqttui_{os.getpid()}", protocol=mqtt.MQTTv5) logging.info("Using MQTT v5") else: mqtt_client = mqtt.Client(client_id=f"mqttui_{os.getpid()}", clean_session=True, protocol=mqtt.MQTTv311) logging.info("Using MQTT v3.1.1") mqtt_broker = os.getenv('MQTT_BROKER', 'localhost') mqtt_port = int(os.getenv('MQTT_PORT', 1883)) mqtt_username = os.getenv('MQTT_USERNAME') mqtt_password = os.getenv('MQTT_PASSWORD') mqtt_keepalive = int(os.getenv('MQTT_KEEPALIVE', 60)) logging.info(f"MQTT Setup - Broker: {mqtt_broker}, Port: {mqtt_port}, Username: {'Set' if mqtt_username else 'Not set'}, Password: {'Set' if mqtt_password else 'Not set'}, Version: {mqtt_version}") messages = [] topics = set() connection_count = 0 active_websockets = 0 error_log = [] @socketio.on('connect') def handle_connect(): global active_websockets active_websockets += 1 debug_bar.record('performance', 'active_websockets', active_websockets) logging.info(f"WebSocket connected. Total active: {active_websockets}") @socketio.on('disconnect') def handle_disconnect(): global active_websockets active_websockets -= 1 debug_bar.record('performance', 'active_websockets', active_websockets) logging.info(f"WebSocket disconnected. Total active: {active_websockets}") def on_connect(client, userdata, flags, rc, properties=None): # MQTT v5 includes 'properties' parameter, v3.1.1 doesn't (fixes issue #8) global connection_count error_message = MQTT_RC_CODES.get(rc, f"Unknown error (rc: {rc})") connection_status = 'Connected' if rc == 0 else f'Failed: {error_message}' debug_bar.record('mqtt', 'connection_status', connection_status) logging.info(f"MQTT Connection attempt - Result: {connection_status}") logging.info(f"MQTT Connection details - Broker: {mqtt_broker}, Port: {mqtt_port}, Username: {'Set' if mqtt_username else 'Not set'}, Password: {'Set' if mqtt_password else 'Not set'}, Protocol: MQTT v{mqtt_version}") if rc == 0: connection_count += 1 # Subscribe to specified topics (supports filtering - issue #6) topics_to_subscribe = [topic.strip() for topic in MQTT_TOPICS.split(',')] for topic in topics_to_subscribe: client.subscribe(topic) logging.info(f"Subscribed to topic: {topic}") logging.info(f"Connected to MQTT broker at {mqtt_broker}:{mqtt_port}. Total connections: {connection_count}") debug_bar.remove('mqtt', 'connection_attempt') # Remove connection attempt entry else: error_log.append(error_message) debug_bar.record('mqtt', 'last_error', error_message) logging.error(f"Connection failed: {error_message}") time.sleep(5) connect_mqtt() # Retry connection def on_disconnect(client, userdata, rc): global connection_count connection_count = max(0, connection_count - 1) error_message = MQTT_RC_CODES.get(rc, f"Unknown error (rc: {rc})") disconnect_reason = 'Clean disconnect' if rc == 0 else f'Unexpected disconnect: {error_message}' debug_bar.record('mqtt', 'last_disconnect', disconnect_reason) error_log.append(f"Disconnected: {disconnect_reason}") logging.warning(f"Disconnected from MQTT broker: {disconnect_reason}") if rc != 0: logging.info("Attempting to reconnect...") client.connect_async(mqtt_broker, mqtt_port, mqtt_keepalive) def on_message(client, userdata, msg): try: payload = msg.payload.decode() except UnicodeDecodeError: payload = msg.payload.hex() timestamp = datetime.now() message = { 'topic': msg.topic, 'payload': payload, 'timestamp': timestamp.isoformat() } # Store in memory for backward compatibility messages.append(message) topics.add(msg.topic) if len(messages) > 100: messages.pop(0) # Store in database if enabled if db: try: db.store_message( topic=msg.topic, payload=payload, timestamp=timestamp, qos=msg.qos, retain=msg.retain ) except Exception as e: logging.error(f"Failed to store message in database: {e}") # Emit to connected clients socketio.emit('mqtt_message', message) debug_bar.record('mqtt', 'last_message', message) logging.debug(f"MQTT message received: {message}") mqtt_client.on_connect = on_connect mqtt_client.on_message = on_message mqtt_client.on_disconnect = on_disconnect # API endpoints for message history @app.route('/api/messages') def get_message_history(): """Get paginated message history with enhanced filtering""" try: # Get query parameters limit = min(int(request.args.get('limit', 100)), 1000) # Max 1000 messages offset = int(request.args.get('offset', 0)) # Basic filters topic_filter = request.args.get('topic') hours = request.args.get('hours') # Messages from last N hours # Enhanced filters content_search = request.args.get('content') # Search in message content regex_topic = request.args.get('regex_topic') # Regex pattern for topic json_path = request.args.get('json_path') # JSON path (e.g., "temperature") json_value = request.args.get('json_value') # Expected value at JSON path since = None if hours: since = datetime.now() - timedelta(hours=int(hours)) if db: # Get from database with enhanced filtering messages_list = db.get_messages( limit=limit, offset=offset, topic_filter=topic_filter, since=since, content_search=content_search, regex_topic=regex_topic, json_path=json_path, json_value=json_value ) # Note: total_count doesn't account for Python filters, so it's approximate total_count = db.get_message_count(topic_filter=topic_filter, since=since) else: # Fallback to in-memory messages (basic filtering only) messages_list = list(reversed(messages)) # Most recent first if topic_filter: messages_list = [m for m in messages_list if m['topic'] == topic_filter] if content_search: messages_list = [m for m in messages_list if content_search.lower() in m['payload'].lower()] total_count = len(messages_list) messages_list = messages_list[offset:offset+limit] return jsonify({ 'messages': messages_list, 'total': total_count, 'limit': limit, 'offset': offset, 'has_more': offset + len(messages_list) < total_count, 'filters_applied': { 'topic': topic_filter, 'content': content_search, 'regex_topic': regex_topic, 'json_path': json_path, 'json_value': json_value, 'hours': hours } }) except Exception as e: logging.error(f"Error getting message history: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/topics') def get_topic_list(): """Get list of all topics with statistics""" try: if db: topics_list = db.get_topics() else: # Fallback to in-memory topics topics_list = [{'topic': topic} for topic in sorted(topics)] return jsonify({'topics': topics_list}) except Exception as e: logging.error(f"Error getting topics: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/database/stats') def get_database_stats(): """Get database size and statistics""" try: if db: stats = db.get_database_size() stats['enabled'] = True else: stats = { 'enabled': False, 'message_count': len(messages), 'topic_count': len(topics) } return jsonify(stats) except Exception as e: logging.error(f"Error getting database stats: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/database/cleanup', methods=['POST']) def cleanup_database(): """Clean up old database records""" try: if not db: return jsonify({'error': 'Database not enabled'}), 400 days = int(request.json.get('days', DB_CLEANUP_DAYS)) deleted = db.cleanup_old_data(days) return jsonify({ 'success': True, 'deleted_messages': deleted, 'days': days }) except Exception as e: logging.error(f"Error cleaning database: {e}") return jsonify({'error': str(e)}), 500 # Filter presets API endpoints @app.route('/api/filter-presets') def get_filter_presets(): """Get all saved filter presets""" try: if not db: return jsonify({'error': 'Database not enabled'}), 400 presets = db.get_filter_presets() return jsonify({'presets': presets}) except Exception as e: logging.error(f"Error getting filter presets: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/filter-presets', methods=['POST']) def save_filter_preset(): """Save a new filter preset""" try: if not db: return jsonify({'error': 'Database not enabled'}), 400 data = request.json name = data.get('name') description = data.get('description', '') filters = data.get('filters', {}) if not name or not filters: return jsonify({'error': 'Name and filters are required'}), 400 success = db.save_filter_preset(name, filters, description) if success: return jsonify({'success': True, 'message': f'Saved preset: {name}'}) else: return jsonify({'error': 'Failed to save preset'}), 500 except Exception as e: logging.error(f"Error saving filter preset: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/filter-presets/', methods=['DELETE']) def delete_filter_preset(name): """Delete a filter preset""" try: if not db: return jsonify({'error': 'Database not enabled'}), 400 success = db.delete_filter_preset(name) if success: return jsonify({'success': True, 'message': f'Deleted preset: {name}'}) else: return jsonify({'error': 'Preset not found'}), 404 except Exception as e: logging.error(f"Error deleting filter preset: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/filter-presets//use', methods=['POST']) def use_filter_preset(name): """Load and use a filter preset""" try: if not db: return jsonify({'error': 'Database not enabled'}), 400 filters = db.use_filter_preset(name) if filters: return jsonify({'success': True, 'filters': filters}) else: return jsonify({'error': 'Preset not found'}), 404 except Exception as e: logging.error(f"Error using filter preset: {e}") return jsonify({'error': str(e)}), 500 @app.route('/') def index(): return render_template('index.html', messages=messages, topics=list(topics)) @app.route('/publish', methods=['POST']) def publish_message(): topic = request.form['topic'] message = request.form['message'] mqtt_client.publish(topic, message) debug_bar.record('mqtt', 'last_publish', {'topic': topic, 'message': message}) return jsonify(success=True) @app.route('/stats') def get_stats(): return jsonify({ 'connection_count': connection_count, 'topic_count': len(topics), 'message_count': len(messages), 'errors': error_log }) @app.route('/static/') def send_static(path): return send_from_directory('static', path) @app.route('/debug-bar') def get_debug_bar_data(): try: data = debug_bar.get_data() return jsonify(data) except Exception as e: logging.error(f"Error fetching debug bar data: {e}") return jsonify({"error": "Failed to fetch debug bar data"}), 500 @app.route('/toggle-debug-bar', methods=['POST']) def toggle_debug_bar(): if debug_bar.enabled: debug_bar.disable() else: debug_bar.enable() return jsonify(enabled=debug_bar.enabled) @app.route('/record-client-performance', methods=['POST']) def record_client_performance(): data = request.json debug_bar.record('performance', 'page_load_time', f"{data['pageLoadTime']}ms") debug_bar.record('performance', 'dom_ready_time', f"{data['domReadyTime']}ms") return jsonify(success=True) @app.route('/version') def get_version(): return jsonify({'version': __version__}) if __name__ == '__main__' or __name__ == 'app': if mqtt_username and mqtt_password: mqtt_client.username_pw_set(mqtt_username, mqtt_password) def connect_mqtt(): if mqtt_username and mqtt_password: mqtt_client.username_pw_set(mqtt_username, mqtt_password) logging.info("MQTT credentials set") else: logging.info("No MQTT credentials provided") debug_bar.record('mqtt', 'connection_attempt', f"Connecting to {mqtt_broker}:{mqtt_port}") debug_bar.record('mqtt', 'broker', mqtt_broker) debug_bar.record('mqtt', 'port', mqtt_port) debug_bar.record('mqtt', 'username', mqtt_username if mqtt_username else 'Not set') debug_bar.record('mqtt', 'password', 'Set' if mqtt_password else 'Not set') debug_bar.record('mqtt', 'protocol', f'MQTT v{mqtt_version}') debug_bar.record('mqtt', 'subscribed_topics', MQTT_TOPICS) logging.info(f"Attempting to connect to MQTT broker at {mqtt_broker}:{mqtt_port}") try: mqtt_client.connect(mqtt_broker, mqtt_port, mqtt_keepalive) mqtt_client.loop_start() except Exception as e: error_message = f"Failed to connect to MQTT broker: {str(e)}" debug_bar.record('mqtt', 'connection_error', error_message) debug_bar.record('mqtt', 'connection_status', 'Failed') error_log.append(error_message) logging.error(error_message) time.sleep(5) connect_mqtt() # Retry connection debug_bar.record('mqtt', 'connection_status', 'Failed') connect_mqtt() # Start the Flask-SocketIO server when running directly # This prevents the app from exiting prematurely (fixes issue #12) if __name__ == '__main__': socketio.run(app, host=HOST, port=PORT, debug=DEBUG) ================================================ FILE: database.py ================================================ """ Database module for MQTT message persistence """ import sqlite3 import threading import logging import re import json from datetime import datetime, timedelta from typing import List, Dict, Optional, Union import os class MessageDatabase: def __init__(self, db_path: str = "mqtt_messages.db", max_messages: int = 10000): self.db_path = db_path self.max_messages = max_messages self._local = threading.local() self.init_database() def get_connection(self): """Get thread-local database connection""" if not hasattr(self._local, 'connection'): self._local.connection = sqlite3.connect( self.db_path, check_same_thread=False, timeout=30.0 ) self._local.connection.row_factory = sqlite3.Row # Add REGEXP function for advanced topic filtering self._local.connection.create_function("REGEXP", 2, self._regexp) return self._local.connection def _regexp(self, pattern, value): """Custom REGEXP function for SQLite""" try: return re.search(pattern, value, re.IGNORECASE) is not None except Exception: return False def init_database(self): """Initialize database schema""" conn = self.get_connection() try: cursor = conn.cursor() # Messages table cursor.execute(''' CREATE TABLE IF NOT EXISTS messages ( id INTEGER PRIMARY KEY AUTOINCREMENT, topic TEXT NOT NULL, payload TEXT NOT NULL, timestamp DATETIME NOT NULL, qos INTEGER DEFAULT 0, retain BOOLEAN DEFAULT 0, payload_size INTEGER DEFAULT 0, created_at DATETIME DEFAULT CURRENT_TIMESTAMP ) ''') # Topics table for faster topic queries cursor.execute(''' CREATE TABLE IF NOT EXISTS topics ( topic TEXT PRIMARY KEY, last_message_at DATETIME NOT NULL, message_count INTEGER DEFAULT 1, first_seen DATETIME DEFAULT CURRENT_TIMESTAMP ) ''') # Filter presets table for saved searches cursor.execute(''' CREATE TABLE IF NOT EXISTS filter_presets ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT UNIQUE NOT NULL, description TEXT, filters TEXT NOT NULL, -- JSON string of filter parameters created_at DATETIME DEFAULT CURRENT_TIMESTAMP, last_used DATETIME ) ''') # Create indexes for better performance cursor.execute(''' CREATE INDEX IF NOT EXISTS idx_messages_topic ON messages(topic) ''') cursor.execute(''' CREATE INDEX IF NOT EXISTS idx_messages_timestamp ON messages(timestamp DESC) ''') cursor.execute(''' CREATE INDEX IF NOT EXISTS idx_messages_topic_timestamp ON messages(topic, timestamp DESC) ''') conn.commit() logging.info("Database initialized successfully") except Exception as e: logging.error(f"Database initialization error: {e}") conn.rollback() raise def store_message(self, topic: str, payload: str, timestamp: datetime = None, qos: int = 0, retain: bool = False) -> bool: """Store a message in the database""" if timestamp is None: timestamp = datetime.now() conn = self.get_connection() try: cursor = conn.cursor() # Store message cursor.execute(''' INSERT INTO messages (topic, payload, timestamp, qos, retain, payload_size) VALUES (?, ?, ?, ?, ?, ?) ''', (topic, payload, timestamp, qos, retain, len(payload))) # Update topics table cursor.execute(''' INSERT OR REPLACE INTO topics (topic, last_message_at, message_count, first_seen) VALUES (?, ?, COALESCE((SELECT message_count FROM topics WHERE topic = ?) + 1, 1), COALESCE((SELECT first_seen FROM topics WHERE topic = ?), ?) ) ''', (topic, timestamp, topic, topic, timestamp)) conn.commit() # Clean up old messages if needed self._cleanup_old_messages() return True except Exception as e: logging.error(f"Error storing message: {e}") conn.rollback() return False def get_messages(self, limit: int = 100, offset: int = 0, topic_filter: str = None, since: datetime = None, content_search: str = None, regex_topic: str = None, json_path: str = None, json_value: str = None) -> List[Dict]: """Retrieve messages from database with enhanced filtering""" conn = self.get_connection() try: cursor = conn.cursor() query = ''' SELECT topic, payload, timestamp, qos, retain, payload_size FROM messages WHERE 1=1 ''' params = [] # Basic topic filtering (wildcard support) if topic_filter: if '%' in topic_filter or '_' in topic_filter: query += ' AND topic LIKE ?' else: query += ' AND topic = ?' params.append(topic_filter) # Regex topic filtering (more powerful than LIKE) if regex_topic: query += ' AND topic REGEXP ?' params.append(regex_topic) # Content search (case-insensitive) if content_search: query += ' AND LOWER(payload) LIKE LOWER(?)' params.append(f'%{content_search}%') # Time filtering if since: query += ' AND timestamp >= ?' params.append(since) query += ' ORDER BY timestamp DESC LIMIT ? OFFSET ?' params.extend([limit, offset]) cursor.execute(query, params) rows = cursor.fetchall() messages = [dict(row) for row in rows] # Apply Python-based filters that can't be done in SQL if json_path or regex_topic: messages = self._apply_python_filters( messages, regex_topic, json_path, json_value ) return messages except Exception as e: logging.error(f"Error retrieving messages: {e}") return [] def _apply_python_filters(self, messages: List[Dict], regex_topic: str = None, json_path: str = None, json_value: str = None) -> List[Dict]: """Apply filters that require Python processing""" filtered = [] for msg in messages: # Regex topic filter (if not handled by SQL REGEXP) if regex_topic and not re.search(regex_topic, msg['topic'], re.IGNORECASE): continue # JSON path filtering if json_path and json_value: try: payload_json = json.loads(msg['payload']) # Simple JSON path support (e.g., "temperature", "sensors.temp") value = self._get_json_path_value(payload_json, json_path) if value is None or str(value).lower() != json_value.lower(): continue except (json.JSONDecodeError, KeyError): continue filtered.append(msg) return filtered def _get_json_path_value(self, json_obj: dict, path: str): """Extract value from JSON using simple dot notation path""" try: keys = path.split('.') current = json_obj for key in keys: if isinstance(current, dict) and key in current: current = current[key] else: return None return current except Exception: return None def get_topics(self) -> List[Dict]: """Get all topics with statistics""" conn = self.get_connection() try: cursor = conn.cursor() cursor.execute(''' SELECT topic, message_count, last_message_at, first_seen FROM topics ORDER BY last_message_at DESC ''') rows = cursor.fetchall() return [dict(row) for row in rows] except Exception as e: logging.error(f"Error retrieving topics: {e}") return [] def get_message_count(self, topic_filter: str = None, since: datetime = None) -> int: """Get total message count with optional filtering""" conn = self.get_connection() try: cursor = conn.cursor() query = 'SELECT COUNT(*) FROM messages WHERE 1=1' params = [] if topic_filter: if '%' in topic_filter or '_' in topic_filter: query += ' AND topic LIKE ?' else: query += ' AND topic = ?' params.append(topic_filter) if since: query += ' AND timestamp >= ?' params.append(since) cursor.execute(query, params) return cursor.fetchone()[0] except Exception as e: logging.error(f"Error counting messages: {e}") return 0 def _cleanup_old_messages(self): """Remove old messages to stay within limit""" conn = self.get_connection() try: cursor = conn.cursor() # Check if we need cleanup cursor.execute('SELECT COUNT(*) FROM messages') count = cursor.fetchone()[0] if count > self.max_messages: # Delete oldest messages messages_to_delete = count - self.max_messages + 1000 # Delete extra to avoid frequent cleanups cursor.execute(''' DELETE FROM messages WHERE id IN ( SELECT id FROM messages ORDER BY timestamp ASC LIMIT ? ) ''', (messages_to_delete,)) # Update topic counts (this is approximate) cursor.execute(''' UPDATE topics SET message_count = ( SELECT COUNT(*) FROM messages WHERE messages.topic = topics.topic ) ''') conn.commit() logging.info(f"Cleaned up {messages_to_delete} old messages") except Exception as e: logging.error(f"Error during cleanup: {e}") conn.rollback() def cleanup_old_data(self, days: int = 30): """Remove messages older than specified days""" cutoff_date = datetime.now() - timedelta(days=days) conn = self.get_connection() try: cursor = conn.cursor() cursor.execute('DELETE FROM messages WHERE timestamp < ?', (cutoff_date,)) deleted = cursor.rowcount # Clean up topics with no messages cursor.execute(''' DELETE FROM topics WHERE topic NOT IN ( SELECT DISTINCT topic FROM messages ) ''') conn.commit() logging.info(f"Cleaned up {deleted} messages older than {days} days") return deleted except Exception as e: logging.error(f"Error during data cleanup: {e}") conn.rollback() return 0 def get_database_size(self) -> Dict: """Get database size information""" try: size_bytes = os.path.getsize(self.db_path) if os.path.exists(self.db_path) else 0 conn = self.get_connection() cursor = conn.cursor() cursor.execute('SELECT COUNT(*) FROM messages') message_count = cursor.fetchone()[0] cursor.execute('SELECT COUNT(*) FROM topics') topic_count = cursor.fetchone()[0] return { 'size_bytes': size_bytes, 'size_mb': round(size_bytes / 1024 / 1024, 2), 'message_count': message_count, 'topic_count': topic_count } except Exception as e: logging.error(f"Error getting database size: {e}") return {'size_bytes': 0, 'size_mb': 0, 'message_count': 0, 'topic_count': 0} def save_filter_preset(self, name: str, filters: Dict, description: str = None) -> bool: """Save a filter preset""" conn = self.get_connection() try: cursor = conn.cursor() cursor.execute(''' INSERT OR REPLACE INTO filter_presets (name, description, filters, last_used) VALUES (?, ?, ?, ?) ''', (name, description, json.dumps(filters), datetime.now())) conn.commit() logging.info(f"Saved filter preset: {name}") return True except Exception as e: logging.error(f"Error saving filter preset: {e}") conn.rollback() return False def get_filter_presets(self) -> List[Dict]: """Get all filter presets""" conn = self.get_connection() try: cursor = conn.cursor() cursor.execute(''' SELECT name, description, filters, created_at, last_used FROM filter_presets ORDER BY last_used DESC, created_at DESC ''') rows = cursor.fetchall() presets = [] for row in rows: preset = dict(row) preset['filters'] = json.loads(preset['filters']) presets.append(preset) return presets except Exception as e: logging.error(f"Error getting filter presets: {e}") return [] def delete_filter_preset(self, name: str) -> bool: """Delete a filter preset""" conn = self.get_connection() try: cursor = conn.cursor() cursor.execute('DELETE FROM filter_presets WHERE name = ?', (name,)) conn.commit() if cursor.rowcount > 0: logging.info(f"Deleted filter preset: {name}") return True return False except Exception as e: logging.error(f"Error deleting filter preset: {e}") conn.rollback() return False def use_filter_preset(self, name: str) -> Optional[Dict]: """Load and mark a filter preset as used""" conn = self.get_connection() try: cursor = conn.cursor() # Get the preset cursor.execute(''' SELECT filters FROM filter_presets WHERE name = ? ''', (name,)) row = cursor.fetchone() if not row: return None # Update last_used timestamp cursor.execute(''' UPDATE filter_presets SET last_used = ? WHERE name = ? ''', (datetime.now(), name)) conn.commit() return json.loads(row['filters']) except Exception as e: logging.error(f"Error using filter preset: {e}") return None def close(self): """Close database connection""" if hasattr(self._local, 'connection'): self._local.connection.close() ================================================ FILE: debug_bar.py ================================================ import time import psutil from flask import request from threading import Lock import logging class DebugBarPanel: def __init__(self, name): self.name = name self.data = {} def record(self, key, value): self.data[key] = value def get_data(self): return self.data class DebugBar: def __init__(self): self.panels = {} self.enabled = False self.start_time = None self.lock = Lock() try: self.process = psutil.Process() except Exception as e: logging.error(f"Failed to initialize psutil Process: {e}") self.process = None def add_panel(self, name): with self.lock: if name not in self.panels: self.panels[name] = DebugBarPanel(name) def record(self, panel, key, value): with self.lock: if panel in self.panels: self.panels[panel].record(key, value) def start_request(self): self.start_time = time.time() def end_request(self): if self.start_time: duration = time.time() - self.start_time self.record('request', 'duration', f"{duration:.2f}s") self.start_time = None def get_data(self): with self.lock: #self.update_performance_metrics() return {name: panel.get_data() for name, panel in self.panels.items()} def enable(self): self.enabled = True def disable(self): self.enabled = False def remove(self, panel_name, key): with self.lock: if panel_name in self.panels: panel = self.panels[panel_name] if key in panel.data: del panel.data[key] debug_bar = DebugBar() # Initialize default panels debug_bar.add_panel('mqtt') debug_bar.add_panel('request') debug_bar.add_panel('performance') def debug_bar_middleware(): debug_bar.start_request() debug_bar.record('request', 'path', request.path) debug_bar.record('request', 'method', request.method) ================================================ FILE: demo.sh ================================================ #!/bin/bash # ============================================================================= # MQTTUI v2.1 Demo Script # Populates realistic MQTT data across multiple brokers to test all features # Usage: ./demo.sh # Requires: docker compose running (mosquitto + mosquitto2 + mqttui containers) # ============================================================================= set -e BROKER1="mosquitto" BROKER2="mosquitto2" API_URL="http://localhost:8088/api/v1" PUB1="docker exec $BROKER1 mosquitto_pub" PUB2="docker exec $BROKER2 mosquitto_pub" # Colors GREEN='\033[0;32m' BLUE='\033[0;34m' YELLOW='\033[1;33m' NC='\033[0m' echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${BLUE} MQTTUI v2.1 Demo Data Generator (Multi-Broker)${NC}" echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" # Check containers are running for c in $BROKER1 $BROKER2 mqttui; do if ! docker ps --format '{{.Names}}' | grep -q "^${c}$"; then echo "Error: $c container not running. Run: docker compose up -d" exit 1 fi done # --------------------------------------------------------------------------- # 1. Login and get session cookie # --------------------------------------------------------------------------- echo -e "${YELLOW}[1/8] Authenticating...${NC}" COOKIE_JAR="/tmp/mqttui-demo-cookies.txt" LOGIN_RESP=$(curl -s -c "$COOKIE_JAR" -X POST "http://localhost:8088/login" \ -d "username=admin&password=admin" \ -o /dev/null -w "%{http_code}") if [ "$LOGIN_RESP" != "302" ] && [ "$LOGIN_RESP" != "200" ]; then echo " Login failed (HTTP $LOGIN_RESP). Check MQTTUI_ADMIN_USER/PASSWORD." exit 1 fi echo -e " ${GREEN}✓ Logged in as admin${NC}" # --------------------------------------------------------------------------- # 2. Set up second broker connection # --------------------------------------------------------------------------- echo -e "${YELLOW}[2/8] Setting up second broker...${NC}" # Check if Broker 2 already exists EXISTING=$(curl -s -b "$COOKIE_JAR" "$API_URL/brokers/" | python3 -c " import sys,json d=json.load(sys.stdin) brokers = (d.get('data') or d).get('brokers',[]) print(len([b for b in brokers if b.get('host') == '$BROKER2'])) " 2>/dev/null) if [ "$EXISTING" = "0" ]; then curl -s -b "$COOKIE_JAR" -X POST "$API_URL/brokers/" \ -H "Content-Type: application/json" \ -d "{ \"name\": \"Warehouse\", \"host\": \"$BROKER2\", \"port\": 1883, \"topics\": \"#\", \"is_active\": true }" > /dev/null echo -e " ${GREEN}✓ Added 'Warehouse' broker ($BROKER2:1883)${NC}" sleep 2 # Wait for connection else echo -e " ${GREEN}✓ Second broker already exists${NC}" fi BROKER_COUNT=$(curl -s -b "$COOKIE_JAR" "$API_URL/brokers/" | python3 -c " import sys,json; d=json.load(sys.stdin) brokers = (d.get('data') or d).get('brokers',[]) connected = sum(1 for b in brokers if b.get('connected')) print(f'{connected}/{len(brokers)} connected') " 2>/dev/null) echo -e " Brokers: ${GREEN}$BROKER_COUNT${NC}" # --------------------------------------------------------------------------- # 3. Publish to Broker 1 — Home automation sensors # --------------------------------------------------------------------------- echo -e "${YELLOW}[3/8] Publishing to Broker 1 (Default) — Home sensors...${NC}" # Temperature sensors for i in $(seq 1 40); do temp=$(echo "scale=1; 18 + ($RANDOM % 200) / 10" | bc) $PUB1 -t "home/living-room/temperature" -m "{\"value\": $temp, \"unit\": \"C\", \"device\": \"DHT22\"}" $PUB1 -t "home/bedroom/temperature" -m "{\"value\": $temp, \"unit\": \"C\", \"device\": \"DHT22\"}" done echo -e " ${GREEN}✓ 80 temperature readings${NC}" # Humidity for i in $(seq 1 20); do humidity=$((40 + RANDOM % 40)) $PUB1 -t "home/living-room/humidity" -m "{\"value\": $humidity, \"unit\": \"%\"}" done echo -e " ${GREEN}✓ 20 humidity readings${NC}" # Motion sensors for i in $(seq 1 15); do room=$(echo "hallway kitchen garage front-door" | tr ' ' '\n' | sort -R 2>/dev/null | head -1 || awk 'BEGIN{srand()}{a[NR]=$0}END{print a[int(rand()*NR)+1]}') $PUB1 -t "home/$room/motion" -m "{\"detected\": true, \"confidence\": $((70 + RANDOM % 30))}" done echo -e " ${GREEN}✓ 15 motion events${NC}" # Battery levels for i in $(seq 1 10); do device=$(echo "thermostat doorbell smoke-detector leak-sensor" | tr ' ' '\n' | sort -R 2>/dev/null | head -1 || awk 'BEGIN{srand()}{a[NR]=$0}END{print a[int(rand()*NR)+1]}') battery=$((5 + RANDOM % 95)) $PUB1 -t "home/$device/battery" -m "{\"level\": $battery, \"charging\": false}" done echo -e " ${GREEN}✓ 10 battery levels${NC}" # Retained status $PUB1 -t "system/home/status" -m '{"status": "online", "version": "2.1.0"}' -r echo -e " ${GREEN}✓ 1 retained status message${NC}" echo -e " ${GREEN}Broker 1 total: 126 messages${NC}" # --------------------------------------------------------------------------- # 4. Publish to Broker 2 — Warehouse / industrial sensors # --------------------------------------------------------------------------- echo -e "${YELLOW}[4/8] Publishing to Broker 2 (Warehouse) — Industrial sensors...${NC}" # Warehouse temperature (cold storage monitoring) for i in $(seq 1 40); do temp=$(echo "scale=1; -5 + ($RANDOM % 150) / 10" | bc) zone=$(echo "zone-A zone-B zone-C" | tr ' ' '\n' | sort -R 2>/dev/null | head -1 || awk 'BEGIN{srand()}{a[NR]=$0}END{print a[int(rand()*NR)+1]}') $PUB2 -t "warehouse/$zone/temperature" -m "{\"value\": $temp, \"unit\": \"C\", \"sensor\": \"PT100\"}" done echo -e " ${GREEN}✓ 40 cold storage temperature readings${NC}" # Conveyor belt speed for i in $(seq 1 20); do line=$(echo "line-1 line-2 line-3" | tr ' ' '\n' | sort -R 2>/dev/null | head -1 || awk 'BEGIN{srand()}{a[NR]=$0}END{print a[int(rand()*NR)+1]}') speed=$(echo "scale=1; 1 + ($RANDOM % 50) / 10" | bc) $PUB2 -t "warehouse/$line/conveyor/speed" -m "{\"value\": $speed, \"unit\": \"m/s\"}" done echo -e " ${GREEN}✓ 20 conveyor speed readings${NC}" # Door access events for i in $(seq 1 15); do door=$(echo "main loading-dock office emergency" | tr ' ' '\n' | sort -R 2>/dev/null | head -1 || awk 'BEGIN{srand()}{a[NR]=$0}END{print a[int(rand()*NR)+1]}') action=$(echo "opened closed" | tr ' ' '\n' | sort -R 2>/dev/null | head -1 || awk 'BEGIN{srand()}{a[NR]=$0}END{print a[int(rand()*NR)+1]}') $PUB2 -t "warehouse/doors/$door" -m "{\"action\": \"$action\", \"badge_id\": \"EMP-$((100 + RANDOM % 900))\"}" done echo -e " ${GREEN}✓ 15 door access events${NC}" # Power meters for i in $(seq 1 10); do meter=$(echo "main-panel hvac compressor lighting" | tr ' ' '\n' | sort -R 2>/dev/null | head -1 || awk 'BEGIN{srand()}{a[NR]=$0}END{print a[int(rand()*NR)+1]}') watts=$((500 + RANDOM % 9500)) $PUB2 -t "warehouse/power/$meter" -m "{\"watts\": $watts, \"voltage\": 240}" done echo -e " ${GREEN}✓ 10 power meter readings${NC}" # Retained status $PUB2 -t "system/warehouse/status" -m '{"status": "online", "zones": 3, "conveyors": 3}' -r echo -e " ${GREEN}✓ 1 retained status message${NC}" echo -e " ${GREEN}Broker 2 total: 86 messages${NC}" # --------------------------------------------------------------------------- # 5. Create automation rules (spanning both brokers) # --------------------------------------------------------------------------- echo -e "${YELLOW}[5/8] Creating automation rules...${NC}" # Rule 1: High temperature alert (home) curl -s -b "$COOKIE_JAR" -X POST "$API_URL/rules/" \ -H "Content-Type: application/json" \ -d '{ "name": "Home High Temp Alert", "description": "Alert when home temperature exceeds 35°C", "trigger_topic": "home/+/temperature", "condition": {"path": "value", "op": "gt", "value": 35}, "action": {"type": "publish", "topic": "alerts/high-temp", "payload": "{\"alert\": \"Home temperature exceeded 35°C\"}"}, "rate_limit_per_min": 5 }' > /dev/null echo -e " ${GREEN}✓ Home High Temp Alert${NC}" # Rule 2: Cold storage warning (warehouse — temp too high for cold storage) curl -s -b "$COOKIE_JAR" -X POST "$API_URL/rules/" \ -H "Content-Type: application/json" \ -d '{ "name": "Cold Storage Warning", "description": "Alert when warehouse temp rises above 5°C", "trigger_topic": "warehouse/+/temperature", "condition": {"path": "value", "op": "gt", "value": 5}, "action": {"type": "log", "severity": "critical", "message": "Cold storage temperature exceeded threshold"}, "rate_limit_per_min": 10 }' > /dev/null echo -e " ${GREEN}✓ Cold Storage Warning (warehouse temp > 5°C)${NC}" # Rule 3: Low battery warning curl -s -b "$COOKIE_JAR" -X POST "$API_URL/rules/" \ -H "Content-Type: application/json" \ -d '{ "name": "Low Battery Warning", "description": "Log when any device battery drops below 20%", "trigger_topic": "home/+/battery", "condition": {"path": "level", "op": "lt", "value": 20}, "action": {"type": "log", "severity": "warning", "message": "Low battery detected"}, "rate_limit_per_min": 10 }' > /dev/null echo -e " ${GREEN}✓ Low Battery Warning${NC}" # Rule 4: Emergency door monitor (warehouse) curl -s -b "$COOKIE_JAR" -X POST "$API_URL/rules/" \ -H "Content-Type: application/json" \ -d '{ "name": "Emergency Door Monitor", "description": "Log when emergency door is opened", "trigger_topic": "warehouse/doors/emergency", "condition": {"path": "action", "op": "eq", "value": "opened"}, "action": {"type": "log", "severity": "critical", "message": "Emergency door opened!"}, "rate_limit_per_min": 30 }' > /dev/null echo -e " ${GREEN}✓ Emergency Door Monitor${NC}" # Rule 5: Motion logger curl -s -b "$COOKIE_JAR" -X POST "$API_URL/rules/" \ -H "Content-Type: application/json" \ -d '{ "name": "Motion Event Logger", "description": "Log all motion detection events", "trigger_topic": "home/+/motion", "condition": {}, "action": {"type": "log", "severity": "info", "message": "Motion detected"}, "rate_limit_per_min": 30 }' > /dev/null echo -e " ${GREEN}✓ Motion Event Logger${NC}" # --------------------------------------------------------------------------- # 6. Trigger rules with matching messages # --------------------------------------------------------------------------- echo -e "${YELLOW}[6/8] Triggering rules with matching messages...${NC}" # Trigger home high temp for i in $(seq 1 5); do temp=$(echo "scale=1; 36 + ($RANDOM % 50) / 10" | bc) $PUB1 -t "home/living-room/temperature" -m "{\"value\": $temp, \"unit\": \"C\", \"device\": \"DHT22\"}" sleep 0.3 done echo -e " ${GREEN}✓ 5 high-temp messages on Broker 1${NC}" # Trigger cold storage warnings for i in $(seq 1 5); do temp=$(echo "scale=1; 6 + ($RANDOM % 40) / 10" | bc) zone=$(echo "zone-A zone-B zone-C" | tr ' ' '\n' | sort -R 2>/dev/null | head -1 || awk 'BEGIN{srand()}{a[NR]=$0}END{print a[int(rand()*NR)+1]}') $PUB2 -t "warehouse/$zone/temperature" -m "{\"value\": $temp, \"unit\": \"C\", \"sensor\": \"PT100\"}" sleep 0.3 done echo -e " ${GREEN}✓ 5 cold storage warnings on Broker 2${NC}" # Trigger low battery for i in $(seq 1 3); do device=$(echo "thermostat doorbell smoke-detector" | tr ' ' '\n' | sort -R 2>/dev/null | head -1 || awk 'BEGIN{srand()}{a[NR]=$0}END{print a[int(rand()*NR)+1]}') battery=$((3 + RANDOM % 15)) $PUB1 -t "home/$device/battery" -m "{\"level\": $battery, \"charging\": false}" sleep 0.3 done echo -e " ${GREEN}✓ 3 low-battery messages${NC}" # Trigger emergency door $PUB2 -t "warehouse/doors/emergency" -m '{"action": "opened", "badge_id": "EMP-999"}' echo -e " ${GREEN}✓ 1 emergency door event on Broker 2${NC}" sleep 3 # --------------------------------------------------------------------------- # 7. Create filter presets + bookmarks # --------------------------------------------------------------------------- echo -e "${YELLOW}[7/8] Creating filter presets and bookmarks...${NC}" curl -s -b "$COOKIE_JAR" -X POST "$API_URL/filter-presets" \ -H "Content-Type: application/json" \ -d '{"name": "Home Sensors", "description": "All home sensor data", "filters": {"regex_topic": "home/.*"}}' > /dev/null echo -e " ${GREEN}✓ 'Home Sensors' preset${NC}" curl -s -b "$COOKIE_JAR" -X POST "$API_URL/filter-presets" \ -H "Content-Type: application/json" \ -d '{"name": "Warehouse Only", "description": "All warehouse data", "filters": {"regex_topic": "warehouse/.*"}}' > /dev/null echo -e " ${GREEN}✓ 'Warehouse Only' preset${NC}" curl -s -b "$COOKIE_JAR" -X POST "$API_URL/filter-presets" \ -H "Content-Type: application/json" \ -d '{"name": "Alerts", "description": "Alert topics only", "filters": {"regex_topic": "alerts/.*"}}' > /dev/null echo -e " ${GREEN}✓ 'Alerts' preset${NC}" for topic in "home/living-room/temperature" "warehouse/zone-A/temperature" "warehouse/doors/emergency"; do curl -s -b "$COOKIE_JAR" -X POST "$API_URL/topics/$(echo $topic | sed 's/\//%2F/g')/bookmark" > /dev/null 2>&1 echo -e " ${GREEN}✓ Bookmarked: $topic${NC}" done # --------------------------------------------------------------------------- # 8. Verify data via API # --------------------------------------------------------------------------- echo -e "${YELLOW}[8/8] Verifying data via API...${NC}" MSG_COUNT=$(curl -s -b "$COOKIE_JAR" "$API_URL/messages?limit=1" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('data',d).get('total',0))" 2>/dev/null) echo -e " Messages stored: ${GREEN}$MSG_COUNT${NC}" BROKER_STATUS=$(curl -s -b "$COOKIE_JAR" "$API_URL/brokers/" | python3 -c " import sys,json d=json.load(sys.stdin) brokers = (d.get('data') or d).get('brokers',[]) for b in brokers: status = '✓ Connected' if b.get('connected') else '✗ Disconnected' print(f\" {b['name']} ({b['host']}:{b['port']}): {status}\") " 2>/dev/null) echo -e " Brokers:" echo -e "${GREEN}$BROKER_STATUS${NC}" RULE_COUNT=$(curl -s -b "$COOKIE_JAR" "$API_URL/rules/" | python3 -c "import sys,json; d=json.load(sys.stdin); print(len(d.get('data',d).get('rules',[])))" 2>/dev/null) echo -e " Rules created: ${GREEN}$RULE_COUNT${NC}" ALERT_COUNT=$(curl -s -b "$COOKIE_JAR" "$API_URL/alerts/" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('data',d).get('total',0))" 2>/dev/null) echo -e " Alerts fired: ${GREEN}$ALERT_COUNT${NC}" ANALYTICS=$(curl -s -b "$COOKIE_JAR" "$API_URL/analytics/topics?limit=5" | python3 -c " import sys,json d=json.load(sys.stdin) topics = d.get('data',d).get('topics',[]) for t in topics[:5]: print(f\" {t['topic']}: {t.get('rate_per_min',0):.0f}/min, {t.get('message_count',0)} msgs\") " 2>/dev/null) echo -e " Analytics (top 5 topics):" echo -e "${GREEN}$ANALYTICS${NC}" # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- echo "" echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo -e "${BLUE} Demo Data Ready! (Multi-Broker)${NC}" echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" echo "" echo -e "Open ${GREEN}http://localhost:8088${NC} and check:" echo "" echo -e " ${YELLOW}Dashboard${NC}" echo " - Messages from BOTH brokers (look for broker name badges)" echo " - Topic graph with home/* and warehouse/* nodes" echo " - Use Broker dropdown in Advanced Search to filter by broker" echo " - Use ★ Favorites Only to see bookmarked topics" echo "" echo -e " ${YELLOW}Brokers${NC}" echo " - Default (mosquitto) — home automation sensors" echo " - Warehouse (mosquitto2) — industrial sensors" echo " - Both should show 'Connected' with green indicator" echo "" echo -e " ${YELLOW}Rules${NC}" echo " - 5 rules spanning both brokers" echo " - Home High Temp + Cold Storage + Low Battery + Emergency Door + Motion" echo "" echo -e " ${YELLOW}Alerts${NC}" echo " - Alerts from rules on both brokers" echo " - Filter by rule to see per-broker alerts" echo "" echo -e " ${YELLOW}Analytics${NC}" echo " - Topics from both brokers in the same view" echo " - Temperature histograms for home AND warehouse" echo "" rm -f "$COOKIE_JAR" ================================================ FILE: docker-compose.yml ================================================ services: mosquitto: image: eclipse-mosquitto:latest container_name: mosquitto ports: - "1883:1883" - "9002:9001" volumes: - ./mosquitto.conf:/mosquitto/config/mosquitto.conf mosquitto2: image: eclipse-mosquitto:latest container_name: mosquitto2 ports: - "1884:1883" volumes: - ./mosquitto.conf:/mosquitto/config/mosquitto.conf web: container_name: mqttui build: . ports: - "8088:5000" environment: - DEBUG=False - HOST=0.0.0.0 - PORT=5000 - MQTT_BROKER=mosquitto - MQTT_PORT=1883 - MQTT_USERNAME= - MQTT_PASSWORD= - MQTT_KEEPALIVE=60 - MQTT_VERSION=3.1.1 - SECRET_KEY=mqttui-dev-secret-change-in-prod - LOG_LEVEL=INFO - MQTT_TOPICS=# - DB_ENABLED=True - DB_PATH=/app/data/mqtt_messages.db - DB_MAX_MESSAGES=10000 - DB_CLEANUP_DAYS=30 - MQTTUI_ADMIN_USER=admin - MQTTUI_ADMIN_PASSWORD=admin - MQTTUI_RATE_LIMIT=30/minute volumes: - mqtt-data:/app/data depends_on: - mosquitto volumes: mqtt-data: ================================================ FILE: entrypoint.sh ================================================ #!/bin/sh # Default to port 5000 if PORT is not set PORT="${PORT:-5000}" # Set default LOG_LEVEL if not provided LOG_LEVEL="${LOG_LEVEL:-info}" # Convert to lowercase for comparison (sh compatible) LOG_LEVEL_LOWER=$(echo "$LOG_LEVEL" | tr '[:upper:]' '[:lower:]') # Validate and normalize log level for gunicorn case "$LOG_LEVEL_LOWER" in debug|info|warning|warn|error|critical) # Convert WARN to WARNING for gunicorn compatibility if [ "$LOG_LEVEL_LOWER" = "warn" ]; then LOG_LEVEL="warning" else LOG_LEVEL="$LOG_LEVEL_LOWER" fi ;; *) echo "Invalid LOG_LEVEL: $LOG_LEVEL. Using default: info" LOG_LEVEL="info" ;; esac if [ "$DEBUG" = "True" ] || [ "$DEBUG" = "1" ] || [ "$DEBUG" = "true" ]; then echo "Running in DEBUG mode with log level: $LOG_LEVEL" exec python wsgi.py else echo "Running in PRODUCTION mode with log level: $LOG_LEVEL" exec gunicorn --log-level "$LOG_LEVEL" --worker-class geventwebsocket.gunicorn.workers.GeventWebSocketWorker -w 1 -b "0.0.0.0:$PORT" "wsgi:app" fi ================================================ FILE: mosquitto.conf ================================================ listener 1883 allow_anonymous true persistence false # WebSocket support listener 9001 protocol websockets ================================================ FILE: mqttui/__init__.py ================================================ __version__ = "2.0.0-dev" ================================================ FILE: mqttui/analytics.py ================================================ """Per-topic analytics engine with rate counters and numeric payload histograms. Provides real-time message rate tracking and statistical aggregation for numeric JSON payload fields. Subscribes to mqtt_message_received signal for automatic data collection. """ import json import logging import time from collections import deque from mqttui.events import mqtt_message_received logger = logging.getLogger(__name__) # Maximum timestamps stored per topic (automatic memory bounding) MAX_TIMESTAMPS = 10000 class TopicAnalytics: """Track per-topic message rates and numeric payload histograms. Data structures per topic: _timestamps: dict[str, deque] -- rolling window of message timestamps _histograms: dict[str, dict] -- per-topic numeric stats per field Each field: {min, max, sum, count} """ def __init__(self): self._timestamps: dict[str, deque] = {} self._histograms: dict[str, dict] = {} def record(self, topic: str, payload: str, timestamp: float): """Record a message for a topic. Args: topic: MQTT topic string payload: Raw payload string (may be JSON) timestamp: Unix timestamp of message receipt """ # Track timestamp for rate calculation if topic not in self._timestamps: self._timestamps[topic] = deque(maxlen=MAX_TIMESTAMPS) # Ensure timestamp is a Unix float for rate calculations if hasattr(timestamp, 'timestamp'): timestamp = timestamp.timestamp() self._timestamps[topic].append(timestamp) # Try to extract numeric fields from JSON payload try: data = json.loads(payload) if isinstance(data, dict): for field_name, value in data.items(): if isinstance(value, (int, float)) and not isinstance(value, bool): self._update_histogram(topic, field_name, float(value)) except (json.JSONDecodeError, TypeError, ValueError): pass def _update_histogram(self, topic: str, field_name: str, value: float): """Update histogram stats for a numeric field.""" if topic not in self._histograms: self._histograms[topic] = {} if field_name not in self._histograms[topic]: self._histograms[topic][field_name] = { "min": value, "max": value, "sum": value, "count": 1, } else: stats = self._histograms[topic][field_name] stats["min"] = min(stats["min"], value) stats["max"] = max(stats["max"], value) stats["sum"] += value stats["count"] += 1 def get_rate(self, topic: str, window: int = 60) -> float: """Count messages within the last `window` seconds. Args: topic: MQTT topic string window: Time window in seconds (default 60) Returns: Number of messages received within the window. """ if topic not in self._timestamps: return 0.0 cutoff = time.time() - window count = sum(1 for ts in self._timestamps[topic] if ts >= cutoff) return float(count) def get_topic_stats(self, topic: str) -> dict: """Return full stats dict for a single topic. Returns: Dict with topic, rate_per_min, rate_per_hour, message_count, histograms """ ts_deque = self._timestamps.get(topic, deque()) return { "topic": topic, "rate_per_min": self.get_rate(topic, 60), "rate_per_hour": self.get_rate(topic, 3600), "message_count": len(ts_deque), "histograms": self._histograms.get(topic, {}), } def get_all_stats(self, limit: int = 20) -> list: """Return stats for all tracked topics, sorted by rate_per_min descending. Args: limit: Maximum number of topics to return (default 20) Returns: List of topic stats dicts. """ all_topics = list(self._timestamps.keys()) stats_list = [self.get_topic_stats(t) for t in all_topics] stats_list.sort(key=lambda s: s["rate_per_min"], reverse=True) return stats_list[:limit] # --------------------------------------------------------------------------- # Module-level singleton # --------------------------------------------------------------------------- _analytics = None def get_analytics() -> TopicAnalytics: """Return the module-level TopicAnalytics singleton.""" global _analytics if _analytics is None: _analytics = TopicAnalytics() return _analytics # --------------------------------------------------------------------------- # Signal subscriber # --------------------------------------------------------------------------- def _on_mqtt_message(sender, **kwargs): """Handle mqtt_message_received signal -- record message in analytics.""" get_analytics().record( kwargs["topic"], kwargs["payload"], kwargs["timestamp"], ) ================================================ FILE: mqttui/app.py ================================================ import os import structlog from flask import Flask from mqttui.extensions import socketio, sa, login_manager from mqttui.events import mqtt_message_received, rule_fired, alert_triggered from mqttui.logging_config import configure_logging def _on_mqtt_message(sender, **kwargs): """Forward MQTT messages to WebSocket clients and persist to database.""" topic = kwargs['topic'] payload = kwargs['payload'] timestamp = kwargs['timestamp'] qos = kwargs.get('qos', 0) retain = kwargs.get('retain', False) # Emit to connected browsers via batch emitter (100ms batching) from mqttui.socketio_batch import get_batch_emitter emitter = get_batch_emitter() msg_data = { 'topic': topic, 'payload': payload, 'timestamp': timestamp.isoformat(), 'retain': retain, 'broker_id': kwargs.get('broker_id'), 'broker_name': kwargs.get('broker_name'), } if emitter: emitter.enqueue(msg_data) else: socketio.emit('mqtt_message', msg_data) # Persist to database import mqttui.extensions as ext if ext.db: try: ext.db.store_message( topic=topic, payload=payload, timestamp=timestamp, qos=qos, retain=retain, ) except Exception as e: structlog.get_logger(__name__).error("Failed to store message", error=str(e)) def create_app(config=None): """Application factory for mqttui.""" app = Flask( __name__, static_folder='../static', template_folder='../templates' ) # Load config from environment variables app.config['SECRET_KEY'] = os.getenv('SECRET_KEY', 'your-secret-key') app.config['DEBUG'] = os.getenv('DEBUG', 'False').lower() in ('true', '1', 't') app.config['HOST'] = os.getenv('HOST', '0.0.0.0') app.config['PORT'] = int(os.getenv('PORT', 5000)) app.config['MQTT_BROKER'] = os.getenv('MQTT_BROKER', 'localhost') app.config['MQTT_PORT'] = int(os.getenv('MQTT_PORT', 1883)) app.config['MQTT_USERNAME'] = os.getenv('MQTT_USERNAME') app.config['MQTT_PASSWORD'] = os.getenv('MQTT_PASSWORD') app.config['MQTT_KEEPALIVE'] = int(os.getenv('MQTT_KEEPALIVE', 60)) app.config['MQTT_VERSION'] = os.getenv('MQTT_VERSION', '3.1.1') app.config['MQTT_TOPICS'] = os.getenv('MQTT_TOPICS', '#') app.config['MQTT_TLS'] = os.getenv('MQTT_TLS', 'false') app.config['MQTT_TLS_CA_CERTS'] = os.getenv('MQTT_TLS_CA_CERTS', '') app.config['MQTT_TLS_CERTFILE'] = os.getenv('MQTT_TLS_CERTFILE', '') app.config['MQTT_TLS_KEYFILE'] = os.getenv('MQTT_TLS_KEYFILE', '') app.config['MQTT_TLS_INSECURE'] = os.getenv('MQTT_TLS_INSECURE', 'false') app.config['DB_ENABLED'] = os.getenv('DB_ENABLED', 'True').lower() in ('true', '1', 't') app.config['DB_PATH'] = os.getenv('DB_PATH', 'mqtt_messages.db') app.config['DB_MAX_MESSAGES'] = int(os.getenv('DB_MAX_MESSAGES', 10000)) app.config['DB_CLEANUP_DAYS'] = int(os.getenv('DB_CLEANUP_DAYS', 30)) # Override with passed config dict if config: app.config.update(config) # Set up structured logging via structlog log_level_name = os.getenv('LOG_LEVEL', 'DEBUG' if app.config['DEBUG'] else 'INFO').upper() configure_logging(debug=app.config['DEBUG'], log_level=log_level_name) # SECRET_KEY production guard flask_env = os.getenv('FLASK_ENV', 'development') insecure_keys = {'dev', 'change-me', 'your-secret-key'} if flask_env == 'production' and app.config['SECRET_KEY'] in insecure_keys: raise RuntimeError( "SECRET_KEY is insecure. Set a strong SECRET_KEY environment variable for production." ) # Initialize SocketIO with gevent async mode socketio.init_app(app, async_mode='gevent') # Initialize batch emitter for Socket.IO message batching (100ms windows) from mqttui.socketio_batch import init_batch_emitter init_batch_emitter(socketio, interval_ms=100) # Enable CORS for API endpoints from flask_cors import CORS CORS(app, resources={r"/api/*": {"origins": "*"}}) # Configure and initialize SQLAlchemy for user database db_dir = os.path.dirname(os.path.abspath(app.config.get('DB_PATH', 'mqtt_messages.db'))) app.config.setdefault('SQLALCHEMY_DATABASE_URI', f"sqlite:///{os.path.join(db_dir, 'mqttui_users.db')}") app.config.setdefault('SQLALCHEMY_TRACK_MODIFICATIONS', False) sa.init_app(app) login_manager.init_app(app) # Initialize rate limiter from mqttui.extensions import limiter rate_limit = os.getenv('MQTTUI_RATE_LIMIT', '30/minute') app.config['RATELIMIT_DEFAULT'] = rate_limit limiter.init_app(app) with app.app_context(): from mqttui.models import User # noqa: F811 from mqttui.rules.models import Rule, AlertHistory # noqa: F401 from mqttui.plugins.models import PluginConfig # noqa: F401 sa.create_all() # Initialize database if enabled if app.config['DB_ENABLED']: try: from mqttui.database import MessageDatabase import mqttui.extensions as ext db_instance = MessageDatabase( app.config['DB_PATH'], app.config['DB_MAX_MESSAGES'] ) ext.db = db_instance structlog.get_logger(__name__).info("Database initialized", path=app.config['DB_PATH']) except Exception as e: structlog.get_logger(__name__).error("Failed to initialize database", error=str(e)) # Register blueprints from mqttui.routes.main import bp as main_bp from mqttui.routes.api import bp as api_bp # TODO: Remove legacy /api/ routes in Phase 5 after frontend migrates to /api/v1/ from mqttui.routes.debug import bp as debug_bp from mqttui.routes.api_v1 import api_v1_bp from mqttui.routes.rules import rules_bp from mqttui.routes.alerts import alerts_bp from mqttui.routes.metrics import metrics_bp from mqttui.routes.analytics import analytics_bp from mqttui.routes.plugins import plugins_bp from mqttui.routes.brokers import brokers_bp app.register_blueprint(main_bp) app.register_blueprint(api_bp) app.register_blueprint(debug_bp) app.register_blueprint(api_v1_bp) app.register_blueprint(rules_bp) app.register_blueprint(alerts_bp) app.register_blueprint(metrics_bp) app.register_blueprint(analytics_bp) app.register_blueprint(plugins_bp) app.register_blueprint(brokers_bp) # Register auth blueprint and seed admin user from mqttui.auth import auth_bp, seed_admin_user app.register_blueprint(auth_bp) seed_admin_user(app) # Initialize MQTT client (paho-mqtt 2.x with event bus) from mqttui.mqtt_client import init_mqtt init_mqtt(app) # Wire event bus: forward MQTT messages to SocketIO and database mqtt_message_received.connect(_on_mqtt_message) # Wire analytics subscriber to track per-topic rates and histograms from mqttui.analytics import _on_mqtt_message as _on_analytics_message mqtt_message_received.connect(_on_analytics_message) # Wire Prometheus metric signal subscribers from mqttui.routes.metrics import ( _on_message_for_metrics, _on_rule_fired_for_metrics, _on_alert_for_metrics ) mqtt_message_received.connect(_on_message_for_metrics) rule_fired.connect(_on_rule_fired_for_metrics) alert_triggered.connect(_on_alert_for_metrics) # Initialize rules engine (Phase 3) from mqttui.rules.engine import RuleEngine rule_engine = RuleEngine(app=app) rule_engine.connect() # Initialize plugin registry (Phase 7) from mqttui.plugins.registry import init_plugin_registry init_plugin_registry(app) # Initialize plugin runner and wire to event bus (Phase 7) from mqttui.plugins.runner import init_plugin_runner plugin_runner = init_plugin_runner(app) mqtt_message_received.connect(plugin_runner.on_mqtt_message) rule_fired.connect(plugin_runner.on_rule_trigger) return app ================================================ FILE: mqttui/auth.py ================================================ import os import logging from flask import Blueprint, render_template, redirect, url_for, request, flash from flask_login import login_user, logout_user, login_required, current_user from mqttui.models import User from mqttui.extensions import sa, login_manager logger = logging.getLogger(__name__) auth_bp = Blueprint('auth', __name__) @auth_bp.before_app_request def load_user_from_api_key(): """Check X-API-Key header before session auth.""" api_key = request.headers.get('X-API-Key') if api_key: user = User.query.filter_by(api_token=api_key).first() if user and user.is_active: login_user(user) @login_manager.user_loader def load_user(user_id): return sa.session.get(User, int(user_id)) def seed_admin_user(app): """Create default admin user from environment variables if not exists.""" admin_username = os.environ.get('MQTTUI_ADMIN_USER', app.config.get('MQTTUI_ADMIN_USER', 'admin')) admin_password = os.environ.get('MQTTUI_ADMIN_PASSWORD', app.config.get('MQTTUI_ADMIN_PASSWORD', 'admin')) with app.app_context(): existing = User.query.filter_by(username=admin_username).first() if existing: logger.info("Admin user already exists") return user = User(username=admin_username) user.set_password(admin_password) user.generate_api_token() sa.session.add(user) sa.session.commit() logger.info("Admin user seeded") @auth_bp.route('/login', methods=['GET']) def login(): if current_user.is_authenticated: return redirect(url_for('main.index')) return render_template('login.html') @auth_bp.route('/login', methods=['POST']) def login_post(): username = request.form.get('username', '') password = request.form.get('password', '') user = User.query.filter_by(username=username).first() if user and user.check_password(password): login_user(user) next_page = request.args.get('next') return redirect(next_page or '/') flash('Invalid username or password', 'error') return render_template('login.html'), 200 @auth_bp.route('/logout') def logout(): logout_user() return redirect(url_for('auth.login')) ================================================ FILE: mqttui/broker_manager.py ================================================ """Multi-broker connection manager. Manages multiple paho-mqtt client connections, each tied to a Broker model. Replaces the single-client approach in mqtt_client.py. """ import os import logging import ssl from datetime import datetime import paho.mqtt.client as mqtt from paho.mqtt.enums import CallbackAPIVersion from mqttui.events import mqtt_message_received import mqttui.state as state logger = logging.getLogger(__name__) MQTT_RC_CODES = { 0: "Connection successful", 1: "Incorrect protocol version", 2: "Invalid client identifier", 3: "Server unavailable", 4: "Bad username or password", 5: "Not authorised", 128: "Unspecified error", 134: "Bad user name or password", 135: "Not authorized", 136: "Server unavailable", } # Module-level singleton _broker_manager = None def get_broker_manager(): return _broker_manager class ManagedConnection: """A single broker connection with its paho-mqtt client.""" def __init__(self, broker_id, name, host, port, username=None, password=None, mqtt_version='3.1.1', topics='#', tls_enabled=False, tls_ca_certs=None, tls_insecure=False): self.broker_id = broker_id self.name = name self.host = host self.port = port self.topics_str = topics self.connected = False self.error = None # Create paho-mqtt client protocol = mqtt.MQTTv5 if mqtt_version == '5' else mqtt.MQTTv311 self.client = mqtt.Client( callback_api_version=CallbackAPIVersion.VERSION2, client_id=f"mqttui_{os.getpid()}_{broker_id}", protocol=protocol, ) if username and password: self.client.username_pw_set(username, password) if tls_enabled: self.client.tls_set( ca_certs=tls_ca_certs if tls_ca_certs else None, cert_reqs=ssl.CERT_NONE if tls_insecure else ssl.CERT_REQUIRED, ) if tls_insecure: self.client.tls_insecure_set(True) # Wire callbacks self.client.on_connect = self._on_connect self.client.on_disconnect = self._on_disconnect self.client.on_message = self._on_message def _on_connect(self, client, userdata, connect_flags, reason_code, properties=None): rc = reason_code.value if hasattr(reason_code, 'value') else int(reason_code) if rc == 0: self.connected = True self.error = None state.connection_count += 1 for topic in [t.strip() for t in self.topics_str.split(',')]: client.subscribe(topic) logger.info(f"[{self.name}] Subscribed to: {topic}") logger.info(f"[{self.name}] Connected to {self.host}:{self.port}") else: self.connected = False self.error = MQTT_RC_CODES.get(rc, f"Unknown error (rc: {rc})") logger.error(f"[{self.name}] Connection failed: {self.error}") def _on_disconnect(self, client, userdata, disconnect_flags, reason_code, properties=None): rc = reason_code.value if hasattr(reason_code, 'value') else int(reason_code) self.connected = False state.connection_count = max(0, state.connection_count - 1) if rc != 0: self.error = MQTT_RC_CODES.get(rc, f"Unexpected disconnect (rc: {rc})") logger.warning(f"[{self.name}] Disconnected: {self.error}") def _on_message(self, client, userdata, msg): try: payload = msg.payload.decode() except UnicodeDecodeError: payload = msg.payload.hex() timestamp = datetime.now() # Update shared in-memory state message = { 'topic': msg.topic, 'payload': payload, 'timestamp': timestamp.isoformat(), 'broker_id': self.broker_id, 'broker_name': self.name, } state.messages.append(message) state.topics.add(msg.topic) if len(state.messages) > 100: state.messages.pop(0) # Debug bar try: from debug_bar import debug_bar debug_bar.record('mqtt', 'last_topic', msg.topic) debug_bar.record('mqtt', 'last_payload', payload[:200]) debug_bar.record('mqtt', 'last_broker', f"{self.name} ({self.host}:{self.port})") debug_bar.record('mqtt', 'last_timestamp', timestamp.isoformat()) except Exception: pass # Fire event bus signal with broker context mqtt_message_received.send( 'broker_manager', topic=msg.topic, payload=payload, timestamp=timestamp, qos=msg.qos, retain=msg.retain, broker_id=self.broker_id, broker_name=self.name, ) def start(self): try: self.client.connect(self.host, self.port, keepalive=60) self.client.loop_start() logger.info(f"[{self.name}] Connecting to {self.host}:{self.port}") except Exception as e: self.connected = False self.error = str(e) logger.error(f"[{self.name}] Failed to connect: {e}") def stop(self): try: self.client.loop_stop() self.client.disconnect() except Exception: pass self.connected = False def publish(self, topic, payload, qos=0, retain=False): if self.client and self.connected: self.client.publish(topic, payload, qos=qos, retain=retain) def status_dict(self): return { 'broker_id': self.broker_id, 'name': self.name, 'host': self.host, 'port': self.port, 'connected': self.connected, 'error': self.error, } class BrokerManager: """Manages multiple MQTT broker connections.""" def __init__(self, app=None): self.app = app self.connections = {} # broker_id -> ManagedConnection def add_connection(self, broker): """Add and start a connection from a Broker model instance or dict.""" if isinstance(broker, dict): broker_id = broker['id'] conn = ManagedConnection( broker_id=broker_id, name=broker['name'], host=broker['host'], port=broker.get('port', 1883), username=broker.get('username'), password=broker.get('password'), mqtt_version=broker.get('mqtt_version', '3.1.1'), topics=broker.get('topics', '#'), tls_enabled=broker.get('tls_enabled', False), tls_ca_certs=broker.get('tls_ca_certs'), tls_insecure=broker.get('tls_insecure', False), ) else: broker_id = broker.id conn = ManagedConnection( broker_id=broker.id, name=broker.name, host=broker.host, port=broker.port, username=broker.username, password=broker.password, mqtt_version=broker.mqtt_version, topics=broker.topics, tls_enabled=broker.tls_enabled, tls_ca_certs=broker.tls_ca_certs, tls_insecure=broker.tls_insecure, ) # Stop existing connection if replacing if broker_id in self.connections: self.connections[broker_id].stop() self.connections[broker_id] = conn conn.start() return conn def remove_connection(self, broker_id): conn = self.connections.pop(broker_id, None) if conn: conn.stop() def get_connection(self, broker_id): return self.connections.get(broker_id) def get_default_connection(self): """Return the first connected broker, or first broker overall.""" for conn in self.connections.values(): if conn.connected: return conn # Fallback to first if self.connections: return next(iter(self.connections.values())) return None def publish(self, topic, payload, qos=0, retain=False, broker_id=None): """Publish to a specific broker or the default one.""" if broker_id and broker_id in self.connections: self.connections[broker_id].publish(topic, payload, qos, retain) else: conn = self.get_default_connection() if conn: conn.publish(topic, payload, qos, retain) def all_statuses(self): return [conn.status_dict() for conn in self.connections.values()] def start_all(self, app): """Load active brokers from DB and start connections.""" with app.app_context(): from mqttui.models import Broker brokers = Broker.query.filter_by(is_active=True).all() for broker in brokers: self.add_connection(broker) logger.info(f"BrokerManager started {len(brokers)} connection(s)") def stop_all(self): for conn in self.connections.values(): conn.stop() self.connections.clear() def load_env_broker(self, app): """Create a default broker from environment variables if no brokers in DB.""" with app.app_context(): from mqttui.models import Broker if Broker.query.count() > 0: return # DB already has brokers # Seed from env vars (backward compatible with v1.x) broker = Broker( name='Default', host=app.config['MQTT_BROKER'], port=app.config['MQTT_PORT'], username=app.config.get('MQTT_USERNAME'), password=app.config.get('MQTT_PASSWORD'), mqtt_version=app.config['MQTT_VERSION'], topics=app.config['MQTT_TOPICS'], tls_enabled=app.config.get('MQTT_TLS', 'false').lower() in ('true', '1', 'yes'), tls_ca_certs=app.config.get('MQTT_TLS_CA_CERTS') or None, tls_insecure=app.config.get('MQTT_TLS_INSECURE', 'false').lower() in ('true', '1', 'yes'), is_active=True, is_default=True, ) from mqttui.extensions import sa sa.session.add(broker) sa.session.commit() logger.info(f"Seeded default broker from env: {broker.host}:{broker.port}") def init_broker_manager(app): """Initialize the global broker manager. Call from create_app().""" global _broker_manager _broker_manager = BrokerManager(app=app) _broker_manager.load_env_broker(app) _broker_manager.start_all(app) return _broker_manager ================================================ FILE: mqttui/database.py ================================================ """ Database module for MQTT message persistence """ import sqlite3 import threading import logging import re import json from datetime import datetime, timedelta from typing import List, Dict, Optional, Union import os class MessageDatabase: def __init__(self, db_path: str = "mqtt_messages.db", max_messages: int = 10000): self.db_path = db_path self.max_messages = max_messages self._local = threading.local() self.init_database() def get_connection(self): """Get thread-local database connection""" if not hasattr(self._local, 'connection'): self._local.connection = sqlite3.connect( self.db_path, check_same_thread=False, timeout=30.0 ) self._local.connection.row_factory = sqlite3.Row # Enable WAL mode for concurrent read/write self._local.connection.execute('PRAGMA journal_mode=WAL') self._local.connection.execute('PRAGMA busy_timeout=5000') # Add REGEXP function for advanced topic filtering self._local.connection.create_function("REGEXP", 2, self._regexp) return self._local.connection def _regexp(self, pattern, value): """Custom REGEXP function for SQLite""" try: return re.search(pattern, value, re.IGNORECASE) is not None except Exception: return False def init_database(self): """Initialize database schema""" conn = self.get_connection() try: cursor = conn.cursor() # Messages table cursor.execute(''' CREATE TABLE IF NOT EXISTS messages ( id INTEGER PRIMARY KEY AUTOINCREMENT, topic TEXT NOT NULL, payload TEXT NOT NULL, timestamp DATETIME NOT NULL, qos INTEGER DEFAULT 0, retain BOOLEAN DEFAULT 0, payload_size INTEGER DEFAULT 0, created_at DATETIME DEFAULT CURRENT_TIMESTAMP ) ''') # Topics table for faster topic queries cursor.execute(''' CREATE TABLE IF NOT EXISTS topics ( topic TEXT PRIMARY KEY, last_message_at DATETIME NOT NULL, message_count INTEGER DEFAULT 1, first_seen DATETIME DEFAULT CURRENT_TIMESTAMP ) ''') # Filter presets table for saved searches cursor.execute(''' CREATE TABLE IF NOT EXISTS filter_presets ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT UNIQUE NOT NULL, description TEXT, filters TEXT NOT NULL, -- JSON string of filter parameters created_at DATETIME DEFAULT CURRENT_TIMESTAMP, last_used DATETIME ) ''') # Create indexes for better performance cursor.execute(''' CREATE INDEX IF NOT EXISTS idx_messages_topic ON messages(topic) ''') cursor.execute(''' CREATE INDEX IF NOT EXISTS idx_messages_timestamp ON messages(timestamp DESC) ''') cursor.execute(''' CREATE INDEX IF NOT EXISTS idx_messages_topic_timestamp ON messages(topic, timestamp DESC) ''') conn.commit() logging.info("Database initialized successfully") except Exception as e: logging.error(f"Database initialization error: {e}") conn.rollback() raise def store_message(self, topic: str, payload: str, timestamp: datetime = None, qos: int = 0, retain: bool = False) -> bool: """Store a message in the database""" if timestamp is None: timestamp = datetime.now() conn = self.get_connection() try: cursor = conn.cursor() # Store message cursor.execute(''' INSERT INTO messages (topic, payload, timestamp, qos, retain, payload_size) VALUES (?, ?, ?, ?, ?, ?) ''', (topic, payload, timestamp, qos, retain, len(payload))) # Update topics table cursor.execute(''' INSERT OR REPLACE INTO topics (topic, last_message_at, message_count, first_seen) VALUES (?, ?, COALESCE((SELECT message_count FROM topics WHERE topic = ?) + 1, 1), COALESCE((SELECT first_seen FROM topics WHERE topic = ?), ?) ) ''', (topic, timestamp, topic, topic, timestamp)) conn.commit() # Clean up old messages if needed self._cleanup_old_messages() return True except Exception as e: logging.error(f"Error storing message: {e}") conn.rollback() return False def get_messages(self, limit: int = 100, offset: int = 0, topic_filter: str = None, since: datetime = None, content_search: str = None, regex_topic: str = None, json_path: str = None, json_value: str = None) -> List[Dict]: """Retrieve messages from database with enhanced filtering""" conn = self.get_connection() try: cursor = conn.cursor() query = ''' SELECT topic, payload, timestamp, qos, retain, payload_size FROM messages WHERE 1=1 ''' params = [] # Basic topic filtering (wildcard support) if topic_filter: if '%' in topic_filter or '_' in topic_filter: query += ' AND topic LIKE ?' else: query += ' AND topic = ?' params.append(topic_filter) # Regex topic filtering (more powerful than LIKE) if regex_topic: query += ' AND topic REGEXP ?' params.append(regex_topic) # Content search (case-insensitive) if content_search: query += ' AND LOWER(payload) LIKE LOWER(?)' params.append(f'%{content_search}%') # Time filtering if since: query += ' AND timestamp >= ?' params.append(since) query += ' ORDER BY timestamp DESC LIMIT ? OFFSET ?' params.extend([limit, offset]) cursor.execute(query, params) rows = cursor.fetchall() messages = [dict(row) for row in rows] # Apply Python-based filters that can't be done in SQL if json_path or regex_topic: messages = self._apply_python_filters( messages, regex_topic, json_path, json_value ) return messages except Exception as e: logging.error(f"Error retrieving messages: {e}") return [] def _apply_python_filters(self, messages: List[Dict], regex_topic: str = None, json_path: str = None, json_value: str = None) -> List[Dict]: """Apply filters that require Python processing""" filtered = [] for msg in messages: # Regex topic filter (if not handled by SQL REGEXP) if regex_topic and not re.search(regex_topic, msg['topic'], re.IGNORECASE): continue # JSON path filtering if json_path and json_value: try: payload_json = json.loads(msg['payload']) value = self._get_json_path_value(payload_json, json_path) if value is None or str(value).lower() != json_value.lower(): continue except (json.JSONDecodeError, KeyError): continue filtered.append(msg) return filtered def _get_json_path_value(self, json_obj: dict, path: str): """Extract value from JSON using simple dot notation path""" try: keys = path.split('.') current = json_obj for key in keys: if isinstance(current, dict) and key in current: current = current[key] else: return None return current except Exception: return None def get_topics(self) -> List[Dict]: """Get all topics with statistics""" conn = self.get_connection() try: cursor = conn.cursor() cursor.execute(''' SELECT topic, message_count, last_message_at, first_seen FROM topics ORDER BY last_message_at DESC ''') rows = cursor.fetchall() return [dict(row) for row in rows] except Exception as e: logging.error(f"Error retrieving topics: {e}") return [] def get_message_count(self, topic_filter: str = None, since: datetime = None) -> int: """Get total message count with optional filtering""" conn = self.get_connection() try: cursor = conn.cursor() query = 'SELECT COUNT(*) FROM messages WHERE 1=1' params = [] if topic_filter: if '%' in topic_filter or '_' in topic_filter: query += ' AND topic LIKE ?' else: query += ' AND topic = ?' params.append(topic_filter) if since: query += ' AND timestamp >= ?' params.append(since) cursor.execute(query, params) return cursor.fetchone()[0] except Exception as e: logging.error(f"Error counting messages: {e}") return 0 def _cleanup_old_messages(self): """Remove old messages to stay within limit""" conn = self.get_connection() try: cursor = conn.cursor() cursor.execute('SELECT COUNT(*) FROM messages') count = cursor.fetchone()[0] if count > self.max_messages: messages_to_delete = count - self.max_messages + 1000 cursor.execute(''' DELETE FROM messages WHERE id IN ( SELECT id FROM messages ORDER BY timestamp ASC LIMIT ? ) ''', (messages_to_delete,)) cursor.execute(''' UPDATE topics SET message_count = ( SELECT COUNT(*) FROM messages WHERE messages.topic = topics.topic ) ''') conn.commit() logging.info(f"Cleaned up {messages_to_delete} old messages") except Exception as e: logging.error(f"Error during cleanup: {e}") conn.rollback() def cleanup_old_data(self, days: int = 30): """Remove messages older than specified days""" cutoff_date = datetime.now() - timedelta(days=days) conn = self.get_connection() try: cursor = conn.cursor() cursor.execute('DELETE FROM messages WHERE timestamp < ?', (cutoff_date,)) deleted = cursor.rowcount cursor.execute(''' DELETE FROM topics WHERE topic NOT IN ( SELECT DISTINCT topic FROM messages ) ''') conn.commit() logging.info(f"Cleaned up {deleted} messages older than {days} days") return deleted except Exception as e: logging.error(f"Error during data cleanup: {e}") conn.rollback() return 0 def get_database_size(self) -> Dict: """Get database size information""" try: size_bytes = os.path.getsize(self.db_path) if os.path.exists(self.db_path) else 0 conn = self.get_connection() cursor = conn.cursor() cursor.execute('SELECT COUNT(*) FROM messages') message_count = cursor.fetchone()[0] cursor.execute('SELECT COUNT(*) FROM topics') topic_count = cursor.fetchone()[0] return { 'size_bytes': size_bytes, 'size_mb': round(size_bytes / 1024 / 1024, 2), 'message_count': message_count, 'topic_count': topic_count } except Exception as e: logging.error(f"Error getting database size: {e}") return {'size_bytes': 0, 'size_mb': 0, 'message_count': 0, 'topic_count': 0} def save_filter_preset(self, name: str, filters: Dict, description: str = None) -> bool: """Save a filter preset""" conn = self.get_connection() try: cursor = conn.cursor() cursor.execute(''' INSERT OR REPLACE INTO filter_presets (name, description, filters, last_used) VALUES (?, ?, ?, ?) ''', (name, description, json.dumps(filters), datetime.now())) conn.commit() logging.info(f"Saved filter preset: {name}") return True except Exception as e: logging.error(f"Error saving filter preset: {e}") conn.rollback() return False def get_filter_presets(self) -> List[Dict]: """Get all filter presets""" conn = self.get_connection() try: cursor = conn.cursor() cursor.execute(''' SELECT name, description, filters, created_at, last_used FROM filter_presets ORDER BY last_used DESC, created_at DESC ''') rows = cursor.fetchall() presets = [] for row in rows: preset = dict(row) preset['filters'] = json.loads(preset['filters']) presets.append(preset) return presets except Exception as e: logging.error(f"Error getting filter presets: {e}") return [] def delete_filter_preset(self, name: str) -> bool: """Delete a filter preset""" conn = self.get_connection() try: cursor = conn.cursor() cursor.execute('DELETE FROM filter_presets WHERE name = ?', (name,)) conn.commit() if cursor.rowcount > 0: logging.info(f"Deleted filter preset: {name}") return True return False except Exception as e: logging.error(f"Error deleting filter preset: {e}") conn.rollback() return False def use_filter_preset(self, name: str) -> Optional[Dict]: """Load and mark a filter preset as used""" conn = self.get_connection() try: cursor = conn.cursor() cursor.execute(''' SELECT filters FROM filter_presets WHERE name = ? ''', (name,)) row = cursor.fetchone() if not row: return None cursor.execute(''' UPDATE filter_presets SET last_used = ? WHERE name = ? ''', (datetime.now(), name)) conn.commit() return json.loads(row['filters']) except Exception as e: logging.error(f"Error using filter preset: {e}") return None def close(self): """Close database connection""" if hasattr(self._local, 'connection'): self._local.connection.close() ================================================ FILE: mqttui/events.py ================================================ from blinker import Namespace mqttui_signals = Namespace() # Fired when an MQTT message is received from the broker # sender: mqtt_client module, kwargs: topic, payload, timestamp, qos, retain mqtt_message_received = mqttui_signals.signal('mqtt-message-received') # Fired when an automation rule fires (Phase 3 will use this) # sender: rules engine, kwargs: rule_id, rule_name, topic, payload rule_fired = mqttui_signals.signal('rule-fired') # Fired when an alert is triggered (Phase 4 will use this) # sender: alerting module, kwargs: alert_id, rule_id, message alert_triggered = mqttui_signals.signal('alert-triggered') # Fired when a rule is created, updated, or deleted (Phase 3 hot-reload) # sender: rules API, kwargs: action ('created'|'updated'|'deleted'), rule_id rule_changed = mqttui_signals.signal('rule-changed') ================================================ FILE: mqttui/extensions.py ================================================ from flask_socketio import SocketIO from flask_sqlalchemy import SQLAlchemy from flask_login import LoginManager from flask_limiter import Limiter from flask_limiter.util import get_remote_address socketio = SocketIO() db = None # Will be set in create_app (MessageDatabase instance) sa = SQLAlchemy() login_manager = LoginManager() login_manager.login_view = 'auth.login' limiter = Limiter(key_func=get_remote_address, storage_uri="memory://") ================================================ FILE: mqttui/helpers.py ================================================ """API response helpers for consistent JSON envelope format.""" from flask import jsonify def api_success(data=None, status_code=200): """Standard success envelope.""" return jsonify({"status": "success", "data": data, "error": None}), status_code def api_error(message, code="UNKNOWN_ERROR", status_code=400): """Standard error envelope.""" return jsonify({ "status": "error", "data": None, "error": {"code": code, "message": message} }), status_code ================================================ FILE: mqttui/logging_config.py ================================================ """Structured logging configuration using structlog. Provides JSON output in production and colored console output in development. """ import structlog import logging import sys def configure_logging(debug=False, log_level='INFO'): """Configure structlog with JSON (prod) or console (dev) output. Args: debug: If True, use colored console renderer. If False, use JSON. log_level: Standard logging level name (DEBUG, INFO, WARNING, ERROR, CRITICAL). """ shared_processors = [ structlog.contextvars.merge_contextvars, structlog.stdlib.add_log_level, structlog.stdlib.add_logger_name, structlog.processors.TimeStamper(fmt="iso"), structlog.processors.StackInfoRenderer(), structlog.processors.format_exc_info, ] if debug: renderer = structlog.dev.ConsoleRenderer(colors=True) else: renderer = structlog.processors.JSONRenderer() structlog.configure( processors=shared_processors + [ structlog.stdlib.ProcessorFormatter.wrap_for_formatter, ], logger_factory=structlog.stdlib.LoggerFactory(), wrapper_class=structlog.stdlib.BoundLogger, cache_logger_on_first_use=True, ) formatter = structlog.stdlib.ProcessorFormatter( processors=[ structlog.stdlib.ProcessorFormatter.remove_processors_meta, renderer, ], ) handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) root = logging.getLogger() root.handlers.clear() root.addHandler(handler) root.setLevel(getattr(logging, log_level.upper(), logging.INFO)) ================================================ FILE: mqttui/models.py ================================================ from mqttui.extensions import sa from flask_login import UserMixin from werkzeug.security import generate_password_hash, check_password_hash import secrets class User(UserMixin, sa.Model): __tablename__ = 'users' id = sa.Column(sa.Integer, primary_key=True) username = sa.Column(sa.String(80), unique=True, nullable=False) password_hash = sa.Column(sa.String(256), nullable=False) api_token = sa.Column(sa.String(64), unique=True, nullable=True) is_active_user = sa.Column(sa.Boolean, default=True) created_at = sa.Column(sa.DateTime, server_default=sa.func.now()) def set_password(self, password): self.password_hash = generate_password_hash(password, method='pbkdf2:sha256') def check_password(self, password): return check_password_hash(self.password_hash, password) def generate_api_token(self): self.api_token = secrets.token_hex(32) return self.api_token @property def is_active(self): return self.is_active_user class TopicFavorite(sa.Model): __tablename__ = 'topic_favorites' id = sa.Column(sa.Integer, primary_key=True) user_id = sa.Column(sa.Integer, sa.ForeignKey('users.id'), nullable=False) topic = sa.Column(sa.String(500), nullable=False) created_at = sa.Column(sa.DateTime, server_default=sa.func.now()) __table_args__ = (sa.UniqueConstraint('user_id', 'topic', name='uq_user_topic'),) def to_dict(self): return { 'id': self.id, 'user_id': self.user_id, 'topic': self.topic, 'created_at': self.created_at.isoformat() if self.created_at else None, } class Broker(sa.Model): __tablename__ = 'brokers' id = sa.Column(sa.Integer, primary_key=True) name = sa.Column(sa.String(200), nullable=False) host = sa.Column(sa.String(500), nullable=False) port = sa.Column(sa.Integer, default=1883) username = sa.Column(sa.String(200), nullable=True) password = sa.Column(sa.String(200), nullable=True) mqtt_version = sa.Column(sa.String(10), default='3.1.1') topics = sa.Column(sa.String(1000), default='#') tls_enabled = sa.Column(sa.Boolean, default=False) tls_ca_certs = sa.Column(sa.String(500), nullable=True) tls_insecure = sa.Column(sa.Boolean, default=False) is_active = sa.Column(sa.Boolean, default=True) is_default = sa.Column(sa.Boolean, default=False) created_at = sa.Column(sa.DateTime, server_default=sa.func.now()) def to_dict(self): return { 'id': self.id, 'name': self.name, 'host': self.host, 'port': self.port, 'username': self.username, 'has_password': bool(self.password), 'mqtt_version': self.mqtt_version, 'topics': self.topics, 'tls_enabled': self.tls_enabled, 'tls_insecure': self.tls_insecure, 'is_active': self.is_active, 'is_default': self.is_default, 'created_at': self.created_at.isoformat() if self.created_at else None, } ================================================ FILE: mqttui/mqtt_client.py ================================================ """MQTT client compatibility layer. Delegates to BrokerManager for multi-broker support while keeping the same publish() API that rules engine and other modules use. """ import logging logger = logging.getLogger(__name__) def init_mqtt(app): """Initialize MQTT connections via BrokerManager. Call inside create_app().""" from mqttui.broker_manager import init_broker_manager init_broker_manager(app) logger.info("MQTT client initialized via BrokerManager") def get_client(): """Get the default MQTT client instance (backward compat).""" from mqttui.broker_manager import get_broker_manager mgr = get_broker_manager() if mgr: conn = mgr.get_default_connection() if conn: return conn.client return None def publish(topic, payload, qos=0, retain=False, broker_id=None): """Publish a message. Routes to specific broker or default.""" from mqttui.broker_manager import get_broker_manager mgr = get_broker_manager() if mgr: mgr.publish(topic, payload, qos=qos, retain=retain, broker_id=broker_id) ================================================ FILE: mqttui/plugins/__init__.py ================================================ ================================================ FILE: mqttui/plugins/examples/__init__.py ================================================ ================================================ FILE: mqttui/plugins/examples/json_formatter.py ================================================ #!/usr/bin/env python3 """JSON formatter example plugin for mqttui. Reads a JSON event from stdin, pretty-prints JSON payloads, and writes the result to stdout following the plugin protocol. Protocol: stdin: {"event": "on_message", "data": {"topic": "...", "payload": "..."}} stdout: {"actions": [{"type": "transform", "result": "..."}]} """ import json import sys def main(): try: line = sys.stdin.readline() if not line: print(json.dumps({"actions": []})) return envelope = json.loads(line) event = envelope.get("event", "") data = envelope.get("data", {}) if event != "on_message": print(json.dumps({"actions": []})) return payload = data.get("payload", "") try: parsed = json.loads(payload) pretty = json.dumps(parsed, indent=2) print(json.dumps({"actions": [{"type": "transform", "result": pretty}]})) except (json.JSONDecodeError, TypeError): print(json.dumps({"actions": []})) except Exception: print(json.dumps({"actions": []})) if __name__ == "__main__": main() ================================================ FILE: mqttui/plugins/examples/topic_logger.py ================================================ #!/usr/bin/env python3 """Topic logger example plugin for mqttui. Reads a JSON event from stdin, logs the topic and payload to stderr, and writes a log action to stdout following the plugin protocol. Protocol: stdin: {"event": "on_message", "data": {"topic": "...", "payload": "..."}} stdout: {"actions": [{"type": "log", "message": "Topic: ..., Payload: ..."}]} """ import json import sys def main(): try: line = sys.stdin.readline() if not line: print(json.dumps({"actions": []})) return envelope = json.loads(line) event = envelope.get("event", "") data = envelope.get("data", {}) if event != "on_message": print(json.dumps({"actions": []})) return topic = data.get("topic", "") payload = data.get("payload", "") # Log to stderr (stdout is reserved for protocol) print(f"[topic_logger] Topic: {topic}, Payload: {payload}", file=sys.stderr) # Return log action via protocol message = f"Topic: {topic}, Payload: {payload}" print(json.dumps({"actions": [{"type": "log", "message": message}]})) except Exception: print(json.dumps({"actions": []})) if __name__ == "__main__": main() ================================================ FILE: mqttui/plugins/hookspec.py ================================================ """Plugin hook specifications for mqttui.""" from __future__ import annotations import pluggy PROJECT_NAME = "mqttui" hookspec = pluggy.HookspecMarker(PROJECT_NAME) hookimpl = pluggy.HookimplMarker(PROJECT_NAME) class MQTTUIPlugin: """Hook specifications for mqttui plugins. Plugin authors implement these hooks using the @hookimpl decorator. """ @hookspec def on_message(self, topic: str, payload: str) -> dict | None: """Called when an MQTT message is received. Args: topic: The MQTT topic string. payload: The message payload as a string. Returns: Optional dict with transformed/enriched data, or None. """ @hookspec def on_connect(self) -> None: """Called when the MQTT client connects to the broker.""" @hookspec def on_rule_trigger(self, rule_name: str, topic: str, payload: str) -> dict | None: """Called when an automation rule fires. Args: rule_name: Name of the triggered rule. topic: The MQTT topic that triggered the rule. payload: The message payload. Returns: Optional dict with additional context or modifications. """ ================================================ FILE: mqttui/plugins/models.py ================================================ """SQLAlchemy model for plugin configuration persistence.""" from mqttui.extensions import sa class PluginConfig(sa.Model): __tablename__ = 'plugin_configs' id = sa.Column(sa.Integer, primary_key=True) name = sa.Column(sa.String(200), unique=True, nullable=False) entry_point = sa.Column(sa.String(500), nullable=False) enabled = sa.Column(sa.Boolean, default=False) config_json = sa.Column(sa.Text, default='{}') version = sa.Column(sa.String(50), nullable=True) description = sa.Column(sa.Text, nullable=True) installed_at = sa.Column(sa.DateTime, server_default=sa.func.now()) def to_dict(self): return { 'id': self.id, 'name': self.name, 'entry_point': self.entry_point, 'enabled': self.enabled, 'config_json': self.config_json, 'version': self.version, 'description': self.description, 'installed_at': self.installed_at.isoformat() if self.installed_at else None, } ================================================ FILE: mqttui/plugins/registry.py ================================================ """Plugin registry: discovers, loads, and manages mqttui plugins.""" from __future__ import annotations import importlib.metadata from importlib.metadata import entry_points import pluggy import structlog from mqttui.plugins.hookspec import MQTTUIPlugin, PROJECT_NAME from mqttui.plugins.models import PluginConfig from mqttui.extensions import sa logger = structlog.get_logger(__name__) class PluginRegistry: """Discovers plugins via entry_points and manages their lifecycle.""" def __init__(self, app=None): self.app = app self.pm = pluggy.PluginManager(PROJECT_NAME) self.pm.add_hookspecs(MQTTUIPlugin) def discover(self) -> list[dict]: """Scan importlib.metadata entry_points for 'mqttui.plugins' group. For each discovered entry point, upsert a PluginConfig row if not already present. Returns list of discovered plugin dicts. """ discovered = [] try: eps = entry_points(group='mqttui.plugins') except TypeError: # Python 3.9 compat: entry_points() returns a dict eps = entry_points().get('mqttui.plugins', []) for ep in eps: name = ep.name value = ep.value version = None description = None try: if ep.dist: version = ep.dist.version description = ep.dist.metadata.get('Summary', '') except Exception: pass existing = PluginConfig.query.filter_by(name=name).first() if not existing: pc = PluginConfig( name=name, entry_point=value, enabled=False, version=version, description=description, ) sa.session.add(pc) sa.session.commit() discovered.append(pc.to_dict()) logger.info("Discovered new plugin", name=name, version=version) else: # Update version/entry_point if changed existing.version = version existing.entry_point = value sa.session.commit() discovered.append(existing.to_dict()) return discovered def list_plugins(self) -> list[dict]: """Return all registered plugins as list of dicts.""" return [pc.to_dict() for pc in PluginConfig.query.all()] def get_enabled_plugins(self) -> list[PluginConfig]: """Return only plugins with enabled=True.""" return PluginConfig.query.filter_by(enabled=True).all() def enable(self, name: str) -> bool: """Enable a plugin by name. Returns False if not found.""" pc = PluginConfig.query.filter_by(name=name).first() if not pc: return False pc.enabled = True sa.session.commit() logger.info("Plugin enabled", name=name) return True def disable(self, name: str) -> bool: """Disable a plugin by name. Returns False if not found.""" pc = PluginConfig.query.filter_by(name=name).first() if not pc: return False pc.enabled = False sa.session.commit() logger.info("Plugin disabled", name=name) return True # Module-level singleton _registry = None def get_plugin_registry() -> PluginRegistry: """Return the singleton PluginRegistry instance.""" return _registry def init_plugin_registry(app) -> PluginRegistry: """Create and initialize the PluginRegistry singleton.""" global _registry _registry = PluginRegistry(app) with app.app_context(): _registry.discover() logger.info("Plugin registry initialized") return _registry ================================================ FILE: mqttui/plugins/runner.py ================================================ """Plugin runner: subprocess isolation layer for mqttui plugins. Runs plugin code in separate processes communicating via newline-delimited JSON on stdin/stdout. Plugins have no access to app internals (empty env). """ from __future__ import annotations import json import subprocess import sys import structlog import mqttui.mqtt_client from mqttui.plugins.registry import get_plugin_registry logger = structlog.get_logger(__name__) PLUGIN_TIMEOUT = 5 # seconds class PluginRunner: """Executes plugins in isolated subprocesses with JSON protocol.""" def __init__(self, app=None): self.app = app self.logger = structlog.get_logger(__name__) def call_plugin(self, plugin_config, event_type: str, data: dict) -> list[dict]: """Run a single plugin in a subprocess and return its actions. Args: plugin_config: PluginConfig model instance with entry_point. event_type: Hook name (e.g. 'on_message', 'on_rule_trigger'). data: Event data dict to pass to the plugin. Returns: List of action dicts from the plugin, or empty list on error. """ cmd = [sys.executable, "-m", plugin_config.entry_point] input_json = json.dumps({"event": event_type, "data": data}) + "\n" try: proc = subprocess.Popen( cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={}, ) stdout, stderr = proc.communicate(input=input_json, timeout=PLUGIN_TIMEOUT) if stderr: self.logger.debug( "Plugin stderr output", plugin=plugin_config.name, stderr=stderr.strip(), ) result = json.loads(stdout) return result.get("actions", []) except subprocess.TimeoutExpired: proc.kill() self.logger.warning( "Plugin timed out, killed", plugin=plugin_config.name, timeout=PLUGIN_TIMEOUT, ) return [] except json.JSONDecodeError: self.logger.warning( "Plugin returned invalid JSON", plugin=plugin_config.name, stdout=stdout[:200] if stdout else "", ) return [] except Exception as exc: self.logger.error( "Plugin execution error", plugin=plugin_config.name, error=str(exc), ) return [] def dispatch_message(self, topic: str, payload: str) -> list[dict]: """Run all enabled plugins for an MQTT message event. Returns: Aggregated list of action dicts from all plugins. """ registry = get_plugin_registry() if registry is None: return [] all_actions = [] try: if self.app: with self.app.app_context(): enabled = registry.get_enabled_plugins() else: enabled = registry.get_enabled_plugins() except (RuntimeError, Exception): enabled = [] for plugin in enabled: actions = self.call_plugin( plugin, "on_message", {"topic": topic, "payload": payload} ) all_actions.extend(actions) return all_actions def dispatch_actions(self, actions: list[dict]): """Execute action dicts returned by plugins. Supported action types: - publish: Publish an MQTT message (topic, payload). - log: Log a message via structlog. """ for action in actions: action_type = action.get("type") if action_type == "publish": mqttui.mqtt_client.publish(action["topic"], action["payload"]) elif action_type == "log": self.logger.info( "Plugin log action", message=action.get("message", ""), ) else: self.logger.warning( "Unknown plugin action type", action_type=action_type, action=action, ) def on_mqtt_message(self, sender, **kwargs): """Blinker signal handler for mqtt_message_received.""" topic = kwargs.get("topic", "") payload = kwargs.get("payload", "") payload_str = payload if isinstance(payload, str) else str(payload) actions = self.dispatch_message(topic, payload_str) if actions: self.dispatch_actions(actions) def on_rule_trigger(self, sender, **kwargs): """Blinker signal handler for rule_fired.""" rule_name = kwargs.get("rule_name", "") topic = kwargs.get("topic", "") payload = kwargs.get("payload", "") payload_str = payload if isinstance(payload, str) else str(payload) registry = get_plugin_registry() if registry is None: return all_actions = [] for plugin in registry.get_enabled_plugins(): actions = self.call_plugin( plugin, "on_rule_trigger", {"rule_name": rule_name, "topic": topic, "payload": payload_str}, ) all_actions.extend(actions) if all_actions: self.dispatch_actions(all_actions) # Module-level singleton _runner = None def get_plugin_runner() -> PluginRunner | None: """Return the singleton PluginRunner instance.""" return _runner def init_plugin_runner(app) -> PluginRunner: """Create and initialize the PluginRunner singleton.""" global _runner _runner = PluginRunner(app) logger.info("Plugin runner initialized") return _runner ================================================ FILE: mqttui/routes/__init__.py ================================================ ================================================ FILE: mqttui/routes/alerts.py ================================================ """Alert history REST API blueprint. Provides paginated, filterable listing of AlertHistory records. All endpoints require authentication via @login_required. """ from flask import Blueprint, request from flask_login import login_required from mqttui.extensions import sa from mqttui.helpers import api_success, api_error from mqttui.rules.models import AlertHistory alerts_bp = Blueprint('alerts', __name__, url_prefix='/api/v1/alerts') @alerts_bp.route('/') @login_required def list_alerts(): """List alert history with pagination and optional filters. Query params: page (int): Page number, default 1. per_page (int): Items per page, default 20, max 100. rule_id (int): Filter by rule ID. severity (str): Filter by severity level. Returns: JSON envelope with alerts list, total count, page, and per_page. """ page = request.args.get('page', 1, type=int) per_page = min(request.args.get('per_page', 20, type=int), 100) rule_id = request.args.get('rule_id', type=int) severity = request.args.get('severity', type=str) query = AlertHistory.query.order_by(AlertHistory.fired_at.desc()) if rule_id is not None: query = query.filter(AlertHistory.rule_id == rule_id) if severity: query = query.filter(AlertHistory.severity == severity) total = query.count() alerts = query.offset((page - 1) * per_page).limit(per_page).all() return api_success({ "alerts": [a.to_dict() for a in alerts], "total": total, "page": page, "per_page": per_page, }) ================================================ FILE: mqttui/routes/analytics.py ================================================ """Analytics REST API blueprint. Provides per-topic message rate and numeric payload histogram data. All endpoints return JSON envelope: {"status": "success"|"error", "data": ..., "error": ...} """ from flask import Blueprint, request from flask_login import login_required import logging from mqttui.helpers import api_success, api_error from mqttui.analytics import get_analytics analytics_bp = Blueprint('analytics', __name__, url_prefix='/api/v1/analytics') logger = logging.getLogger(__name__) @analytics_bp.route('/topics') @login_required def get_topics(): """Get top-N topic analytics sorted by message rate. Query params: limit (int): Maximum topics to return (default 20) window (int): Rate window in seconds (default 60) Returns: JSON envelope with topics array and window_seconds. """ limit = request.args.get('limit', 20, type=int) window = request.args.get('window', 60, type=int) analytics = get_analytics() topics = analytics.get_all_stats(limit=limit) return api_success({ "topics": topics, "window_seconds": window, }) @analytics_bp.route('/topics/') @login_required def get_topic(topic): """Get analytics for a single topic. Uses path converter to support MQTT topics with slashes (e.g., home/sensor/temp). Returns: JSON envelope with topic stats or 404 if no data. """ analytics = get_analytics() stats = analytics.get_topic_stats(topic) if stats["message_count"] == 0: return api_error("Topic not found", "NOT_FOUND", 404) return api_success(stats) ================================================ FILE: mqttui/routes/api.py ================================================ # TODO: Remove legacy /api/ routes in Phase 5 after frontend migrates to /api/v1/ # All new API development should use mqttui/routes/api_v1.py from flask import Blueprint, request, jsonify from datetime import datetime, timedelta import logging from mqttui import state from mqttui import extensions as ext bp = Blueprint('api', __name__) @bp.route('/api/messages') def get_message_history(): """Get paginated message history with enhanced filtering""" try: limit = min(int(request.args.get('limit', 100)), 1000) offset = int(request.args.get('offset', 0)) topic_filter = request.args.get('topic') hours = request.args.get('hours') content_search = request.args.get('content') regex_topic = request.args.get('regex_topic') json_path = request.args.get('json_path') json_value = request.args.get('json_value') since = None if hours: since = datetime.now() - timedelta(hours=int(hours)) if ext.db: messages_list = ext.db.get_messages( limit=limit, offset=offset, topic_filter=topic_filter, since=since, content_search=content_search, regex_topic=regex_topic, json_path=json_path, json_value=json_value ) total_count = ext.db.get_message_count(topic_filter=topic_filter, since=since) else: messages_list = list(reversed(state.messages)) if topic_filter: messages_list = [m for m in messages_list if m['topic'] == topic_filter] if content_search: messages_list = [m for m in messages_list if content_search.lower() in m['payload'].lower()] total_count = len(messages_list) messages_list = messages_list[offset:offset + limit] return jsonify({ 'messages': messages_list, 'total': total_count, 'limit': limit, 'offset': offset, 'has_more': offset + len(messages_list) < total_count, 'filters_applied': { 'topic': topic_filter, 'content': content_search, 'regex_topic': regex_topic, 'json_path': json_path, 'json_value': json_value, 'hours': hours } }) except Exception as e: logging.error(f"Error getting message history: {e}") return jsonify({'error': str(e)}), 500 @bp.route('/api/topics') def get_topic_list(): """Get list of all topics with statistics""" try: if ext.db: topics_list = ext.db.get_topics() else: topics_list = [{'topic': topic} for topic in sorted(state.topics)] return jsonify({'topics': topics_list}) except Exception as e: logging.error(f"Error getting topics: {e}") return jsonify({'error': str(e)}), 500 @bp.route('/api/database/stats') def get_database_stats(): """Get database size and statistics""" try: if ext.db: stats = ext.db.get_database_size() stats['enabled'] = True else: stats = { 'enabled': False, 'message_count': len(state.messages), 'topic_count': len(state.topics) } return jsonify(stats) except Exception as e: logging.error(f"Error getting database stats: {e}") return jsonify({'error': str(e)}), 500 @bp.route('/api/database/cleanup', methods=['POST']) def cleanup_database(): """Clean up old database records""" try: if not ext.db: return jsonify({'error': 'Database not enabled'}), 400 from flask import current_app days = int(request.json.get('days', current_app.config['DB_CLEANUP_DAYS'])) deleted = ext.db.cleanup_old_data(days) return jsonify({ 'success': True, 'deleted_messages': deleted, 'days': days }) except Exception as e: logging.error(f"Error cleaning database: {e}") return jsonify({'error': str(e)}), 500 @bp.route('/api/filter-presets') def get_filter_presets(): """Get all saved filter presets""" try: if not ext.db: return jsonify({'error': 'Database not enabled'}), 400 presets = ext.db.get_filter_presets() return jsonify({'presets': presets}) except Exception as e: logging.error(f"Error getting filter presets: {e}") return jsonify({'error': str(e)}), 500 @bp.route('/api/filter-presets', methods=['POST']) def save_filter_preset(): """Save a new filter preset""" try: if not ext.db: return jsonify({'error': 'Database not enabled'}), 400 data = request.json name = data.get('name') description = data.get('description', '') filters = data.get('filters', {}) if not name or not filters: return jsonify({'error': 'Name and filters are required'}), 400 success = ext.db.save_filter_preset(name, filters, description) if success: return jsonify({'success': True, 'message': f'Saved preset: {name}'}) else: return jsonify({'error': 'Failed to save preset'}), 500 except Exception as e: logging.error(f"Error saving filter preset: {e}") return jsonify({'error': str(e)}), 500 @bp.route('/api/filter-presets/', methods=['DELETE']) def delete_filter_preset(name): """Delete a filter preset""" try: if not ext.db: return jsonify({'error': 'Database not enabled'}), 400 success = ext.db.delete_filter_preset(name) if success: return jsonify({'success': True, 'message': f'Deleted preset: {name}'}) else: return jsonify({'error': 'Preset not found'}), 404 except Exception as e: logging.error(f"Error deleting filter preset: {e}") return jsonify({'error': str(e)}), 500 @bp.route('/api/filter-presets//use', methods=['POST']) def use_filter_preset(name): """Load and use a filter preset""" try: if not ext.db: return jsonify({'error': 'Database not enabled'}), 400 filters = ext.db.use_filter_preset(name) if filters: return jsonify({'success': True, 'filters': filters}) else: return jsonify({'error': 'Preset not found'}), 404 except Exception as e: logging.error(f"Error using filter preset: {e}") return jsonify({'error': str(e)}), 500 ================================================ FILE: mqttui/routes/api_v1.py ================================================ """Versioned API v1 endpoints for MQTTUI. All endpoints return JSON envelope: {"status": "success"|"error", "data": ..., "error": ...} """ from flask import Blueprint, request, jsonify, current_app from flask_login import login_required, current_user from datetime import datetime, timedelta import logging from mqttui import __version__ from mqttui import state from mqttui import extensions as ext from mqttui.extensions import sa, limiter from mqttui.helpers import api_success, api_error from mqttui.models import TopicFavorite api_v1_bp = Blueprint('api_v1', __name__, url_prefix='/api/v1') logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Messages # --------------------------------------------------------------------------- @api_v1_bp.route('/messages') @login_required def get_messages(): """Get paginated message history with filtering. --- get: summary: Retrieve MQTT messages parameters: - name: limit in: query schema: {type: integer, default: 100} - name: offset in: query schema: {type: integer, default: 0} - name: topic in: query schema: {type: string} - name: hours in: query schema: {type: integer} - name: content in: query schema: {type: string} - name: regex_topic in: query schema: {type: string} - name: json_path in: query schema: {type: string} - name: json_value in: query schema: {type: string} responses: 200: description: Paginated message list """ try: limit = min(int(request.args.get('limit', 100)), 1000) offset = int(request.args.get('offset', 0)) topic_filter = request.args.get('topic') hours = request.args.get('hours') content_search = request.args.get('content') regex_topic = request.args.get('regex_topic') json_path = request.args.get('json_path') json_value = request.args.get('json_value') since = None if hours: since = datetime.now() - timedelta(hours=int(hours)) if ext.db: messages_list = ext.db.get_messages( limit=limit, offset=offset, topic_filter=topic_filter, since=since, content_search=content_search, regex_topic=regex_topic, json_path=json_path, json_value=json_value, ) total_count = ext.db.get_message_count( topic_filter=topic_filter, since=since ) else: messages_list = list(reversed(state.messages)) if topic_filter: messages_list = [m for m in messages_list if m['topic'] == topic_filter] if content_search: messages_list = [ m for m in messages_list if content_search.lower() in m['payload'].lower() ] total_count = len(messages_list) messages_list = messages_list[offset:offset + limit] return api_success({ 'messages': messages_list, 'total': total_count, 'limit': limit, 'offset': offset, 'has_more': offset + len(messages_list) < total_count, 'filters_applied': { 'topic': topic_filter, 'content': content_search, 'regex_topic': regex_topic, 'json_path': json_path, 'json_value': json_value, 'hours': hours, } }) except Exception as e: logger.error(f"Error getting message history: {e}") return api_error(str(e), "MESSAGES_ERROR", 500) # --------------------------------------------------------------------------- # Topics # --------------------------------------------------------------------------- @api_v1_bp.route('/topics') @login_required def get_topics(): """Get list of all topics with statistics. --- get: summary: List MQTT topics responses: 200: description: Topic list """ try: if ext.db: topics_list = ext.db.get_topics() else: topics_list = [{'topic': topic} for topic in sorted(state.topics)] # Annotate with is_favorite for the current user fav_topics = set() if current_user.is_authenticated: favs = TopicFavorite.query.filter_by(user_id=current_user.id).all() fav_topics = {f.topic for f in favs} for t in topics_list: t['is_favorite'] = t.get('topic', '') in fav_topics return api_success({'topics': topics_list}) except Exception as e: logger.error(f"Error getting topics: {e}") return api_error(str(e), "TOPICS_ERROR", 500) @api_v1_bp.route('/topics/favorites') @login_required def get_favorites(): """Get current user's bookmarked topics. --- get: summary: List favorite topics responses: 200: description: Favorites list """ try: favs = TopicFavorite.query.filter_by(user_id=current_user.id).all() return api_success({'favorites': [f.to_dict() for f in favs]}) except Exception as e: logger.error(f"Error getting favorites: {e}") return api_error(str(e), "FAVORITES_ERROR", 500) @api_v1_bp.route('/topics//bookmark', methods=['POST']) @login_required def toggle_bookmark(topic): """Toggle bookmark on a topic. --- post: summary: Toggle topic bookmark parameters: - name: topic in: path required: true schema: {type: string} responses: 201: description: Bookmark created 200: description: Bookmark removed """ try: existing = TopicFavorite.query.filter_by( user_id=current_user.id, topic=topic ).first() if existing: sa.session.delete(existing) sa.session.commit() return api_success({'bookmarked': False, 'topic': topic}) else: fav = TopicFavorite(user_id=current_user.id, topic=topic) sa.session.add(fav) sa.session.commit() return api_success({'bookmarked': True, 'topic': topic}, 201) except Exception as e: sa.session.rollback() logger.error(f"Error toggling bookmark: {e}") return api_error(str(e), "BOOKMARK_ERROR", 500) # --------------------------------------------------------------------------- # Database # --------------------------------------------------------------------------- @api_v1_bp.route('/database/stats') @login_required def get_database_stats(): """Get database size and statistics. --- get: summary: Database statistics responses: 200: description: Database stats """ try: if ext.db: stats = ext.db.get_database_size() stats['enabled'] = True else: stats = { 'enabled': False, 'message_count': len(state.messages), 'topic_count': len(state.topics), } return api_success(stats) except Exception as e: logger.error(f"Error getting database stats: {e}") return api_error(str(e), "DB_STATS_ERROR", 500) @api_v1_bp.route('/database/cleanup', methods=['POST']) @login_required def cleanup_database(): """Clean up old database records. --- post: summary: Cleanup old messages requestBody: content: application/json: schema: type: object properties: days: type: integer default: 30 responses: 200: description: Cleanup result """ try: if not ext.db: return api_error("Database not enabled", "DB_DISABLED", 400) days = int(request.json.get('days', current_app.config['DB_CLEANUP_DAYS'])) deleted = ext.db.cleanup_old_data(days) return api_success({'deleted_messages': deleted, 'days': days}) except Exception as e: logger.error(f"Error cleaning database: {e}") return api_error(str(e), "DB_CLEANUP_ERROR", 500) # --------------------------------------------------------------------------- # Filter Presets # --------------------------------------------------------------------------- @api_v1_bp.route('/filter-presets') @login_required def get_filter_presets(): """Get all saved filter presets. --- get: summary: List filter presets responses: 200: description: Preset list """ try: if not ext.db: return api_error("Database not enabled", "DB_DISABLED", 400) presets = ext.db.get_filter_presets() return api_success({'presets': presets}) except Exception as e: logger.error(f"Error getting filter presets: {e}") return api_error(str(e), "PRESETS_ERROR", 500) @api_v1_bp.route('/filter-presets', methods=['POST']) @login_required def save_filter_preset(): """Save a new filter preset. --- post: summary: Create filter preset requestBody: content: application/json: schema: type: object properties: name: type: string description: type: string filters: type: object responses: 201: description: Preset saved """ try: if not ext.db: return api_error("Database not enabled", "DB_DISABLED", 400) data = request.json name = data.get('name') description = data.get('description', '') filters = data.get('filters', {}) if not name or not filters: return api_error("Name and filters are required", "VALIDATION_ERROR", 400) success = ext.db.save_filter_preset(name, filters, description) if success: return api_success({'message': f'Saved preset: {name}'}, 201) else: return api_error("Failed to save preset", "PRESET_SAVE_FAILED", 500) except Exception as e: logger.error(f"Error saving filter preset: {e}") return api_error(str(e), "PRESETS_ERROR", 500) @api_v1_bp.route('/filter-presets/', methods=['DELETE']) @login_required def delete_filter_preset(name): """Delete a filter preset. --- delete: summary: Delete a filter preset parameters: - name: name in: path required: true schema: {type: string} responses: 200: description: Preset deleted 404: description: Preset not found """ try: if not ext.db: return api_error("Database not enabled", "DB_DISABLED", 400) success = ext.db.delete_filter_preset(name) if success: return api_success({'message': f'Deleted preset: {name}'}) else: return api_error("Preset not found", "NOT_FOUND", 404) except Exception as e: logger.error(f"Error deleting filter preset: {e}") return api_error(str(e), "PRESETS_ERROR", 500) @api_v1_bp.route('/filter-presets//use', methods=['POST']) @login_required def use_filter_preset(name): """Load and use a filter preset. --- post: summary: Use a filter preset parameters: - name: name in: path required: true schema: {type: string} responses: 200: description: Preset filters 404: description: Preset not found """ try: if not ext.db: return api_error("Database not enabled", "DB_DISABLED", 400) filters = ext.db.use_filter_preset(name) if filters: return api_success({'filters': filters}) else: return api_error("Preset not found", "NOT_FOUND", 404) except Exception as e: logger.error(f"Error using filter preset: {e}") return api_error(str(e), "PRESETS_ERROR", 500) # --------------------------------------------------------------------------- # Publish # --------------------------------------------------------------------------- @api_v1_bp.route('/publish', methods=['POST']) @login_required @limiter.limit("30/minute") def publish_message(): """Publish an MQTT message (JSON API). --- post: summary: Publish MQTT message requestBody: content: application/json: schema: type: object required: [topic, message] properties: topic: type: string message: type: string responses: 200: description: Message published """ try: data = request.json if not data: return api_error("JSON body required", "VALIDATION_ERROR", 400) topic = data.get('topic') message = data.get('message') if not topic or message is None: return api_error("topic and message are required", "VALIDATION_ERROR", 400) from mqttui.mqtt_client import publish as mqtt_publish mqtt_publish(topic, message) return api_success({'published': True}) except Exception as e: logger.error(f"Error publishing message: {e}") return api_error(str(e), "PUBLISH_ERROR", 500) # --------------------------------------------------------------------------- # Stats & Version # --------------------------------------------------------------------------- @api_v1_bp.route('/stats') @login_required def get_stats(): """Get application statistics. --- get: summary: Application statistics responses: 200: description: Stats """ try: return api_success({ 'connection_count': state.connection_count, 'topic_count': len(state.topics), 'message_count': len(state.messages), 'errors': state.error_log, }) except Exception as e: logger.error(f"Error getting stats: {e}") return api_error(str(e), "STATS_ERROR", 500) @api_v1_bp.route('/version') def get_version(): """Get application version. --- get: summary: App version responses: 200: description: Version info """ return api_success({'version': __version__}) # --------------------------------------------------------------------------- # OpenAPI Documentation # --------------------------------------------------------------------------- @api_v1_bp.route('/docs') def api_docs(): """Serve Swagger UI for API documentation.""" html = """ MQTTUI API Docs
""" return html @api_v1_bp.route('/openapi.json') def openapi_spec(): """Return OpenAPI 3.0 specification.""" from apispec import APISpec from apispec_webframeworks.flask import FlaskPlugin spec = APISpec( title="MQTTUI API", version="1.0.0", openapi_version="3.0.3", info={"description": "MQTT monitoring and automation API"}, plugins=[FlaskPlugin()], ) # Register paths from current app's url_map with current_app.test_request_context(): for rule in current_app.url_map.iter_rules(): if rule.rule.startswith('/api/v1/') and rule.endpoint != 'static': view = current_app.view_functions.get(rule.endpoint) if view: spec.path(view=view, app=current_app) return jsonify(spec.to_dict()) # --------------------------------------------------------------------------- # Token CRUD # --------------------------------------------------------------------------- @api_v1_bp.route('/auth/token', methods=['GET']) @login_required def get_token(): """Get current user's API token. --- get: summary: Get API token security: [{session: []}, {apiKey: []}] responses: 200: {description: Current token} """ return api_success({"api_token": current_user.api_token}) @api_v1_bp.route('/auth/token', methods=['POST']) @login_required def regenerate_token(): """Regenerate API token. --- post: summary: Regenerate API token responses: 200: {description: New token generated} """ token = current_user.generate_api_token() sa.session.commit() return api_success({"api_token": token}) @api_v1_bp.route('/auth/token', methods=['DELETE']) @login_required def revoke_token(): """Revoke API token. --- delete: summary: Revoke API token responses: 200: {description: Token revoked} """ current_user.api_token = None sa.session.commit() return api_success({"message": "API token revoked"}) # --------------------------------------------------------------------------- # Blueprint error handlers # --------------------------------------------------------------------------- @api_v1_bp.errorhandler(429) def handle_429(e): response = api_error("Rate limit exceeded", "RATE_LIMIT_EXCEEDED", 429) # Flask-Limiter sets the description with retry info resp = response[0] resp.headers['Retry-After'] = str(60) return resp, 429 @api_v1_bp.errorhandler(404) def handle_404(e): return api_error("Resource not found", "NOT_FOUND", 404) @api_v1_bp.errorhandler(500) def handle_500(e): return api_error("Internal server error", "INTERNAL_ERROR", 500) ================================================ FILE: mqttui/routes/brokers.py ================================================ """Broker management REST API endpoints.""" from flask import Blueprint, request from flask_login import login_required from mqttui.helpers import api_success, api_error from mqttui.extensions import sa from mqttui.models import Broker from mqttui.broker_manager import get_broker_manager brokers_bp = Blueprint('brokers', __name__, url_prefix='/api/v1/brokers') @brokers_bp.route('/', methods=['GET']) @login_required def list_brokers(): """List all configured brokers with connection status.""" brokers = Broker.query.order_by(Broker.created_at).all() mgr = get_broker_manager() result = [] for b in brokers: d = b.to_dict() if mgr: conn = mgr.get_connection(b.id) d['connected'] = conn.connected if conn else False d['connection_error'] = conn.error if conn else None else: d['connected'] = False d['connection_error'] = None result.append(d) return api_success({'brokers': result}) @brokers_bp.route('/', methods=['POST']) @login_required def create_broker(): """Add a new broker connection.""" data = request.get_json() if not data or not data.get('name') or not data.get('host'): return api_error("name and host are required", "VALIDATION", 400) broker = Broker( name=data['name'], host=data['host'], port=data.get('port', 1883), username=data.get('username'), password=data.get('password'), mqtt_version=data.get('mqtt_version', '3.1.1'), topics=data.get('topics', '#'), tls_enabled=data.get('tls_enabled', False), tls_ca_certs=data.get('tls_ca_certs'), tls_insecure=data.get('tls_insecure', False), is_active=data.get('is_active', True), ) sa.session.add(broker) sa.session.commit() # Start connection if active if broker.is_active: mgr = get_broker_manager() if mgr: mgr.add_connection(broker) return api_success({'broker': broker.to_dict()}), 201 @brokers_bp.route('/', methods=['GET']) @login_required def get_broker(broker_id): """Get a single broker's details and connection status.""" broker = sa.session.get(Broker, broker_id) if not broker: return api_error("Broker not found", "NOT_FOUND", 404) d = broker.to_dict() mgr = get_broker_manager() if mgr: conn = mgr.get_connection(broker.id) d['connected'] = conn.connected if conn else False d['connection_error'] = conn.error if conn else None return api_success({'broker': d}) @brokers_bp.route('/', methods=['PUT']) @login_required def update_broker(broker_id): """Update broker configuration. Reconnects if active.""" broker = sa.session.get(Broker, broker_id) if not broker: return api_error("Broker not found", "NOT_FOUND", 404) data = request.get_json() if not data: return api_error("No data provided", "VALIDATION", 400) for field in ['name', 'host', 'port', 'username', 'password', 'mqtt_version', 'topics', 'tls_enabled', 'tls_ca_certs', 'tls_insecure', 'is_active']: if field in data: setattr(broker, field, data[field]) sa.session.commit() # Reconnect with new settings mgr = get_broker_manager() if mgr: if broker.is_active: mgr.add_connection(broker) # Stops old, starts new else: mgr.remove_connection(broker.id) return api_success({'broker': broker.to_dict()}) @brokers_bp.route('/', methods=['DELETE']) @login_required def delete_broker(broker_id): """Delete a broker and disconnect.""" broker = sa.session.get(Broker, broker_id) if not broker: return api_error("Broker not found", "NOT_FOUND", 404) mgr = get_broker_manager() if mgr: mgr.remove_connection(broker.id) sa.session.delete(broker) sa.session.commit() return api_success({'deleted': broker_id}) @brokers_bp.route('//connect', methods=['POST']) @login_required def connect_broker(broker_id): """Start/restart connection to a broker.""" broker = sa.session.get(Broker, broker_id) if not broker: return api_error("Broker not found", "NOT_FOUND", 404) broker.is_active = True sa.session.commit() mgr = get_broker_manager() if mgr: mgr.add_connection(broker) return api_success({'broker': broker.to_dict(), 'status': 'connecting'}) @brokers_bp.route('//disconnect', methods=['POST']) @login_required def disconnect_broker(broker_id): """Disconnect from a broker.""" broker = sa.session.get(Broker, broker_id) if not broker: return api_error("Broker not found", "NOT_FOUND", 404) broker.is_active = False sa.session.commit() mgr = get_broker_manager() if mgr: mgr.remove_connection(broker.id) return api_success({'broker': broker.to_dict(), 'status': 'disconnected'}) @brokers_bp.route('/status', methods=['GET']) @login_required def broker_status(): """Get live connection status for all brokers.""" mgr = get_broker_manager() statuses = mgr.all_statuses() if mgr else [] return api_success({'connections': statuses}) ================================================ FILE: mqttui/routes/debug.py ================================================ from flask import Blueprint, request, jsonify import logging from mqttui.extensions import limiter bp = Blueprint('debug', __name__) @bp.before_app_request def debug_bar_middleware(): from debug_bar import debug_bar debug_bar.start_request() debug_bar.record('request', 'path', request.path) debug_bar.record('request', 'method', request.method) @bp.after_app_request def after_request(response): from debug_bar import debug_bar debug_bar.record('request', 'status_code', response.status_code) debug_bar.end_request() return response @bp.route('/debug-bar') @limiter.exempt def get_debug_bar_data(): from debug_bar import debug_bar try: data = debug_bar.get_data() return jsonify(data) except Exception as e: logging.error(f"Error fetching debug bar data: {e}") return jsonify({"error": "Failed to fetch debug bar data"}), 500 @bp.route('/toggle-debug-bar', methods=['POST']) def toggle_debug_bar(): from debug_bar import debug_bar if debug_bar.enabled: debug_bar.disable() else: debug_bar.enable() return jsonify(enabled=debug_bar.enabled) @bp.route('/record-client-performance', methods=['POST']) def record_client_performance(): from debug_bar import debug_bar data = request.json debug_bar.record('performance', 'page_load_time', f"{data['pageLoadTime']}ms") debug_bar.record('performance', 'dom_ready_time', f"{data['domReadyTime']}ms") return jsonify(success=True) ================================================ FILE: mqttui/routes/main.py ================================================ from flask import Blueprint, render_template, request, jsonify, send_from_directory from flask_login import login_required import logging from mqttui import __version__ from mqttui.extensions import limiter from mqttui import state bp = Blueprint('main', __name__) @bp.route('/') @login_required def index(): return render_template('index.html', messages=state.messages, topics=list(state.topics)) @bp.route('/publish', methods=['POST']) @login_required def publish_message(): from mqttui.mqtt_client import publish as mqtt_publish topic = request.form['topic'] message = request.form['message'] mqtt_publish(topic, message) return jsonify(success=True) @bp.route('/stats') @limiter.exempt def get_stats(): return jsonify({ 'connection_count': state.connection_count, 'topic_count': len(state.topics), 'message_count': len(state.messages), 'errors': state.error_log }) @bp.route('/version') @limiter.exempt def get_version(): return jsonify({'version': __version__}) @bp.route('/static/') def send_static(path): return send_from_directory('static', path) # --------------------------------------------------------------------------- # Partial routes for htmx (Alerts History UI) # --------------------------------------------------------------------------- @bp.route('/partials/alerts') @login_required def alerts_list_partial(): """Return HTML partial of alert history with pagination and filters.""" from mqttui.rules.models import AlertHistory, Rule page = request.args.get('page', 1, type=int) per_page = min(request.args.get('per_page', 20, type=int), 100) rule_id = request.args.get('rule_id', type=int) severity = request.args.get('severity', type=str) query = AlertHistory.query.order_by(AlertHistory.fired_at.desc()) if rule_id: query = query.filter(AlertHistory.rule_id == rule_id) if severity: query = query.filter(AlertHistory.severity == severity) total = query.count() alerts = query.offset((page - 1) * per_page).limit(per_page).all() has_next = (page * per_page) < total # Get all rules for filter dropdown rules = Rule.query.order_by(Rule.name).all() return render_template('partials/alerts_list.html', alerts=alerts, rules=rules, page=page, per_page=per_page, total=total, has_next=has_next, current_rule_id=rule_id, current_severity=severity) # --------------------------------------------------------------------------- # Partial routes for htmx (Rules Editor UI) # --------------------------------------------------------------------------- @bp.route('/partials/rules') @login_required def rules_list_partial(): """Return HTML partial of all rules for htmx swap.""" from mqttui.rules.models import Rule rules = Rule.query.order_by(Rule.created_at.desc()).all() return render_template('partials/rules_list.html', rules=rules) @bp.route('/partials/rules//row') @login_required def rule_row_partial(rule_id): """Return single rule row partial after update.""" from mqttui.rules.models import Rule from mqttui.extensions import sa rule = sa.session.get(Rule, rule_id) if not rule: return '', 404 return render_template('partials/rule_row.html', rule=rule) @bp.route('/partials/rules/form') @login_required def rule_form_partial(): """Return empty rule creation form partial.""" return render_template('partials/rule_form.html', rule=None) @bp.route('/partials/rules//form') @login_required def rule_edit_form_partial(rule_id): """Return pre-filled rule edit form partial.""" from mqttui.rules.models import Rule from mqttui.extensions import sa rule = sa.session.get(Rule, rule_id) if not rule: return '', 404 return render_template('partials/rule_form.html', rule=rule) @bp.route('/partials/rules//dry-run') @login_required def dry_run_form_partial(rule_id): """Return dry-run test form for a rule.""" return render_template('partials/dry_run_result.html', rule_id=rule_id, result=None) @bp.route('/partials/rules//toggle', methods=['POST']) @login_required def rule_toggle_partial(rule_id): """Toggle rule enabled/disabled and return updated row partial.""" from mqttui.rules.models import Rule from mqttui.extensions import sa from mqttui.events import rule_changed rule = sa.session.get(Rule, rule_id) if not rule: return '', 404 rule.enabled = not rule.enabled sa.session.commit() rule_changed.send('ui', action='updated', rule_id=rule.id) return render_template('partials/rule_row.html', rule=rule) @bp.route('/partials/analytics') @login_required def analytics_partial(): """Return HTML partial for analytics dashboard widget.""" return render_template('partials/analytics.html') @bp.route('/partials/rules//delete', methods=['DELETE']) @login_required def rule_delete_partial(rule_id): """Delete a rule and return empty response to remove DOM element.""" from mqttui.rules.models import Rule from mqttui.extensions import sa from mqttui.events import rule_changed rule = sa.session.get(Rule, rule_id) if not rule: return '', 404 sa.session.delete(rule) sa.session.commit() rule_changed.send('ui', action='deleted', rule_id=rule_id) return '' # Empty response removes the element ================================================ FILE: mqttui/routes/metrics.py ================================================ """Prometheus-compatible /metrics endpoint and metric definitions. Exposes counters, gauges, and histograms for MQTT, rules, webhooks, and WebSocket activity. Designed for unauthenticated Prometheus scraping. """ from prometheus_client import Counter, Gauge, Histogram, generate_latest, CONTENT_TYPE_LATEST from flask import Blueprint, Response metrics_bp = Blueprint('metrics', __name__) # Counters MQTT_MESSAGES = Counter('mqtt_messages_total', 'Total MQTT messages received', ['topic']) RULE_FIRINGS = Counter('rule_firings_total', 'Total rule firings', ['rule_id']) WEBHOOK_DELIVERIES = Counter('webhook_deliveries_total', 'Total webhook deliveries', ['status']) ALERTS = Counter('alerts_total', 'Total alerts triggered') # Gauges MQTT_CONNECTED = Gauge('mqtt_connected', 'MQTT broker connection status (0/1)') ACTIVE_RULES = Gauge('active_rules', 'Number of active rules') WEBSOCKET_CLIENTS = Gauge('websocket_clients', 'Number of connected WebSocket clients') # Histograms WEBHOOK_DURATION = Histogram( 'webhook_delivery_duration_seconds', 'Webhook delivery duration', buckets=[0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0], ) @metrics_bp.route('/metrics') def prometheus_metrics(): """Serve Prometheus metrics. No authentication required for scraping.""" import mqttui.state as state MQTT_CONNECTED.set(1 if state.connection_count > 0 else 0) WEBSOCKET_CLIENTS.set(state.active_websockets) # Update active rules gauge from database try: from mqttui.rules.models import Rule ACTIVE_RULES.set(Rule.query.filter_by(enabled=True).count()) except Exception: pass return Response(generate_latest(), mimetype=CONTENT_TYPE_LATEST) # Signal subscribers for incrementing counters def _on_message_for_metrics(sender, **kwargs): """Increment mqtt_messages_total counter on MQTT message received.""" MQTT_MESSAGES.labels(topic=kwargs.get('topic', 'unknown')).inc() def _on_rule_fired_for_metrics(sender, **kwargs): """Increment rule_firings_total counter on rule fired.""" RULE_FIRINGS.labels(rule_id=str(kwargs.get('rule_id', 'unknown'))).inc() def _on_alert_for_metrics(sender, **kwargs): """Increment alerts_total counter on alert triggered.""" ALERTS.inc() ================================================ FILE: mqttui/routes/plugins.py ================================================ """Plugin management REST API and partials blueprint. Provides endpoints to list, enable, and disable plugins, plus an HTML partial for the plugins management tab. """ from flask import Blueprint, render_template from flask_login import login_required from mqttui.helpers import api_success, api_error from mqttui.plugins.registry import get_plugin_registry plugins_bp = Blueprint('plugins', __name__) @plugins_bp.route('/api/v1/plugins') @login_required def list_plugins(): """List all installed plugins. Returns: JSON envelope with plugins list. """ registry = get_plugin_registry() plugins = registry.list_plugins() if registry else [] return api_success({"plugins": plugins}) @plugins_bp.route('/api/v1/plugins//enable', methods=['POST']) @login_required def enable_plugin(name): """Enable a plugin by name. Returns: JSON success or 404 if plugin not found. """ registry = get_plugin_registry() if not registry or not registry.enable(name): return api_error("Plugin not found", "NOT_FOUND", 404) return api_success({"name": name, "enabled": True}) @plugins_bp.route('/api/v1/plugins//disable', methods=['POST']) @login_required def disable_plugin(name): """Disable a plugin by name. Returns: JSON success or 404 if plugin not found. """ registry = get_plugin_registry() if not registry or not registry.disable(name): return api_error("Plugin not found", "NOT_FOUND", 404) return api_success({"name": name, "enabled": False}) @plugins_bp.route('/partials/plugins') @login_required def plugins_partial(): """Render plugins management HTML partial. Returns: HTML partial with plugin list and enable/disable controls. """ registry = get_plugin_registry() plugins = registry.list_plugins() if registry else [] return render_template('partials/plugins.html', plugins=plugins) ================================================ FILE: mqttui/routes/rules.py ================================================ """Rules CRUD REST API blueprint. All endpoints return JSON envelope: {"status": "success"|"error", "data": ..., "error": ...} All endpoints require @login_required authentication. """ from flask import Blueprint, request from flask_login import login_required import json import logging from mqttui.extensions import sa from mqttui.helpers import api_success, api_error from mqttui.rules.models import Rule from mqttui.rules.evaluator import evaluate, ConditionError from mqttui.rules.ssrf import is_ssrf_safe from mqttui.events import rule_changed rules_bp = Blueprint('rules', __name__, url_prefix='/api/v1/rules') logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # List / Create # --------------------------------------------------------------------------- @rules_bp.route('/') @login_required def list_rules(): """List all rules ordered by creation date (newest first).""" rules = Rule.query.order_by(Rule.created_at.desc()).all() return api_success({"rules": [r.to_dict() for r in rules]}) @rules_bp.route('/', methods=['POST']) @login_required def create_rule(): """Create a new rule. Required fields: name, trigger_topic, action (dict with 'type' key). Optional: description, condition, rate_limit_per_min, schedule_cron, enabled. """ data = request.get_json(silent=True) if not data: return api_error("JSON body required", "VALIDATION_ERROR", 400) # Validate required fields name = data.get('name') if not name or not isinstance(name, str) or not name.strip(): return api_error("name is required", "VALIDATION_ERROR", 400) trigger_topic = data.get('trigger_topic') if not trigger_topic or not isinstance(trigger_topic, str) or not trigger_topic.strip(): return api_error("trigger_topic is required", "VALIDATION_ERROR", 400) action = data.get('action') if not action or not isinstance(action, dict) or 'type' not in action: return api_error("action is required and must have a 'type' key", "VALIDATION_ERROR", 400) # SSRF validation for webhook actions if action.get('type') == 'webhook': webhook_url = action.get('url', '') safe, reason = is_ssrf_safe(webhook_url) if not safe: return api_error(f"Webhook URL rejected: {reason}", "SSRF_BLOCKED", 400) # Build rule rule = Rule( name=name.strip(), description=data.get('description', ''), trigger_topic=trigger_topic.strip(), condition_json=json.dumps(data.get('condition', {})), action_json=json.dumps(action), enabled=data.get('enabled', True), rate_limit_per_min=data.get('rate_limit_per_min', 10), schedule_cron=data.get('schedule_cron'), ) sa.session.add(rule) sa.session.commit() rule_changed.send('api', action='created', rule_id=rule.id) logger.info(f"Rule created: id={rule.id} name={rule.name}") return api_success(rule.to_dict(), 201) # --------------------------------------------------------------------------- # Single rule CRUD # --------------------------------------------------------------------------- def _get_rule_or_404(rule_id): """Helper to fetch a rule by ID or return a 404 error response.""" rule = sa.session.get(Rule, rule_id) if not rule: return None, api_error("Rule not found", "NOT_FOUND", 404) return rule, None @rules_bp.route('/') @login_required def get_rule(rule_id): """Get a single rule by ID.""" rule, err = _get_rule_or_404(rule_id) if err: return err return api_success(rule.to_dict()) @rules_bp.route('/', methods=['PUT']) @login_required def update_rule(rule_id): """Update a rule. Only updates fields present in the request body.""" rule, err = _get_rule_or_404(rule_id) if err: return err data = request.get_json(silent=True) if not data: return api_error("JSON body required", "VALIDATION_ERROR", 400) # SSRF validation for webhook actions if 'action' in data and isinstance(data['action'], dict) and data['action'].get('type') == 'webhook': webhook_url = data['action'].get('url', '') safe, reason = is_ssrf_safe(webhook_url) if not safe: return api_error(f"Webhook URL rejected: {reason}", "SSRF_BLOCKED", 400) # Update only fields present in request if 'name' in data: rule.name = data['name'] if 'description' in data: rule.description = data['description'] if 'trigger_topic' in data: rule.trigger_topic = data['trigger_topic'] if 'condition' in data: rule.condition_json = json.dumps(data['condition']) if 'action' in data: rule.action_json = json.dumps(data['action']) if 'enabled' in data: rule.enabled = data['enabled'] if 'rate_limit_per_min' in data: rule.rate_limit_per_min = data['rate_limit_per_min'] if 'schedule_cron' in data: rule.schedule_cron = data['schedule_cron'] sa.session.commit() rule_changed.send('api', action='updated', rule_id=rule.id) logger.info(f"Rule updated: id={rule.id}") return api_success(rule.to_dict()) @rules_bp.route('/', methods=['DELETE']) @login_required def delete_rule(rule_id): """Delete a rule by ID.""" rule, err = _get_rule_or_404(rule_id) if err: return err sa.session.delete(rule) sa.session.commit() rule_changed.send('api', action='deleted', rule_id=rule_id) logger.info(f"Rule deleted: id={rule_id}") return api_success({"message": f"Rule {rule_id} deleted"}) # --------------------------------------------------------------------------- # Enable / Disable # --------------------------------------------------------------------------- @rules_bp.route('//enable', methods=['POST']) @login_required def enable_rule(rule_id): """Enable a rule.""" rule, err = _get_rule_or_404(rule_id) if err: return err rule.enabled = True sa.session.commit() rule_changed.send('api', action='updated', rule_id=rule.id) return api_success(rule.to_dict()) @rules_bp.route('//disable', methods=['POST']) @login_required def disable_rule(rule_id): """Disable a rule.""" rule, err = _get_rule_or_404(rule_id) if err: return err rule.enabled = False sa.session.commit() rule_changed.send('api', action='updated', rule_id=rule.id) return api_success(rule.to_dict()) # --------------------------------------------------------------------------- # Dry-run / Test # --------------------------------------------------------------------------- @rules_bp.route('//test', methods=['POST']) @login_required def test_rule(rule_id): """Dry-run a rule against a sample topic + payload. Checks topic matching (MQTT wildcard) and condition evaluation. Does NOT fire the rule's action -- purely for testing. """ rule, err = _get_rule_or_404(rule_id) if err: return err data = request.get_json(silent=True) if not data: return api_error("JSON body required", "VALIDATION_ERROR", 400) topic = data.get('topic') payload_str = data.get('payload') if not topic: return api_error("topic is required", "VALIDATION_ERROR", 400) if payload_str is None: return api_error("payload is required", "VALIDATION_ERROR", 400) # Parse payload as JSON if possible payload_dict = None if isinstance(payload_str, dict): payload_dict = payload_str elif isinstance(payload_str, str): try: payload_dict = json.loads(payload_str) except (json.JSONDecodeError, ValueError): payload_dict = None # Check MQTT topic matching (supports wildcards + and #) topic_matches = _topic_matches(rule.trigger_topic, topic) # Check condition evaluation condition = json.loads(rule.condition_json) if rule.condition_json else {} condition_matches = False try: condition_matches = evaluate(condition, payload_dict) except ConditionError as e: return api_error(f"Condition error: {e}", "CONDITION_ERROR", 400) matched = topic_matches and condition_matches actions = [json.loads(rule.action_json)] if matched else [] # Build action_preview when rule matches action_preview = [] if matched and actions: for action in actions: preview = _build_action_preview(action, { 'rule_id': rule.id, 'rule_name': rule.name, 'topic': topic, 'payload': payload_str, }) if preview: action_preview.append(preview) return api_success({ "match": matched, "topic_match": topic_matches, "condition_match": condition_matches, "actions": actions, "action_preview": action_preview, }) def _topic_matches(pattern, topic): """Check if an MQTT topic matches a subscription pattern. Supports MQTT wildcards: - '+' matches a single level - '#' matches any remaining levels (must be last) """ pattern_parts = pattern.split('/') topic_parts = topic.split('/') for i, p in enumerate(pattern_parts): if p == '#': return True # '#' matches everything from here if i >= len(topic_parts): return False if p != '+' and p != topic_parts[i]: return False return len(pattern_parts) == len(topic_parts) def _build_action_preview(action, context): """Build a preview of what an action would produce. Args: action: Parsed action dict with 'type' key. context: dict with rule_id, rule_name, topic, payload. Returns: Preview dict with type-specific fields, or None. """ action_type = action.get('type') if action_type == 'webhook': from mqttui.rules.actions import _build_webhook_payload, _build_default_payload template = action.get('payload_template') if template: payload = _build_webhook_payload(template, context) else: payload = _build_default_payload(context) return { 'type': 'webhook', 'url': action.get('url', ''), 'payload': payload, } elif action_type == 'publish': return { 'type': 'publish', 'topic': action.get('topic', ''), 'payload': action.get('payload', context.get('payload', '')), } elif action_type == 'log': message = action.get( 'message', f"Rule {context.get('rule_name', '')} fired on {context.get('topic', '')}", ) return { 'type': 'log', 'severity': action.get('severity', 'info'), 'message': message, } return None ================================================ FILE: mqttui/rules/__init__.py ================================================ from mqttui.rules.engine import RuleEngine ================================================ FILE: mqttui/rules/actions.py ================================================ """Action executor for the rules engine. Handles publish, log, and webhook action types. Webhook delivery uses httpx in a thread pool for non-blocking HTTP POST. """ import json import logging import time from concurrent.futures import ThreadPoolExecutor from datetime import datetime import httpx logger = logging.getLogger(__name__) # Module-level thread pool for webhook delivery (non-blocking) _webhook_executor = ThreadPoolExecutor(max_workers=4) def execute_action(action_dict, context): """Execute a rule action. Args: action_dict: parsed action JSON with 'type' key. - type 'publish': publishes MQTT message with __source marker - type 'log': creates AlertHistory record in database - type 'webhook': HTTP POST with retry in thread pool context: dict with 'rule_id', 'rule_name', 'topic', 'payload' keys. Returns: dict with 'success' bool and 'detail' string. """ action_type = action_dict.get('type') if action_type == 'publish': return _execute_publish(action_dict, context) elif action_type == 'log': return _execute_log(action_dict, context) elif action_type == 'webhook': return _execute_webhook(action_dict, context) elif action_type == 'telegram': return _execute_telegram(action_dict, context) elif action_type == 'slack': return _execute_slack(action_dict, context) else: return {"success": False, "detail": f"Unknown action type: {action_type}"} def _execute_publish(action_dict, context): """Publish an MQTT message with __source marker for loop prevention.""" from mqttui.mqtt_client import publish as mqtt_publish # Use action payload if specified, otherwise use original context payload outgoing = action_dict.get('payload', context.get('payload')) # Parse as JSON if possible, inject __source marker if isinstance(outgoing, str): try: outgoing_dict = json.loads(outgoing) except (json.JSONDecodeError, TypeError): # Raw string -- wrap in JSON envelope outgoing_dict = {"payload": outgoing} elif isinstance(outgoing, dict): outgoing_dict = outgoing else: outgoing_dict = {"payload": str(outgoing)} outgoing_dict["__source"] = "mqttui-automation" mqtt_publish( action_dict['topic'], json.dumps(outgoing_dict), qos=action_dict.get('qos', 0), retain=action_dict.get('retain', False), ) return {"success": True, "detail": f"Published to {action_dict['topic']}"} def _execute_log(action_dict, context): """Create an AlertHistory record in the database.""" from mqttui.rules.models import AlertHistory from mqttui.extensions import sa message = action_dict.get( 'message', f"Rule {context['rule_name']} fired on {context['topic']}", ) record = AlertHistory( rule_id=context['rule_id'], rule_name=context['rule_name'], topic=context['topic'], severity=action_dict.get('severity', 'info'), message=message, fired_at=datetime.utcnow(), ) sa.session.add(record) sa.session.commit() return {"success": True, "detail": "Alert logged"} def _execute_webhook(action_dict, context): """Submit webhook delivery to thread pool for async execution. Checks per-rule cooldown before submitting. If the rule is within its cooldown window the webhook is suppressed and an AlertHistory record is created with the incremented suppressed_count. Returns immediately with submission confirmation. Actual HTTP delivery happens in background thread with retry logic. """ from mqttui.rules.cooldown import cooldown_tracker rule_id = context['rule_id'] # Check cooldown before submitting if not cooldown_tracker.check(rule_id): suppressed_count = cooldown_tracker.get_suppressed_count(rule_id) cooldown_until = cooldown_tracker.get_cooldown_until(rule_id) # Log suppressed alert to history _log_suppressed_alert( rule_id=rule_id, rule_name=context['rule_name'], topic=context['topic'], url=action_dict.get('url', ''), suppressed_count=suppressed_count, cooldown_until=cooldown_until, ) return {"success": True, "detail": f"Alert suppressed (cooldown), {suppressed_count} suppressed"} url = action_dict.get('url', '') payload_template = action_dict.get('payload_template') # Build payload if payload_template: payload_json = _build_webhook_payload(payload_template, context) else: payload_json = _build_default_payload(context) # Get Flask app for thread pool context try: from flask import current_app app = current_app._get_current_object() except RuntimeError: app = None # Submit to thread pool -- non-blocking _webhook_executor.submit( _deliver_webhook, url=url, payload_json=payload_json, rule_id=context['rule_id'], rule_name=context['rule_name'], topic=context['topic'], app=app, ) return {"success": True, "detail": "Webhook delivery submitted"} def _build_default_payload(context): """Build the default webhook JSON payload.""" return { "topic": context.get('topic', ''), "payload": context.get('payload', ''), "rule_name": context.get('rule_name', ''), "timestamp": datetime.utcnow().isoformat(), "mqttui_source": True, } def _build_webhook_payload(template, context): """Build webhook payload from a template string with {{variable}} substitution. Supports: {{topic}}, {{payload}}, {{rule_name}}, {{timestamp}} Args: template: JSON string with {{variable}} placeholders. context: dict with rule_id, rule_name, topic, payload. Returns: Parsed dict from the substituted template. """ substitutions = { '{{topic}}': str(context.get('topic', '')), '{{payload}}': str(context.get('payload', '')), '{{rule_name}}': str(context.get('rule_name', '')), '{{timestamp}}': datetime.utcnow().isoformat(), } result = template for placeholder, value in substitutions.items(): result = result.replace(placeholder, value) try: return json.loads(result) except (json.JSONDecodeError, TypeError): return {"raw": result} def _deliver_webhook(url, payload_json, rule_id, rule_name, topic, max_retries=3, _sleep_fn=None, app=None): """Deliver webhook HTTP POST with retry logic. Args: url: Destination URL for the POST request. payload_json: Dict payload to send as JSON. rule_id: Rule ID for AlertHistory logging. rule_name: Rule name for AlertHistory logging. topic: MQTT topic that triggered the rule. max_retries: Maximum retry attempts on 5xx/connection errors. _sleep_fn: Override sleep function for testing (default: time.sleep). app: Flask app instance for creating app context in thread pool. Returns: dict with 'success' bool and 'detail' string. """ from mqttui.rules.models import AlertHistory from mqttui.extensions import sa from mqttui.events import alert_triggered # Push app context for this thread (thread pool workers have none) ctx = None if app is not None: ctx = app.app_context() ctx.push() sleep_fn = _sleep_fn or time.sleep last_status = None last_error = None retries = 0 for attempt in range(1 + max_retries): try: response = httpx.post(url, json=payload_json, timeout=10.0) last_status = response.status_code if 200 <= response.status_code < 300: # Success _log_webhook_history( sa, rule_id, rule_name, topic, url, http_status=response.status_code, retry_count=attempt, ) alert_triggered.send( 'webhook', alert_id=None, rule_id=rule_id, message=f"Webhook delivered to {url}", ) try: from mqttui.routes.metrics import WEBHOOK_DELIVERIES WEBHOOK_DELIVERIES.labels(status='success').inc() except Exception: pass logger.info(f"Webhook delivered: rule={rule_name} url={url} status={response.status_code}") if ctx is not None: ctx.pop() return {"success": True, "detail": f"Webhook delivered (HTTP {response.status_code})"} elif 400 <= response.status_code < 500: # Client error -- no retry last_error = f"HTTP {response.status_code}: {response.text[:200]}" logger.warning(f"Webhook client error: rule={rule_name} url={url} status={response.status_code}") _log_webhook_history( sa, rule_id, rule_name, topic, url, http_status=response.status_code, retry_count=0, error_detail=last_error, ) if ctx is not None: ctx.pop() return {"success": False, "detail": last_error} else: # 5xx or other -- retry last_error = f"HTTP {response.status_code}: {response.text[:200]}" retries = attempt if attempt < max_retries: backoff = 5 ** attempt # 1s, 5s, 25s logger.info(f"Webhook retry {attempt + 1}/{max_retries}: rule={rule_name} backoff={backoff}s") sleep_fn(backoff) except (httpx.ConnectError, httpx.TimeoutException, OSError) as e: last_error = str(e) retries = attempt if attempt < max_retries: backoff = 5 ** attempt logger.info(f"Webhook retry {attempt + 1}/{max_retries}: rule={rule_name} error={e}") sleep_fn(backoff) # Exhausted retries _log_webhook_history( sa, rule_id, rule_name, topic, url, http_status=last_status, retry_count=retries, error_detail=last_error, ) try: from mqttui.routes.metrics import WEBHOOK_DELIVERIES WEBHOOK_DELIVERIES.labels(status='failure').inc() except Exception: pass logger.error(f"Webhook failed after {retries} retries: rule={rule_name} url={url}") if ctx is not None: ctx.pop() return {"success": False, "detail": f"Webhook failed after {retries} retries: {last_error}"} def _execute_telegram(action_dict, context): """Send a Telegram message via Bot API. Converts to a webhook action targeting the Telegram Bot API, then delegates to _execute_webhook for cooldown, retry, alert history, and metrics. """ bot_token = action_dict.get('bot_token', '') chat_id = action_dict.get('chat_id', '') template = action_dict.get('message_template', '🔔 *MQTT Alert*\n*Rule:* {{rule_name}}\n*Topic:* `{{topic}}`\n*Payload:* `{{payload}}`') if not bot_token or not chat_id: return {"success": False, "detail": "Telegram requires bot_token and chat_id"} # Substitute placeholders message = _substitute_template(template, context) url = f"https://api.telegram.org/bot{bot_token}/sendMessage" payload_template = json.dumps({"chat_id": chat_id, "text": message, "parse_mode": "Markdown"}) # Delegate to webhook with full cooldown + retry + alert logging return _execute_webhook({'url': url, 'payload_template': payload_template}, context) def _execute_slack(action_dict, context): """Send a Slack message via incoming webhook URL. Converts to a webhook action, then delegates to _execute_webhook for cooldown, retry, alert history, and metrics. """ webhook_url = action_dict.get('webhook_url', '') template = action_dict.get('message_template', '🔔 *MQTT Alert*\n>*Rule:* {{rule_name}}\n>*Topic:* `{{topic}}`\n>*Payload:* `{{payload}}`') if not webhook_url: return {"success": False, "detail": "Slack requires webhook_url"} message = _substitute_template(template, context) payload_template = json.dumps({"text": message}) return _execute_webhook({'url': webhook_url, 'payload_template': payload_template}, context) def _substitute_template(template, context): """Replace {{variable}} placeholders in a template string.""" result = template for placeholder, value in { '{{topic}}': str(context.get('topic', '')), '{{payload}}': str(context.get('payload', '')), '{{rule_name}}': str(context.get('rule_name', '')), '{{timestamp}}': datetime.utcnow().isoformat(), }.items(): result = result.replace(placeholder, value) return result def _log_suppressed_alert(rule_id, rule_name, topic, url, suppressed_count, cooldown_until): """Create an AlertHistory record for a suppressed (cooldown) alert.""" from mqttui.rules.models import AlertHistory from mqttui.extensions import sa try: record = AlertHistory( rule_id=rule_id, rule_name=rule_name, topic=topic, severity='info', message=f"Webhook to {url} suppressed (cooldown)", fired_at=datetime.utcnow(), webhook_url=url, suppressed_count=suppressed_count, cooldown_until=cooldown_until, ) sa.session.add(record) sa.session.commit() except Exception as e: logger.error(f"Failed to log suppressed alert: {e}") try: sa.session.rollback() except Exception: pass def _log_webhook_history(sa, rule_id, rule_name, topic, url, http_status=None, retry_count=0, error_detail=None): """Create an AlertHistory record for webhook delivery.""" from mqttui.rules.models import AlertHistory try: record = AlertHistory( rule_id=rule_id, rule_name=rule_name, topic=topic, severity='info' if http_status and http_status < 400 else 'error', message=f"Webhook to {url}", fired_at=datetime.utcnow(), webhook_url=url, http_status=http_status, retry_count=retry_count, error_detail=error_detail, ) sa.session.add(record) sa.session.commit() except Exception as e: logger.error(f"Failed to log webhook history: {e}") try: sa.session.rollback() except Exception: pass ================================================ FILE: mqttui/rules/cooldown.py ================================================ """Per-rule alert cooldown tracker for deduplication. Prevents alert storms from sustained conditions by enforcing a minimum interval between alert firings for each rule. Suppressed alerts are counted so the suppression can be reported in AlertHistory. """ import time from datetime import datetime, timedelta class CooldownTracker: """In-memory per-rule cooldown tracker. Tracks the last time each rule fired and blocks subsequent firings within the cooldown window. Thread-safe for use from the webhook thread pool (GIL protects dict operations). """ def __init__(self, default_seconds=300): """Initialize with a default cooldown window. Args: default_seconds: Default cooldown window in seconds (default 5 min). """ self._last_fired = {} # rule_id -> monotonic timestamp self._suppressed = {} # rule_id -> suppressed count since last fire self._default = default_seconds def check(self, rule_id, cooldown_seconds=None): """Check if a rule is allowed to fire. Args: rule_id: The rule identifier. cooldown_seconds: Optional per-rule cooldown override. Returns: True if the rule is allowed to fire, False if suppressed. """ window = cooldown_seconds if cooldown_seconds is not None else self._default now = time.monotonic() last = self._last_fired.get(rule_id) if last is None or (now - last) > window: # Allowed -- record firing and reset suppressed count self._last_fired[rule_id] = now self._suppressed[rule_id] = 0 return True # Suppressed self._suppressed[rule_id] = self._suppressed.get(rule_id, 0) + 1 return False def get_suppressed_count(self, rule_id): """Return the number of suppressed alerts since the last firing. Args: rule_id: The rule identifier. Returns: Number of suppressed alerts (0 if rule has never been suppressed). """ return self._suppressed.get(rule_id, 0) def get_cooldown_until(self, rule_id, cooldown_seconds=None): """Return the datetime when the cooldown expires for a rule. Args: rule_id: The rule identifier. cooldown_seconds: Optional per-rule cooldown override. Returns: datetime when cooldown expires, or None if no active cooldown. """ window = cooldown_seconds if cooldown_seconds is not None else self._default last = self._last_fired.get(rule_id) if last is None: return None elapsed = time.monotonic() - last remaining = window - elapsed if remaining <= 0: return None return datetime.utcnow() + timedelta(seconds=remaining) # Module-level singleton used by the action executor cooldown_tracker = CooldownTracker() ================================================ FILE: mqttui/rules/engine.py ================================================ """RuleEngine -- runtime heart of the rules automation system. Subscribes to mqtt_message_received events, evaluates matching rules, executes actions, and enforces safety mechanisms (loop prevention, per-rule rate limiting, global circuit breaker). Also manages time-based rules via APScheduler and hot-reloads rule cache when rules are created, updated, or deleted. """ import json import logging import time from collections import deque from datetime import datetime from paho.mqtt.matcher import MQTTMatcher from mqttui.events import mqtt_message_received, rule_fired, rule_changed from mqttui.rules.evaluator import evaluate, ConditionError from mqttui.rules.actions import execute_action logger = logging.getLogger(__name__) # Module-level singleton reference for APScheduler pickle compatibility _engine = None def _fire_scheduled_rule(app, rule_id): """Module-level function for APScheduler pickle compatibility. APScheduler serialises job targets; lambdas and bound methods cannot be pickled, so we use a plain module-level function with explicit args. """ with app.app_context(): if _engine: _engine.fire_scheduled_rule(rule_id) def get_engine(): """Return the module-level RuleEngine singleton (or None).""" return _engine class RuleEngine: """Evaluate automation rules against live MQTT messages. Attributes: _rules: dict mapping rule_id -> Rule ORM instance (enabled only) _matcher: MQTTMatcher mapping subscription patterns to rule_ids _rule_timestamps: dict mapping rule_id -> deque of firing timestamps _global_timestamps: deque of all firing timestamps (circuit breaker) _GLOBAL_LIMIT: max total firings per window (default 100) _WINDOW: sliding window duration in seconds (default 60) _scheduler: GeventScheduler instance (or None before init_scheduler) """ def __init__(self, app=None): self._rules = {} self._matcher = MQTTMatcher() self._rule_timestamps = {} self._global_timestamps = deque() self._GLOBAL_LIMIT = 100 self._WINDOW = 60.0 self._app = app self._scheduler = None # ------------------------------------------------------------------ # Scheduler # ------------------------------------------------------------------ def init_scheduler(self): """Start the APScheduler GeventScheduler with SQLAlchemy job store.""" from apscheduler.schedulers.gevent import GeventScheduler from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore uri = self._app.config['SQLALCHEMY_DATABASE_URI'] self._scheduler = GeventScheduler( jobstores={'default': SQLAlchemyJobStore(url=uri)}, job_defaults={ 'misfire_grace_time': 30, 'max_instances': 1, 'coalesce': True, }, ) self._scheduler.start() logger.info("APScheduler GeventScheduler started") def fire_scheduled_rule(self, rule_id): """Execute a time-based rule by id (called from scheduler job).""" rule = self._rules.get(rule_id) if rule is None or not rule.enabled: logger.debug(f"Scheduled rule {rule_id} skipped (not found or disabled)") return # Rate limit check if not self._check_rate_limit(rule): logger.debug(f"Scheduled rule {rule_id} rate-limited") return # Build action and context try: action_dict = json.loads(rule.action_json) if rule.action_json else {} except json.JSONDecodeError: logger.error(f"Invalid action JSON for scheduled rule {rule_id}") return context = { 'rule_id': rule.id, 'rule_name': rule.name, 'topic': '__scheduled__', 'payload': '', } try: result = execute_action(action_dict, context) if not result.get('success'): logger.warning(f"Scheduled action failed for rule {rule_id}: {result.get('detail')}") except Exception as e: logger.error(f"Scheduled action error for rule {rule_id}: {e}") return # Update rule stats try: from mqttui.extensions import sa rule.fire_count = (rule.fire_count or 0) + 1 rule.last_fired = datetime.utcnow() sa.session.commit() except Exception as e: logger.error(f"Failed to update stats for scheduled rule {rule_id}: {e}") # Fire signal rule_fired.send( self, rule_id=rule.id, rule_name=rule.name, topic='__scheduled__', payload='', ) def sync_scheduled_jobs(self): """Synchronise APScheduler jobs with the current rule cache. - New cron rules get jobs added. - Deleted rules get jobs removed. - Changed cron expressions are updated (replace_existing=True). """ if self._scheduler is None: return from apscheduler.triggers.cron import CronTrigger existing_jobs = {job.id for job in self._scheduler.get_jobs()} for rule in self._rules.values(): job_id = f'rule_{rule.id}' if rule.schedule_cron and rule.schedule_cron.strip(): self._scheduler.add_job( _fire_scheduled_rule, trigger=CronTrigger.from_crontab(rule.schedule_cron), args=[self._app, rule.id], id=job_id, replace_existing=True, ) existing_jobs.discard(job_id) else: # Rule has no cron -- remove stale job if present if job_id in existing_jobs: self._scheduler.remove_job(job_id) existing_jobs.discard(job_id) # Cleanup jobs for rules that were deleted for job_id in existing_jobs: if job_id.startswith('rule_'): self._scheduler.remove_job(job_id) logger.debug("Scheduler jobs synced with rule cache") # ------------------------------------------------------------------ # Cache # ------------------------------------------------------------------ def reload_cache(self): """Load enabled rules from DB into in-memory cache and rebuild matcher.""" from mqttui.rules.models import Rule ctx = self._app.app_context() if self._app else _nullcontext() with ctx: enabled_rules = Rule.query.filter_by(enabled=True).all() new_rules = {} new_matcher = MQTTMatcher() for rule in enabled_rules: new_rules[rule.id] = rule # MQTTMatcher stores value at subscription key # Multiple rules can share a topic, so store list try: existing = new_matcher[rule.trigger_topic] existing.append(rule.id) except KeyError: new_matcher[rule.trigger_topic] = [rule.id] # Atomic swap self._rules = new_rules self._matcher = new_matcher # Clean stale rate-limit entries active_ids = set(new_rules.keys()) stale_ids = set(self._rule_timestamps.keys()) - active_ids for rid in stale_ids: del self._rule_timestamps[rid] logger.info(f"RuleEngine cache reloaded: {len(new_rules)} enabled rules") # Sync scheduler jobs after cache reload self.sync_scheduled_jobs() # ------------------------------------------------------------------ # Rate limiting # ------------------------------------------------------------------ def _check_rate_limit(self, rule): """Check per-rule and global rate limits using sliding window. Returns True if the rule is allowed to fire, False if blocked. On allow, records the timestamp in both deques. """ now = time.monotonic() cutoff = now - self._WINDOW # --- Global circuit breaker --- while self._global_timestamps and self._global_timestamps[0] < cutoff: self._global_timestamps.popleft() if len(self._global_timestamps) >= self._GLOBAL_LIMIT: logger.warning("Global circuit breaker triggered") return False # --- Per-rule rate limit --- if rule.id not in self._rule_timestamps: self._rule_timestamps[rule.id] = deque() rule_deque = self._rule_timestamps[rule.id] while rule_deque and rule_deque[0] < cutoff: rule_deque.popleft() if len(rule_deque) >= rule.rate_limit_per_min: logger.debug(f"Rate limit hit for rule {rule.id} ({rule.name})") return False # Allowed -- record timestamp rule_deque.append(now) self._global_timestamps.append(now) return True # ------------------------------------------------------------------ # MQTT message handler # ------------------------------------------------------------------ def on_mqtt_message(self, sender, **kwargs): """Handle an incoming MQTT message from the event bus. 1. Parse payload, check for loop marker 2. Match topic against cached rules 3. For each match: check rate limit, evaluate condition, execute action 4. Fire rule_fired signal on success """ topic = kwargs.get('topic', '') payload_raw = kwargs.get('payload', '') # Parse payload try: payload_dict = json.loads(payload_raw) except (json.JSONDecodeError, TypeError): payload_dict = {} # Loop prevention: skip messages from our own automation if isinstance(payload_dict, dict) and payload_dict.get('__source') == 'mqttui-automation': logger.debug(f"Skipping automation message on {topic}") return # Find matching rules via MQTTMatcher matched_rule_ids = [] try: for rule_id_list in self._matcher.iter_match(topic): if isinstance(rule_id_list, list): matched_rule_ids.extend(rule_id_list) else: matched_rule_ids.append(rule_id_list) except (StopIteration, KeyError): return if not matched_rule_ids: return ctx = self._app.app_context() if self._app else _nullcontext() with ctx: for rule_id in matched_rule_ids: rule = self._rules.get(rule_id) if rule is None: continue # Rate limit check if not self._check_rate_limit(rule): continue # Evaluate condition try: condition = json.loads(rule.condition_json) if rule.condition_json else {} except json.JSONDecodeError: logger.error(f"Invalid condition JSON for rule {rule.id}") continue try: if not evaluate(condition, payload_dict): # Condition did not match -- undo rate limit recording if rule.id in self._rule_timestamps and self._rule_timestamps[rule.id]: self._rule_timestamps[rule.id].pop() if self._global_timestamps: self._global_timestamps.pop() continue except ConditionError as e: logger.error(f"Condition error for rule {rule.id}: {e}") continue # Execute action try: action = json.loads(rule.action_json) if rule.action_json else {} except json.JSONDecodeError: logger.error(f"Invalid action JSON for rule {rule.id}") continue context = { 'rule_id': rule.id, 'rule_name': rule.name, 'topic': topic, 'payload': payload_raw, } try: result = execute_action(action, context) if not result.get('success'): logger.warning(f"Action failed for rule {rule.id}: {result.get('detail')}") except Exception as e: logger.error(f"Action execution error for rule {rule.id}: {e}") continue # Update rule stats try: from mqttui.extensions import sa rule.fire_count = (rule.fire_count or 0) + 1 rule.last_fired = datetime.utcnow() sa.session.commit() except Exception as e: logger.error(f"Failed to update rule stats for {rule.id}: {e}") # Fire signal rule_fired.send( self, rule_id=rule.id, rule_name=rule.name, topic=topic, payload=payload_raw, ) # ------------------------------------------------------------------ # Hot-reload # ------------------------------------------------------------------ def _on_rule_changed(self, sender, **kwargs): """Handle rule_changed signal by reloading the cache.""" logger.info(f"Rule changed ({kwargs.get('action', 'unknown')}), reloading cache") self.reload_cache() # ------------------------------------------------------------------ # Lifecycle # ------------------------------------------------------------------ def connect(self): """Connect to the event bus, load rule cache, and start scheduler.""" global _engine _engine = self mqtt_message_received.connect(self.on_mqtt_message) rule_changed.connect(self._on_rule_changed) self.reload_cache() if self._scheduler is None: self.init_scheduler() logger.info("RuleEngine connected to event bus") def disconnect(self): """Disconnect from the event bus and shut down scheduler.""" mqtt_message_received.disconnect(self.on_mqtt_message) try: rule_changed.disconnect(self._on_rule_changed) except Exception: pass # May not be connected if self._scheduler and self._scheduler.running: self._scheduler.shutdown(wait=False) logger.info("RuleEngine disconnected from event bus") class _nullcontext: """Minimal context manager for when no app context is needed.""" def __enter__(self): return self def __exit__(self, *args): pass ================================================ FILE: mqttui/rules/evaluator.py ================================================ """Pure-function condition evaluator for the rules engine. Evaluates structured JSON conditions against message payloads. No side effects, no database access -- purely functional. """ import re class ConditionError(Exception): """Raised when a condition is malformed or uses an unknown operator.""" pass def _get_path(data, path): """Resolve a dot-notation path in a nested dict. Args: data: dict to traverse path: dot-separated path string (e.g. "sensors.outdoor.temp") Returns: The value at the path. Raises: KeyError: if any segment is missing TypeError: if a non-dict is encountered mid-path """ segments = path.split('.') current = data for segment in segments: current = current[segment] return current def _coerce_numeric(actual, expected): """Coerce actual to numeric type if expected is numeric and actual is a string. Handles sensor payloads like {"temp": "28"} where values arrive as strings. """ if isinstance(expected, (int, float)) and isinstance(actual, str): try: return float(actual) except (ValueError, TypeError): return actual return actual def evaluate(condition, payload): """Evaluate a condition dict against a payload dict. Args: condition: dict describing the condition. Supports: - Simple: {"path": "temp", "op": "gt", "value": 30} - Compound: {"all": [...conditions]} or {"any": [...conditions]} - Empty/None: always matches (returns True) payload: dict of the parsed message payload, or None for non-JSON. Returns: bool: True if condition matches, False otherwise. Raises: ConditionError: if an unknown operator is used. """ # Empty or None condition = always match if not condition: return True # Compound conditions if 'all' in condition: return all(evaluate(c, payload) for c in condition['all']) if 'any' in condition: return any(evaluate(c, payload) for c in condition['any']) path = condition.get('path') op = condition.get('op') value = condition.get('value') # Existence checks don't need the actual value resolved first if op == 'exists': if not isinstance(payload, dict): return False try: _get_path(payload, path) return True except (KeyError, TypeError): return False if op == 'not_exists': if not isinstance(payload, dict): return True try: _get_path(payload, path) return False except (KeyError, TypeError): return True # All other ops require resolving the path value if not isinstance(payload, dict): return False try: actual = _get_path(payload, path) except (KeyError, TypeError): return False # Numeric coercion for comparison operators if op in ('gt', 'lt', 'gte', 'lte'): actual = _coerce_numeric(actual, value) if op == 'eq': return actual == value elif op == 'ne': return actual != value elif op == 'gt': return actual > value elif op == 'lt': return actual < value elif op == 'gte': return actual >= value elif op == 'lte': return actual <= value elif op == 'contains': return str(value) in str(actual) elif op == 'not_contains': return str(value) not in str(actual) elif op == 'regex': return bool(re.search(str(value), str(actual))) else: raise ConditionError(f"Unknown operator: {op}") ================================================ FILE: mqttui/rules/models.py ================================================ from mqttui.extensions import sa from datetime import datetime import json class Rule(sa.Model): __tablename__ = 'rules' id = sa.Column(sa.Integer, primary_key=True) name = sa.Column(sa.String(200), nullable=False) description = sa.Column(sa.Text, default='') trigger_topic = sa.Column(sa.String(500), nullable=False) condition_json = sa.Column(sa.Text, nullable=False, default='{}') action_json = sa.Column(sa.Text, nullable=False) enabled = sa.Column(sa.Boolean, default=True) rate_limit_per_min = sa.Column(sa.Integer, default=10) schedule_cron = sa.Column(sa.String(100), nullable=True) last_fired = sa.Column(sa.DateTime, nullable=True) fire_count = sa.Column(sa.Integer, default=0) created_at = sa.Column(sa.DateTime, server_default=sa.func.now()) updated_at = sa.Column(sa.DateTime, server_default=sa.func.now(), onupdate=datetime.utcnow) @property def action(self): try: return json.loads(self.action_json) if self.action_json else {} except (json.JSONDecodeError, TypeError): return {} @property def condition(self): try: return json.loads(self.condition_json) if self.condition_json else {} except (json.JSONDecodeError, TypeError): return {} def to_dict(self): return { 'id': self.id, 'name': self.name, 'description': self.description, 'trigger_topic': self.trigger_topic, 'condition': self.condition, 'action': self.action, 'enabled': self.enabled, 'rate_limit_per_min': self.rate_limit_per_min, 'schedule_cron': self.schedule_cron, 'last_fired': self.last_fired.isoformat() if self.last_fired else None, 'fire_count': self.fire_count, 'created_at': self.created_at.isoformat() if self.created_at else None, 'updated_at': self.updated_at.isoformat() if self.updated_at else None, } class AlertHistory(sa.Model): __tablename__ = 'alert_history' id = sa.Column(sa.Integer, primary_key=True) rule_id = sa.Column(sa.Integer, nullable=True) rule_name = sa.Column(sa.String(200)) topic = sa.Column(sa.String(500)) severity = sa.Column(sa.String(20), default='info') message = sa.Column(sa.Text) fired_at = sa.Column(sa.DateTime, server_default=sa.func.now()) # Webhook delivery fields (Phase 4) webhook_url = sa.Column(sa.String(1000), nullable=True) http_status = sa.Column(sa.Integer, nullable=True) retry_count = sa.Column(sa.Integer, default=0) error_detail = sa.Column(sa.Text, nullable=True) suppressed_count = sa.Column(sa.Integer, default=0) cooldown_until = sa.Column(sa.DateTime, nullable=True) def to_dict(self): return { 'id': self.id, 'rule_id': self.rule_id, 'rule_name': self.rule_name, 'topic': self.topic, 'severity': self.severity, 'message': self.message, 'fired_at': self.fired_at.isoformat() if self.fired_at else None, 'webhook_url': self.webhook_url, 'http_status': self.http_status, 'retry_count': self.retry_count, 'error_detail': self.error_detail, 'suppressed_count': self.suppressed_count, 'cooldown_until': self.cooldown_until.isoformat() if self.cooldown_until else None, } ================================================ FILE: mqttui/rules/ssrf.py ================================================ """SSRF URL validation for webhook destinations. Blocks webhook URLs that resolve to private, loopback, or link-local addresses to prevent Server-Side Request Forgery attacks. """ import ipaddress import logging import socket from urllib.parse import urlparse logger = logging.getLogger(__name__) def is_ssrf_safe(url: str) -> tuple: """Check if a URL is safe from SSRF by resolving and validating the IP. Args: url: The webhook destination URL to validate. Returns: Tuple of (safe: bool, reason: str). If safe is False, reason explains why. """ try: parsed = urlparse(url) except Exception: return False, "Invalid URL" hostname = parsed.hostname if not hostname: return False, "No hostname in URL" # Try to parse hostname directly as IP address first try: addr = ipaddress.ip_address(hostname) return _check_ip(addr) except ValueError: pass # Resolve hostname via DNS try: results = socket.getaddrinfo(hostname, parsed.port or 80, proto=socket.IPPROTO_TCP) except socket.gaierror as e: return False, f"DNS resolution failed: {e}" if not results: return False, "DNS resolution returned no results" # Check ALL resolved addresses -- block if any are private for family, _, _, _, sockaddr in results: ip_str = sockaddr[0] try: addr = ipaddress.ip_address(ip_str) safe, reason = _check_ip(addr) if not safe: return False, reason except ValueError: continue return True, "URL is safe" def _check_ip(addr) -> tuple: """Check a single IP address against blocked ranges. Returns: Tuple of (safe: bool, reason: str). """ if addr.is_private: return False, f"Blocked: {addr} is a private address" if addr.is_loopback: return False, f"Blocked: {addr} is a loopback address" if addr.is_link_local: return False, f"Blocked: {addr} is a link-local address" if addr.is_reserved: return False, f"Blocked: {addr} is a reserved address" if addr.is_multicast: return False, f"Blocked: {addr} is a multicast address" if addr.is_unspecified: return False, f"Blocked: {addr} is an unspecified address" return True, "Address is safe" ================================================ FILE: mqttui/socketio_batch.py ================================================ import threading from flask_socketio import SocketIO class BatchEmitter: """Collects MQTT messages and emits them in batches every 100ms. This prevents UI flooding at high throughput by buffering messages server-side and sending them in consolidated batches. """ def __init__(self, socketio: SocketIO, interval_ms: int = 100): self._socketio = socketio self._interval = interval_ms / 1000.0 self._buffer = [] self._lock = threading.Lock() self._timer = None self._running = False def start(self): """Start the periodic flush timer.""" self._running = True self._schedule_flush() def stop(self): """Stop the periodic flush timer.""" self._running = False if self._timer: self._timer.cancel() def enqueue(self, message: dict): """Add a message to the batch buffer (thread-safe).""" with self._lock: self._buffer.append(message) def _schedule_flush(self): """Schedule the next flush cycle.""" if not self._running: return self._timer = threading.Timer(self._interval, self._flush) self._timer.daemon = True self._timer.start() def _flush(self): """Emit buffered messages as a batch and reschedule.""" with self._lock: batch = self._buffer self._buffer = [] if batch: self._socketio.emit('mqtt_messages_batch', batch) self._schedule_flush() _batch_emitter = None def init_batch_emitter(socketio: SocketIO, interval_ms: int = 100): """Initialize and start the global batch emitter. Called once in create_app after socketio.init_app(). """ global _batch_emitter _batch_emitter = BatchEmitter(socketio, interval_ms) _batch_emitter.start() return _batch_emitter def get_batch_emitter(): """Return the global batch emitter instance.""" return _batch_emitter ================================================ FILE: mqttui/state.py ================================================ """ Module-level shared state for in-memory MQTT data. Imported by routes and MQTT handlers. """ messages = [] topics = set() connection_count = 0 active_websockets = 0 error_log = [] ================================================ FILE: pytest.ini ================================================ [pytest] testpaths = tests python_files = test_*.py python_classes = Test* python_functions = test_* addopts = -v --tb=short ================================================ FILE: requirements-dev.txt ================================================ pytest==8.3.4 pytest-cov==6.0.0 ================================================ FILE: requirements.txt ================================================ Flask==3.1.0 Flask-SocketIO==5.5.1 paho-mqtt==2.1.0 Werkzeug==3.1.3 psutil==6.1.1 python-dotenv==1.0.1 gunicorn==23.0.0 gevent==24.11.1 gevent-websocket==0.10.1 blinker==1.9.0 Flask-SQLAlchemy==3.1.1 Flask-Login==0.6.3 Flask-CORS==5.0.1 apispec==6.8.1 apispec-webframeworks==1.2.0 Flask-Limiter==3.11.0 APScheduler==3.11.2 httpx>=0.27 structlog>=24.1.0 prometheus_client>=0.21.0 pluggy>=1.5.0 ================================================ FILE: static/css/input.css ================================================ @import "tailwindcss"; @theme { --color-gray-750: #2d3748; --color-gray-850: #1a202c; } ================================================ FILE: static/css/output.css ================================================ /* Tailwind CSS v4 compiled output. * Build with: tailwindcss -i static/css/input.css -o static/css/output.css --minify * Watch with: tailwindcss -i static/css/input.css -o static/css/output.css --watch * * This file is a placeholder for local development without the Tailwind CLI. * The CDN fallback in base.html provides Tailwind utilities during development. * The Dockerfile compiles this file during the Docker build. */ ================================================ FILE: static/script.js ================================================ const socket = io(); let messageChart; let network; let nodes; let edges; let topicFilter = 'all'; let messageCount = 0; let pinnedNodes = new Set(); // ============================================================ // Alpine.js global store for shared state // ============================================================ document.addEventListener('alpine:init', () => { Alpine.store('mqtt', { connected: false, selectedTopic: 'all', messageCount: 0, }); }); // ============================================================ // Alpine.js root app component (bound to ) // ============================================================ function mqttuiApp() { return { sidebarOpen: true, activeTab: 'dashboard', alertsLoaded: false, rulesLoaded: false, analyticsLoaded: false, pluginsLoaded: false, brokersLoaded: false, init() { // Initialize Chart.js and Vis.js network initChart(); initNetwork(); setInterval(updateChart, 1000); setInterval(updateStats, 5000); initDebugBar(); trackClientPerformance(); // Load existing topics from API loadTopicsFromAPI(); setInterval(loadTopicsFromAPI, 30000); // Setup advanced search UI setupAdvancedSearchHandlers(); // Handle batch messages from server (100ms batched) socket.on('mqtt_messages_batch', (batch) => { batch.forEach(msg => { window.dispatchEvent(new CustomEvent('mqtt-message', { detail: msg })); Alpine.store('mqtt').messageCount++; messageCount++; updateNetwork(msg); updateTopicFilter(msg.topic); }); }); // Fallback for single messages (backward compat) socket.on('mqtt_message', (data) => { window.dispatchEvent(new CustomEvent('mqtt-message', { detail: data })); Alpine.store('mqtt').messageCount++; messageCount++; updateMessageList(data); updateNetwork(data); updateTopicFilter(data.topic); }); }, toggleSidebar() { this.sidebarOpen = !this.sidebarOpen; // Toggle body class for CSS max-width adjustment if (this.sidebarOpen) { document.body.classList.remove('sidebar-hidden'); } else { document.body.classList.add('sidebar-hidden'); } }, }; } // ============================================================ // Alpine.js message list component // ============================================================ function messageListComponent() { return { messages: [], favorites: [], init() { this.loadMessages(); this.loadFavorites(); window.addEventListener('mqtt-message', (e) => { const msg = e.detail; const topicFilter = Alpine.store('mqtt').selectedTopic; const brokerEl = document.getElementById('broker-filter'); const brokerFilter = brokerEl ? brokerEl.value : 'all'; const topicMatch = topicFilter === 'all' || msg.topic === topicFilter; const brokerMatch = brokerFilter === 'all' || String(msg.broker_id) === brokerFilter; if (topicMatch && brokerMatch) { this.messages.unshift(msg); if (this.messages.length > 100) this.messages.pop(); } }); }, async loadMessages(extraFilters = {}) { const params = new URLSearchParams({ limit: '50' }); const topic = Alpine.store('mqtt').selectedTopic; if (topic && topic !== 'all') params.append('topic', topic); // Apply extra filters (from Advanced Search) Object.entries(extraFilters).forEach(([key, value]) => { if (value) params.append(key, value); }); try { const resp = await fetch(`/api/v1/messages?${params}`); const data = await resp.json(); const payload = data.data || data; this.messages = payload.messages || []; } catch (e) { console.error('Error loading messages:', e); } }, showFavoritesOnly: false, get filteredMessages() { if (!this.showFavoritesOnly) return this.messages; return this.messages.filter(m => this.favorites.includes(m.topic)); }, async loadFavorites() { try { const resp = await fetch('/api/v1/topics/favorites'); const data = await resp.json(); if (data.status === 'success') { this.favorites = data.data.favorites.map(f => f.topic); } } catch (e) { console.error('Error loading favorites:', e); } }, isFavorite(topic) { return this.favorites.includes(topic); }, async toggleFavorite(topic) { try { const resp = await fetch(`/api/v1/topics/${encodeURIComponent(topic)}/bookmark`, { method: 'POST', }); const data = await resp.json(); if (data.status === 'success') { if (data.data.bookmarked) { this.favorites.push(topic); } else { this.favorites = this.favorites.filter(t => t !== topic); } } } catch (e) { console.error('Error toggling favorite:', e); } }, formatPayload(payload) { try { return JSON.stringify(JSON.parse(payload), null, 2); } catch { return payload; } }, formatTime(ts) { return new Date(ts).toLocaleTimeString(); }, }; } // ============================================================ // Alpine.js stats component // ============================================================ function statsComponent() { return { connections: 0, topics: 0, messageTotal: 0, init() { this.fetchStats(); setInterval(() => this.fetchStats(), 5000); }, async fetchStats() { try { const resp = await fetch('/stats'); const data = await resp.json(); this.connections = data.connection_count; this.topics = data.topic_count; this.messageTotal = data.message_count; } catch (e) { console.error('Error fetching stats:', e); } }, }; } // ============================================================ // Alpine.js publish component // ============================================================ function publishComponent() { return { topic: '', message: '', async submit() { await fetch('/publish', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: `topic=${encodeURIComponent(this.topic)}&message=${encodeURIComponent(this.message)}` }); this.topic = ''; this.message = ''; }, }; } // ============================================================ // Chart.js (kept as-is per plan) // ============================================================ function initChart() { if (messageChart) return; const canvas = document.getElementById('messageChart'); if (!canvas) return; const ctx = canvas.getContext('2d'); // Create gradient fill const gradient = ctx.createLinearGradient(0, 0, 0, canvas.parentElement.clientHeight || 160); gradient.addColorStop(0, 'rgba(59, 130, 246, 0.3)'); gradient.addColorStop(1, 'rgba(59, 130, 246, 0.0)'); // Pre-fill with zeros for smooth start const emptyLabels = Array(30).fill(''); const emptyData = Array(30).fill(0); messageChart = new Chart(ctx, { type: 'line', data: { labels: emptyLabels, datasets: [{ label: 'msg/s', data: emptyData, borderColor: 'rgb(59, 130, 246)', backgroundColor: gradient, borderWidth: 2, fill: true, tension: 0.4, pointRadius: 0, pointHoverRadius: 4, pointHoverBackgroundColor: 'rgb(59, 130, 246)', }] }, options: { responsive: true, maintainAspectRatio: false, animation: { duration: 300 }, interaction: { intersect: false, mode: 'index' }, scales: { y: { beginAtZero: true, grid: { color: 'rgba(75, 85, 99, 0.3)', drawBorder: false }, ticks: { color: 'rgb(156, 163, 175)', font: { size: 10 }, maxTicksLimit: 4, callback: v => Number.isInteger(v) ? v : '', }, border: { display: false }, }, x: { display: false, } }, plugins: { legend: { display: false }, tooltip: { backgroundColor: 'rgba(17, 24, 39, 0.9)', titleColor: 'rgb(156, 163, 175)', bodyColor: 'rgb(59, 130, 246)', bodyFont: { weight: 'bold', size: 14 }, padding: 8, displayColors: false, callbacks: { title: (items) => items[0]?.label || '', label: (item) => `${item.raw} msg/s`, } } } } }); } function updateChart() { if (!messageChart) return; const now = new Date(); const timeLabel = now.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', second: '2-digit' }); messageChart.data.labels.push(timeLabel); messageChart.data.datasets[0].data.push(messageCount); if (messageChart.data.labels.length > 30) { messageChart.data.labels.shift(); messageChart.data.datasets[0].data.shift(); } // Update the big rate number const rateDisplay = document.getElementById('rate-display'); if (rateDisplay) rateDisplay.textContent = messageCount; messageChart.update('none'); // skip animation for smoother feel at 1s interval messageCount = 0; } // ============================================================ // Vis.js network (kept as-is per plan) // ============================================================ function initNetwork() { if (network) return; // Already initialized nodes = new vis.DataSet([ { id: 'broker', label: 'MQTT Broker', shape: 'hexagon', color: '#FFA500', size: 30 } ]); edges = new vis.DataSet(); const container = document.getElementById('network-visualization'); const data = { nodes, edges }; const options = { physics: { enabled: true, stabilization: { enabled: true, iterations: 150, updateInterval: 25, fit: true }, barnesHut: { gravitationalConstant: -8000, centralGravity: 0.3, springLength: 200, springConstant: 0.04, damping: 0.09, avoidOverlap: 0.1 } }, nodes: { font: { color: '#FFFFFF', size: 14 }, borderWidth: 2, shadow: true, margin: 10 }, edges: { width: 2, color: { inherit: 'from' }, smooth: { type: 'continuous' }, arrows: { to: { enabled: true, scaleFactor: 0.5 } } }, interaction: { dragNodes: true, dragView: true, zoomView: true }, layout: { improvedLayout: true, clusterThreshold: 150 } }; network = new vis.Network(container, data, options); // Auto-fit network after stabilization network.on('stabilizationIterationsDone', function() { network.fit({ animation: { duration: 1000, easingFunction: 'easeInOutQuad' } }); }); // Double-click to pin/unpin nodes network.on('doubleClick', function(params) { if (params.nodes.length > 0) { const nodeId = params.nodes[0]; if (pinnedNodes.has(nodeId)) { pinnedNodes.delete(nodeId); nodes.update({ id: nodeId, fixed: false, color: nodes.get(nodeId).color || '#97C2FC' }); } else { pinnedNodes.add(nodeId); nodes.update({ id: nodeId, fixed: true, color: '#FF6B6B' }); } } }); // Right-click context menu for pinning network.on('oncontext', function(params) { params.event.preventDefault(); if (params.nodes.length > 0) { const nodeId = params.nodes[0]; const isPinned = pinnedNodes.has(nodeId); const action = isPinned ? 'Unpin' : 'Pin'; if (confirm(`${action} node "${nodes.get(nodeId).label}"?`)) { if (isPinned) { pinnedNodes.delete(nodeId); nodes.update({ id: nodeId, fixed: false, color: nodes.get(nodeId).color || '#97C2FC' }); } else { pinnedNodes.add(nodeId); nodes.update({ id: nodeId, fixed: true, color: '#FF6B6B' }); } } } }); } function updateNetwork(message) { if (!nodes || !edges || !network) { console.error('Network not initialized'); return; } const topicParts = message.topic.split('/'); let parentId = 'broker'; topicParts.forEach((part, index) => { const nodeId = topicParts.slice(0, index + 1).join('/'); if (!nodes.get(nodeId)) { nodes.add({ id: nodeId, label: part, color: getRandomColor(), shape: 'dot', size: 20 - index * 2 }); clearTimeout(window.autoFitTimeout); window.autoFitTimeout = setTimeout(() => { if (network) { network.fit({ animation: { duration: 800, easingFunction: 'easeInOutQuad' } }); } }, 2000); } if (parentId !== nodeId) { const edgeId = `${parentId}-${nodeId}`; if (!edges.get(edgeId)) { edges.add({ id: edgeId, from: parentId, to: nodeId }); } } parentId = nodeId; }); // Animate message flow const edgeIds = edges.getIds(); edgeIds.forEach(edgeId => { edges.update({ id: edgeId, color: { color: '#00ff00' }, width: 4 }); setTimeout(() => { edges.update({ id: edgeId, color: { inherit: 'from' }, width: 2 }); }, 1000); }); // Pulse the final topic node const finalNodeId = topicParts.join('/'); const finalNode = nodes.get(finalNodeId); nodes.update({ id: finalNodeId, size: finalNode.size + 5 }); setTimeout(() => { nodes.update({ id: finalNodeId, size: finalNode.size }); }, 1000); } // ============================================================ // Standalone utility functions (kept as-is per plan) // ============================================================ function getRandomColor() { const letters = '0123456789ABCDEF'; let color = '#'; for (let i = 0; i < 6; i++) { color += letters[Math.floor(Math.random() * 16)]; } return color; } // Legacy updateMessageList for backward-compat single-message handler function updateMessageList(message) { // Now handled by Alpine.js messageListComponent via CustomEvent // This function is kept for any non-Alpine fallback paths } function updateStats() { // Stats are now handled by Alpine.js statsComponent — this is a no-op } function updateTopicFilter(newTopic) { const topicFilterEl = document.getElementById('topic-filter'); if (topicFilterEl && !Array.from(topicFilterEl.options).some(option => option.value === newTopic)) { const option = document.createElement('option'); option.value = newTopic; option.textContent = newTopic; topicFilterEl.appendChild(option); } } function loadTopicsFromAPI() { fetch('/api/v1/topics') .then(response => response.json()) .then(envelope => { const data = envelope.data || envelope; const topicFilterEl = document.getElementById('topic-filter'); if (!topicFilterEl) return; const allTopicsOption = topicFilterEl.querySelector('option[value="all"]'); topicFilterEl.innerHTML = ''; if (allTopicsOption) { topicFilterEl.appendChild(allTopicsOption); } else { const option = document.createElement('option'); option.value = 'all'; option.textContent = 'All Topics'; topicFilterEl.appendChild(option); } // Sort favorites to top const topics = data.topics || []; topics.sort((a, b) => { if (a.is_favorite && !b.is_favorite) return -1; if (!a.is_favorite && b.is_favorite) return 1; return 0; }); topics.forEach(topic => { const option = document.createElement('option'); option.value = topic.topic; const star = topic.is_favorite ? '\u2605 ' : ''; const count = topic.message_count != null ? ` (${topic.message_count})` : ''; option.textContent = `${star}${topic.topic}${count}`; topicFilterEl.appendChild(option); }); }) .catch(error => console.error('Error loading topics:', error)); } function loadFilteredMessages(customFilters = {}) { // Find the Alpine message list component and call loadMessages with filters const messageList = document.getElementById('message-list'); if (!messageList) return; // Read topic from dropdown (not stale global) const topicEl = document.getElementById('topic-filter'); const selectedTopic = topicEl ? topicEl.value : 'all'; if (selectedTopic !== 'all') { Alpine.store('mqtt').selectedTopic = selectedTopic; } // Get the Alpine component on the parent element const alpineEl = messageList.closest('[x-data]'); if (alpineEl && alpineEl._x_dataStack) { const component = alpineEl._x_dataStack[0]; if (component && component.loadMessages) { component.loadMessages(customFilters); return; } } // Fallback: dispatch event with filters window.dispatchEvent(new CustomEvent('apply-filters', { detail: customFilters })); } // ============================================================ // Advanced search handlers (kept as-is per plan) // ============================================================ function setupAdvancedSearchHandlers() { const applyBtn = document.getElementById('apply-filters-btn'); if (applyBtn) { applyBtn.addEventListener('click', function() { const filters = { content: document.getElementById('content-search').value, regex_topic: document.getElementById('regex-topic').value, json_path: document.getElementById('json-path').value, json_value: document.getElementById('json-value').value, hours: document.getElementById('time-filter').value }; Object.keys(filters).forEach(key => { if (!filters[key]) delete filters[key]; }); loadFilteredMessages(filters); }); } const clearBtn = document.getElementById('clear-filters-btn'); if (clearBtn) { clearBtn.addEventListener('click', function() { const topicFilterEl = document.getElementById('topic-filter'); if (topicFilterEl) topicFilterEl.value = 'all'; const contentSearch = document.getElementById('content-search'); if (contentSearch) contentSearch.value = ''; const regexTopic = document.getElementById('regex-topic'); if (regexTopic) regexTopic.value = ''; const jsonPath = document.getElementById('json-path'); if (jsonPath) jsonPath.value = ''; const jsonValue = document.getElementById('json-value'); if (jsonValue) jsonValue.value = ''; const timeFilter = document.getElementById('time-filter'); if (timeFilter) timeFilter.value = ''; topicFilter = 'all'; if (window.Alpine && Alpine.store('mqtt')) { Alpine.store('mqtt').selectedTopic = 'all'; } loadFilteredMessages(); }); } loadFilterPresets(); const loadPresetBtn = document.getElementById('load-preset-btn'); if (loadPresetBtn) { loadPresetBtn.addEventListener('click', function() { const presetName = document.getElementById('preset-select').value; if (!presetName) { alert('Please select a preset to load'); return; } fetch(`/api/v1/filter-presets/${encodeURIComponent(presetName)}/use`, { method: 'POST' }) .then(response => response.json()) .then(data => { const inner = data.data || data; if (data.status === 'success' || inner.filters) { const filters = inner.filters; const contentSearch = document.getElementById('content-search'); if (contentSearch) contentSearch.value = filters.content || ''; const regexTopic = document.getElementById('regex-topic'); if (regexTopic) regexTopic.value = filters.regex_topic || ''; const jsonPath = document.getElementById('json-path'); if (jsonPath) jsonPath.value = filters.json_path || ''; const jsonValue = document.getElementById('json-value'); if (jsonValue) jsonValue.value = filters.json_value || ''; const timeFilter = document.getElementById('time-filter'); if (timeFilter) timeFilter.value = filters.hours || ''; if (filters.topic) { const topicFilterEl = document.getElementById('topic-filter'); if (topicFilterEl) topicFilterEl.value = filters.topic; topicFilter = filters.topic; } loadFilteredMessages(filters); } else { alert('Error loading preset: ' + data.error); } }) .catch(error => { console.error('Error loading preset:', error); alert('Error loading preset'); }); }); } const savePresetBtn = document.getElementById('save-preset-btn'); if (savePresetBtn) { savePresetBtn.addEventListener('click', function() { const name = prompt('Enter a name for this filter preset:'); if (!name) return; const description = prompt('Enter a description (optional):') || ''; const filters = { content: document.getElementById('content-search').value, regex_topic: document.getElementById('regex-topic').value, json_path: document.getElementById('json-path').value, json_value: document.getElementById('json-value').value, hours: document.getElementById('time-filter').value }; if (topicFilter && topicFilter !== 'all') { filters.topic = topicFilter; } Object.keys(filters).forEach(key => { if (!filters[key]) delete filters[key]; }); if (Object.keys(filters).length === 0) { alert('No filters to save'); return; } fetch('/api/v1/filter-presets', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ name: name, description: description, filters: filters }) }) .then(response => response.json()) .then(data => { if (data.status === 'success') { alert('Filter preset saved successfully!'); loadFilterPresets(); } else { alert('Error saving preset: ' + (data.error?.message || data.error)); } }) .catch(error => { console.error('Error saving preset:', error); alert('Error saving preset'); }); }); } // Fullscreen and reset buttons for network const fullscreenBtn = document.getElementById('fullscreen-btn'); if (fullscreenBtn) { fullscreenBtn.addEventListener('click', function() { const networkDiv = document.getElementById('network-visualization'); if (networkDiv.requestFullscreen) { networkDiv.requestFullscreen(); } else if (networkDiv.webkitRequestFullscreen) { networkDiv.webkitRequestFullscreen(); } else if (networkDiv.msRequestFullscreen) { networkDiv.msRequestFullscreen(); } }); } const resetBtn = document.getElementById('reset-nodes-btn'); if (resetBtn) { resetBtn.addEventListener('click', function() { if (network) { pinnedNodes.forEach(nodeId => { const node = nodes.get(nodeId); if (node) { nodes.update({ id: nodeId, fixed: false, color: node.originalColor || '#97C2FC' }); } }); pinnedNodes.clear(); network.setOptions({ physics: { enabled: true, stabilization: { enabled: true, iterations: 150, updateInterval: 25, fit: true } } }); setTimeout(() => { network.fit({ animation: { duration: 1000, easingFunction: 'easeInOutQuad' } }); }, 500); } }); } } // Topic filter change handler document.addEventListener('DOMContentLoaded', function() { const topicFilterEl = document.getElementById('topic-filter'); if (topicFilterEl) { topicFilterEl.addEventListener('change', function(e) { topicFilter = e.target.value; if (window.Alpine && Alpine.store('mqtt')) { Alpine.store('mqtt').selectedTopic = topicFilter; } loadFilteredMessages(); }); } }); function loadFilterPresets() { fetch('/api/v1/filter-presets') .then(response => response.json()) .then(data => { const presetSelect = document.getElementById('preset-select'); if (!presetSelect) return; presetSelect.innerHTML = ''; const presets = (data.data || data).presets || []; presets.forEach(preset => { const option = document.createElement('option'); option.value = preset.name; option.textContent = `${preset.name}${preset.description ? ' - ' + preset.description : ''}`; presetSelect.appendChild(option); }); }) .catch(error => console.error('Error loading presets:', error)); } // ============================================================ // Debug bar (kept as-is per plan) // ============================================================ let debugBar; let debugBarToggle; function initDebugBar() { if (debugBar) return; // Already initialized debugBar = document.createElement('div'); debugBar.id = 'debug-bar'; debugBar.style.display = 'none'; document.body.appendChild(debugBar); debugBarToggle = document.createElement('button'); debugBarToggle.id = 'debug-bar-toggle'; debugBarToggle.innerHTML = 'Debug'; debugBarToggle.onclick = toggleDebugBar; document.body.appendChild(debugBarToggle); const closeButton = document.createElement('button'); closeButton.id = 'debug-bar-close'; closeButton.innerHTML = '×'; closeButton.onclick = closeDebugBar; debugBar.appendChild(closeButton); updateDebugBar(); setInterval(updateDebugBar, 1000); } function toggleDebugBar() { fetch('/toggle-debug-bar', { method: 'POST' }) .then(response => response.json()) .then(data => { debugBar.style.display = data.enabled ? 'block' : 'none'; debugBarToggle.classList.toggle('active', data.enabled); }); } function closeDebugBar() { debugBar.style.display = 'none'; fetch('/toggle-debug-bar', { method: 'POST' }); debugBarToggle.classList.remove('active'); } function trackClientPerformance() { // Delay to ensure timing data is available after full page load setTimeout(() => { let pageLoadTime = 0; let domReadyTime = 0; // Prefer modern Navigation Timing API const entries = performance.getEntriesByType('navigation'); if (entries.length > 0) { const nav = entries[0]; pageLoadTime = Math.round(nav.loadEventEnd); domReadyTime = Math.round(nav.domContentLoadedEventEnd); } else if (performance.timing) { // Fallback to deprecated API const t = performance.timing; pageLoadTime = t.loadEventEnd - t.navigationStart; domReadyTime = t.domContentLoadedEventEnd - t.navigationStart; } if (pageLoadTime > 0) { fetch('/record-client-performance', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ pageLoadTime, domReadyTime }), }); } }, 2000); } function updateDebugBar() { fetch('/debug-bar') .then(response => response.json()) .then(data => { let content = ''; content += '
'; for (const [panelName, panelData] of Object.entries(data)) { content += `

${panelName}

    `; for (const [key, value] of Object.entries(panelData)) { let displayValue = value; if (typeof value === 'object' && value !== null) { displayValue = '
    ' + JSON.stringify(value, null, 2) + '
    '; } content += `
  • ${key}: ${displayValue}
    • `; } content += '
'; } content += '
'; debugBar.innerHTML = content; }); } // ============================================================ // Alpine.js rule form component (create/edit) // ============================================================ function ruleFormComponent(existing = {}) { const condition = existing.condition || {}; const action = existing.action || {}; return { form: { id: existing.id || null, name: existing.name || '', description: existing.description || '', trigger_topic: existing.trigger_topic || '', condition_path: condition.path || '', condition_op: condition.op || '', condition_value: condition.value !== undefined ? String(condition.value) : '', action_type: action.type || 'publish', action_topic: action.topic || '', action_payload: action.payload || '', action_url: action.url || '', action_template: action.payload_template || '', action_severity: action.severity || 'info', action_message: action.message || '', action_bot_token: action.bot_token || '', action_chat_id: action.chat_id || '', action_message_template: action.message_template || '', action_slack_url: action.webhook_url || '', rate_limit_per_min: existing.rate_limit_per_min || 10, }, error: '', success: '', buildPayload() { const payload = { name: this.form.name, description: this.form.description, trigger_topic: this.form.trigger_topic, rate_limit_per_min: this.form.rate_limit_per_min, }; // Build condition if (this.form.condition_op && this.form.condition_path) { const cond = { path: this.form.condition_path, op: this.form.condition_op }; if (!['exists', 'not_exists'].includes(this.form.condition_op)) { let val = this.form.condition_value; const num = Number(val); if (!isNaN(num) && val.trim() !== '') val = num; cond.value = val; } payload.condition = cond; } else { payload.condition = {}; } // Build action const act = { type: this.form.action_type }; if (this.form.action_type === 'publish') { act.topic = this.form.action_topic; act.payload = this.form.action_payload; } else if (this.form.action_type === 'telegram') { act.bot_token = this.form.action_bot_token; act.chat_id = this.form.action_chat_id; if (this.form.action_message_template) act.message_template = this.form.action_message_template; } else if (this.form.action_type === 'slack') { act.webhook_url = this.form.action_slack_url; if (this.form.action_message_template) act.message_template = this.form.action_message_template; } else if (this.form.action_type === 'webhook') { act.url = this.form.action_url; if (this.form.action_template) act.payload_template = this.form.action_template; } else if (this.form.action_type === 'log') { act.severity = this.form.action_severity; act.message = this.form.action_message; } payload.action = act; return payload; }, async submitRule() { this.error = ''; this.success = ''; const payload = this.buildPayload(); const url = this.form.id ? `/api/v1/rules/${this.form.id}` : '/api/v1/rules/'; const method = this.form.id ? 'PUT' : 'POST'; try { const resp = await fetch(url, { method, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload), }); const data = await resp.json(); if (data.status === 'success') { this.success = this.form.id ? 'Rule updated' : 'Rule created'; // Reload rules list via htmx htmx.ajax('GET', '/partials/rules', { target: '#rules-panel', swap: 'innerHTML' }); } else { this.error = data.error?.message || 'Unknown error'; } } catch (e) { this.error = 'Network error: ' + e.message; } }, }; } // ============================================================ // Alpine.js dry-run test component // ============================================================ function dryRunComponent(ruleId) { return { ruleId: ruleId, topic: '', payload: '', result: null, error: '', async runTest() { this.error = ''; this.result = null; let payloadVal = this.payload; try { payloadVal = JSON.parse(this.payload); } catch (e) { /* use raw string */ } try { const resp = await fetch(`/api/v1/rules/${this.ruleId}/test`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ topic: this.topic, payload: payloadVal }), }); const data = await resp.json(); if (data.status === 'success') { this.result = data.data; } else { this.error = data.error?.message || 'Test failed'; } } catch (e) { this.error = 'Network error: ' + e.message; } }, }; } // ============================================================ // Broker management component // ============================================================ function brokersComponent() { return { brokers: [], showForm: false, editId: null, error: '', form: { name: '', host: '', port: 1883, username: '', password: '', mqtt_version: '3.1.1', topics: '#', tls_enabled: false, tls_insecure: false }, async loadBrokers() { try { const resp = await fetch('/api/v1/brokers/'); if (resp.redirected || !resp.ok) return; // Not logged in or error const ct = resp.headers.get('content-type') || ''; if (!ct.includes('json')) return; // Got HTML redirect const data = await resp.json(); this.brokers = (data.data || data).brokers || []; } catch (e) { console.error('Error loading brokers:', e); } }, resetForm() { this.form = { name: '', host: '', port: 1883, username: '', password: '', mqtt_version: '3.1.1', topics: '#', tls_enabled: false, tls_insecure: false }; this.editId = null; this.showForm = false; this.error = ''; }, editBroker(broker) { this.form = { name: broker.name, host: broker.host, port: broker.port, username: broker.username || '', password: '', mqtt_version: broker.mqtt_version, topics: broker.topics, tls_enabled: broker.tls_enabled, tls_insecure: broker.tls_insecure }; this.editId = broker.id; this.showForm = true; }, async saveBroker() { this.error = ''; const url = this.editId ? `/api/v1/brokers/${this.editId}` : '/api/v1/brokers/'; const method = this.editId ? 'PUT' : 'POST'; try { const resp = await fetch(url, { method, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(this.form) }); if (!resp.ok && resp.status !== 201) { this.error = `Server error (${resp.status})`; return; } const data = await resp.json(); if (data.status === 'success') { this.resetForm(); await this.loadBrokers(); } else { this.error = data.error?.message || 'Failed to save broker'; } } catch (e) { this.error = 'Network error: ' + e.message; } }, async deleteBroker(id) { try { await fetch(`/api/v1/brokers/${id}`, { method: 'DELETE' }); await this.loadBrokers(); } catch (e) { console.error('Error deleting broker:', e); } }, async connectBroker(id) { await fetch(`/api/v1/brokers/${id}/connect`, { method: 'POST' }); setTimeout(() => this.loadBrokers(), 1500); }, async disconnectBroker(id) { await fetch(`/api/v1/brokers/${id}/disconnect`, { method: 'POST' }); await this.loadBrokers(); }, }; } ================================================ FILE: static/styles.css ================================================ /* Custom scrollbar for Webkit browsers */ #message-list::-webkit-scrollbar { width: 8px; } #message-list::-webkit-scrollbar-track { background: #1F2937; } #message-list::-webkit-scrollbar-thumb { background: #4B5563; border-radius: 4px; } #message-list::-webkit-scrollbar-thumb:hover { background: #6B7280; } /* Smooth transitions for dark mode */ body { transition: background-color 0.3s, color 0.3s; } /* Allow the content to fill available space minus sidebar. */ .main-content-max { max-width: calc(100vw - 20rem); } /* When sidebar is hidden, allow full width */ .sidebar-hidden .main-content-max { max-width: 100vw; } /* Additional styles for better readability */ #message-list div { background-color: rgba(55, 65, 81, 0.5); padding: 0.5rem; border-radius: 0.25rem; } #network-visualization { border: 1px solid #4B5563; border-radius: 0.25rem; } /* Improve form input styling */ input[type="text"], textarea, select { width: 100%; padding: 0.5rem; border-radius: 0.25rem; background-color: #374151; border: 1px solid #4B5563; color: #F3F4F6; } input[type="text"]:focus, textarea:focus, select:focus { outline: none; border-color: #60A5FA; box-shadow: 0 0 0 2px rgba(96, 165, 250, 0.2); } /* Debug bar styles */ #debug-bar { position: fixed; bottom: 0; left: 0; right: 0; background-color: rgba(0, 0, 0, 0.95); color: #fff; padding: 15px; font-family: 'Courier New', monospace; font-size: 13px; max-height: 60%; overflow-y: auto; z-index: 1000; border-top: 2px solid #3b82f6; box-shadow: 0 -2px 10px rgba(0, 0, 0, 0.5); } #debug-bar h2 { margin-top: 0; font-size: 18px; color: #3b82f6; } #debug-bar h3 { margin-top: 10px; font-size: 16px; color: #60a5fa; border-bottom: 1px solid #3b82f6; padding-bottom: 5px; } #debug-bar ul { list-style-type: none; padding-left: 0; } #debug-bar li { margin-bottom: 8px; word-break: break-all; } #debug-bar pre { background-color: rgba(255, 255, 255, 0.1); padding: 5px; border-radius: 3px; overflow-x: auto; } #debug-bar-toggle { position: fixed; bottom: 10px; right: 10px; background-color: #3b82f6; color: white; border: none; padding: 8px 12px; border-radius: 4px; cursor: pointer; z-index: 1001; font-weight: bold; transition: background-color 0.3s; } #debug-bar-toggle:hover { background-color: #2563eb; } #debug-bar-toggle.active { background-color: #1e40af; } #debug-bar-close { position: absolute; top: 5px; right: 5px; background: none; border: none; color: #fff; font-size: 20px; cursor: pointer; } .debug-content { display: flex; flex-wrap: wrap; gap: 10px; } .debug-panel { background-color: rgba(59, 130, 246, 0.1); border-radius: 4px; padding: 10px; flex: 1 1 calc(33% - 10px); min-width: 250px; margin-bottom: 10px; } .debug-panel h3 { margin-top: 0; } /* Responsive design for smaller screens */ @media (max-width: 768px) { .debug-panel { flex: 1 1 100%; } } ================================================ FILE: templates/base.html ================================================ {% block title %}MQTT Web Interface{% endblock %} {% block head %}{% endblock %} {% block body %}{% endblock %} {% block scripts %}{% endblock %} ================================================ FILE: templates/index.html ================================================ {% extends "base.html" %} {% block head %} {% endblock %} {% block body %}

MQTT Web Interface

Message Flow

Messages

No messages
Loading rules...
Loading alerts...
Loading analytics...
Loading plugins...
{% include 'partials/brokers.html' %}
{% endblock %} {% block scripts %} {% endblock %} ================================================ FILE: templates/login.html ================================================ MQTTUI - Login

MQTTUI

{% with messages = get_flashed_messages(with_categories=true) %} {% if messages %} {% for category, message in messages %}
{{ message }}
{% endfor %} {% endif %} {% endwith %}
================================================ FILE: templates/partials/alert_row.html ================================================
{{ alert.severity }} {{ alert.rule_name or 'Unknown Rule' }} {% if alert.webhook_url %} {% if alert.http_status %}HTTP {{ alert.http_status }}{% else %}Pending{% endif %} {% endif %} {% if alert.suppressed_count > 0 %} ({{ alert.suppressed_count }} suppressed) {% endif %}
{{ alert.message or '' }}
Topic: {{ alert.topic or 'N/A' }} {% if alert.retry_count > 0 %} | Retries: {{ alert.retry_count }} {% endif %} {% if alert.error_detail %} | Error: {{ alert.error_detail[:100] }} {% endif %}
{{ alert.fired_at.strftime('%Y-%m-%d %H:%M:%S') if alert.fired_at else '' }}
================================================ FILE: templates/partials/alerts_list.html ================================================
Showing {{ alerts|length }} of {{ total }} alerts
{% if alerts %}
{% for alert in alerts %} {% include 'partials/alert_row.html' %} {% endfor %}
{% else %}

No alerts recorded yet.

{% endif %} {% if has_next %}
{% endif %}
================================================ FILE: templates/partials/analytics.html ================================================

Top Topics by Rate

No analytics data yet

Payload Stats:

No numeric fields detected in payloads
================================================ FILE: templates/partials/brokers.html ================================================

Broker Connections

Comma-separated. Supports wildcards: +, #
No brokers configured. Add one above.
================================================ FILE: templates/partials/dry_run_result.html ================================================

Dry-Run Test

Topic match:
Condition match:
================================================ FILE: templates/partials/plugins.html ================================================
{% if plugins %} {% for plugin in plugins %}
{{ plugin.name }} {% if plugin.version %} v{{ plugin.version }} {% endif %}
{% if plugin.enabled %} Enabled {% else %} Disabled {% endif %}
{% if plugin.description %}

{{ plugin.description }}

{% endif %}
{{ plugin.entry_point }} {% if plugin.installed_at %} Installed: {{ plugin.installed_at }} {% endif %}
{% endfor %} {% else %}

No plugins installed

Install plugins via pip and restart MQTTUI to discover them.

{% endif %}
================================================ FILE: templates/partials/rule_form.html ================================================
{% if rule %} {% endif %}

{{ 'Edit Rule' if rule else 'New Rule' }}

Supports MQTT wildcards: + (single level), # (multi level)
Custom message template (optional — has a good default) Supports Markdown formatting
Custom message template (optional — has a good default) Uses Slack mrkdwn formatting
================================================ FILE: templates/partials/rule_row.html ================================================
{{ rule.name }} {{ 'Active' if rule.enabled else 'Disabled' }} {% if rule.schedule_cron %} Scheduled {% endif %}
Topic: {{ rule.trigger_topic }} | Action: {{ rule.action.type if rule.action else 'none' }} | Fired: {{ rule.fire_count }}x
{% if rule.description %}
{{ rule.description }}
{% endif %}
{% if rule.enabled %} {% else %} {% endif %}
================================================ FILE: templates/partials/rules_list.html ================================================

Automation Rules

{% if rules %}
{% for rule in rules %} {% include 'partials/rule_row.html' %} {% endfor %}
{% else %}

No rules defined yet. Create one to get started.

{% endif %}
================================================ FILE: tests/__init__.py ================================================ ================================================ FILE: tests/conftest.py ================================================ import pytest import tempfile import os from unittest.mock import patch, MagicMock @pytest.fixture def app(tmp_path): """Create application for testing with mocked MQTT.""" db_path = str(tmp_path / "test.db") # Patch init_mqtt to prevent real MQTT broker connection with patch('mqttui.mqtt_client.init_mqtt') as mock_init: from mqttui.app import create_app app = create_app({ 'TESTING': True, 'SECRET_KEY': 'test-secret-key', 'SQLALCHEMY_DATABASE_URI': f'sqlite:///{tmp_path}/test_users.db', 'SQLALCHEMY_TRACK_MODIFICATIONS': False, 'DB_ENABLED': True, 'DB_PATH': db_path, 'DB_MAX_MESSAGES': 1000, 'MQTT_BROKER': '127.0.0.1', 'MQTT_PORT': 1883, 'MQTT_VERSION': '3.1.1', 'MQTT_TOPICS': '#', 'MQTT_USERNAME': None, 'MQTT_PASSWORD': None, 'MQTT_KEEPALIVE': 60, }) yield app @pytest.fixture def client(app): """Flask test client.""" return app.test_client() @pytest.fixture def test_db(tmp_path): """Isolated MessageDatabase for testing.""" db_path = str(tmp_path / "test_messages.db") from mqttui.database import MessageDatabase db = MessageDatabase(db_path, max_messages=100) yield db db.close() @pytest.fixture def auth_client(app): """Flask test client with authenticated session.""" client = app.test_client() with app.app_context(): from mqttui.models import User from mqttui.extensions import sa user = User.query.filter_by(username='admin').first() if not user: user = User(username='admin') user.set_password('admin') user.generate_api_token() sa.session.add(user) sa.session.commit() # Log in via test client client.post('/login', data={'username': 'admin', 'password': 'admin'}, follow_redirects=False) return client @pytest.fixture def api_token(app): """Return a valid API token for testing.""" with app.app_context(): from mqttui.models import User user = User.query.filter_by(username='admin').first() return user.api_token @pytest.fixture def mock_mqtt(): """Mock MQTT client to prevent broker connections.""" with patch('mqttui.mqtt_client._mqtt_client') as mock_client: mock_client.publish = MagicMock() mock_client.connect = MagicMock() mock_client.subscribe = MagicMock() yield mock_client ================================================ FILE: tests/test_alerts_api.py ================================================ """Tests for the Alerts REST API and dry-run action_preview.""" import json from datetime import datetime import pytest from mqttui.extensions import sa from mqttui.rules.models import AlertHistory, Rule # --------------------------------------------------------------------------- # Helper to seed alerts # --------------------------------------------------------------------------- def _seed_alerts(app, count=25, rule_id=1, severity='info'): """Seed AlertHistory records for testing.""" with app.app_context(): for i in range(count): record = AlertHistory( rule_id=rule_id, rule_name=f'rule-{rule_id}', topic=f'sensors/temp/{i}', severity=severity, message=f'Test alert {i}', fired_at=datetime(2026, 1, 1, 0, i % 60), ) sa.session.add(record) sa.session.commit() def _seed_rule(app, rule_id=None, action_type='webhook'): """Seed a Rule record and return its ID.""" with app.app_context(): action = {'type': action_type} if action_type == 'webhook': action['url'] = 'https://example.com/hook' action['payload_template'] = '{"topic": "{{topic}}", "rule": "{{rule_name}}"}' elif action_type == 'publish': action['topic'] = 'output/topic' action['payload'] = '{"forwarded": true}' elif action_type == 'log': action['message'] = 'Rule fired: {{rule_name}}' action['severity'] = 'warning' rule = Rule( name='test-rule', trigger_topic='sensors/+', condition_json='{"op": "gt", "path": "temp", "value": 30}', action_json=json.dumps(action), enabled=True, ) sa.session.add(rule) sa.session.commit() return rule.id # --------------------------------------------------------------------------- # GET /api/v1/alerts tests # --------------------------------------------------------------------------- class TestListAlerts: """Tests for the alert history listing endpoint.""" def test_list_alerts_empty(self, auth_client, app): """Empty database returns empty list with total 0.""" resp = auth_client.get('/api/v1/alerts/') assert resp.status_code == 200 data = resp.get_json() assert data['status'] == 'success' assert data['data']['alerts'] == [] assert data['data']['total'] == 0 def test_list_alerts_paginated(self, auth_client, app): """With 25 alerts, page=1&per_page=10 returns 10 items and total=25.""" _seed_alerts(app, count=25) resp = auth_client.get('/api/v1/alerts/?page=1&per_page=10') assert resp.status_code == 200 data = resp.get_json()['data'] assert len(data['alerts']) == 10 assert data['total'] == 25 assert data['page'] == 1 assert data['per_page'] == 10 def test_list_alerts_filter_rule_id(self, auth_client, app): """Filter by rule_id returns only matching alerts.""" _seed_alerts(app, count=5, rule_id=1) _seed_alerts(app, count=3, rule_id=2) resp = auth_client.get('/api/v1/alerts/?rule_id=1') data = resp.get_json()['data'] assert data['total'] == 5 for alert in data['alerts']: assert alert['rule_id'] == 1 def test_list_alerts_filter_severity(self, auth_client, app): """Filter by severity returns only matching alerts.""" _seed_alerts(app, count=4, severity='info') _seed_alerts(app, count=2, severity='error') resp = auth_client.get('/api/v1/alerts/?severity=error') data = resp.get_json()['data'] assert data['total'] == 2 for alert in data['alerts']: assert alert['severity'] == 'error' def test_list_alerts_requires_auth(self, client): """Unauthenticated request should be rejected.""" resp = client.get('/api/v1/alerts/') # Flask-Login redirects to login page (302) or returns 401 assert resp.status_code in (302, 401) # --------------------------------------------------------------------------- # Dry-run action_preview test # --------------------------------------------------------------------------- class TestDryRunActionPreview: """Test that dry-run test endpoint returns action_preview.""" def test_dry_run_action_preview(self, auth_client, app): """POST test with matching payload returns action_preview with rendered webhook payload.""" rule_id = _seed_rule(app, action_type='webhook') resp = auth_client.post( f'/api/v1/rules/{rule_id}/test', json={'topic': 'sensors/temp', 'payload': '{"temp": 42}'}, content_type='application/json', ) assert resp.status_code == 200 data = resp.get_json()['data'] assert data['match'] is True assert 'action_preview' in data preview = data['action_preview'] assert len(preview) > 0 # Webhook preview should contain the URL and rendered payload wp = preview[0] assert wp['type'] == 'webhook' assert 'url' in wp assert 'payload' in wp def test_dry_run_action_preview_publish(self, auth_client, app): """Publish action preview shows topic and payload.""" rule_id = _seed_rule(app, action_type='publish') resp = auth_client.post( f'/api/v1/rules/{rule_id}/test', json={'topic': 'sensors/temp', 'payload': '{"temp": 42}'}, content_type='application/json', ) assert resp.status_code == 200 data = resp.get_json()['data'] if data['match']: assert 'action_preview' in data wp = data['action_preview'] assert len(wp) > 0 assert wp[0]['type'] == 'publish' def test_dry_run_action_preview_log(self, auth_client, app): """Log action preview shows message.""" rule_id = _seed_rule(app, action_type='log') resp = auth_client.post( f'/api/v1/rules/{rule_id}/test', json={'topic': 'sensors/temp', 'payload': '{"temp": 42}'}, content_type='application/json', ) assert resp.status_code == 200 data = resp.get_json()['data'] if data['match']: assert 'action_preview' in data wp = data['action_preview'] assert len(wp) > 0 assert wp[0]['type'] == 'log' def test_dry_run_no_preview_when_no_match(self, auth_client, app): """No action_preview when rule doesn't match.""" rule_id = _seed_rule(app, action_type='webhook') resp = auth_client.post( f'/api/v1/rules/{rule_id}/test', json={'topic': 'sensors/temp', 'payload': '{"temp": 10}'}, content_type='application/json', ) assert resp.status_code == 200 data = resp.get_json()['data'] assert data['match'] is False # action_preview should be empty or absent when no match preview = data.get('action_preview', []) assert len(preview) == 0 ================================================ FILE: tests/test_analytics.py ================================================ """Tests for the per-topic analytics engine.""" import time import json import pytest from mqttui.analytics import TopicAnalytics, get_analytics class TestTopicAnalytics: """Unit tests for TopicAnalytics engine.""" def setup_method(self): self.analytics = TopicAnalytics() def test_record_increments_message_count(self): """Test 1: record() increments message count for topic.""" self.analytics.record("home/temp", '{"temperature": 22.5}', time.time()) self.analytics.record("home/temp", '{"temperature": 23.0}', time.time()) stats = self.analytics.get_topic_stats("home/temp") assert stats["message_count"] == 2 def test_get_rate_per_minute(self): """Test 2: get_rate(topic, window=60) returns msgs in last 60s.""" now = time.time() for i in range(5): self.analytics.record("sensor/a", "payload", now - i) rate = self.analytics.get_rate("sensor/a", window=60) assert rate == 5.0 def test_get_rate_per_hour(self): """Test 3: get_rate(topic, window=3600) returns msgs in last hour.""" now = time.time() for i in range(10): self.analytics.record("sensor/b", "payload", now - i * 60) rate = self.analytics.get_rate("sensor/b", window=3600) assert rate == 10.0 def test_expired_timestamps_not_counted(self): """Test 4: Timestamps outside window are not counted.""" now = time.time() # Record 3 messages: 2 within window, 1 expired self.analytics.record("sensor/c", "x", now - 120) # 2 min ago (outside 60s) self.analytics.record("sensor/c", "x", now - 30) # 30s ago (inside 60s) self.analytics.record("sensor/c", "x", now - 10) # 10s ago (inside 60s) rate = self.analytics.get_rate("sensor/c", window=60) assert rate == 2.0 def test_numeric_payload_updates_histogram(self): """Test 5: Numeric JSON payload updates histogram stats.""" self.analytics.record("sensor/d", '{"temperature": 22.5}', time.time()) self.analytics.record("sensor/d", '{"temperature": 25.0}', time.time()) self.analytics.record("sensor/d", '{"temperature": 20.0}', time.time()) stats = self.analytics.get_topic_stats("sensor/d") hist = stats["histograms"]["temperature"] assert hist["min"] == 20.0 assert hist["max"] == 25.0 assert hist["count"] == 3 assert abs(hist["sum"] - 67.5) < 0.01 def test_non_numeric_payload_does_not_crash(self): """Test 6: Non-numeric/non-JSON payloads are handled gracefully.""" self.analytics.record("sensor/e", "not json", time.time()) self.analytics.record("sensor/e", '{"status": "online"}', time.time()) self.analytics.record("sensor/e", "", time.time()) stats = self.analytics.get_topic_stats("sensor/e") assert stats["message_count"] == 3 assert stats["histograms"] == {} def test_get_topic_stats_returns_full_dict(self): """Test 7: get_topic_stats() returns dict with rate and histogram data.""" now = time.time() self.analytics.record("sensor/f", '{"humidity": 60}', now) stats = self.analytics.get_topic_stats("sensor/f") assert stats["topic"] == "sensor/f" assert "rate_per_min" in stats assert "rate_per_hour" in stats assert "message_count" in stats assert "histograms" in stats assert "humidity" in stats["histograms"] def test_get_all_stats_sorted_by_rate(self): """Test 8: get_all_stats() returns topics sorted by rate descending.""" now = time.time() # Topic A: 1 message self.analytics.record("topic/a", "x", now) # Topic B: 3 messages (highest rate) for _ in range(3): self.analytics.record("topic/b", "x", now) # Topic C: 2 messages for _ in range(2): self.analytics.record("topic/c", "x", now) all_stats = self.analytics.get_all_stats(limit=20) assert len(all_stats) == 3 assert all_stats[0]["topic"] == "topic/b" assert all_stats[1]["topic"] == "topic/c" assert all_stats[2]["topic"] == "topic/a" class TestAnalyticsAPI: """Integration tests for analytics REST API endpoints.""" def test_get_topics_returns_200(self, auth_client, app): """GET /api/v1/analytics/topics returns 200 with topics array.""" # Seed some analytics data from mqttui.analytics import get_analytics analytics = get_analytics() now = time.time() analytics.record("test/topic", '{"value": 42}', now) resp = auth_client.get('/api/v1/analytics/topics') assert resp.status_code == 200 data = resp.get_json() assert data["status"] == "success" assert "topics" in data["data"] assert isinstance(data["data"]["topics"], list) def test_get_single_topic_returns_200(self, auth_client, app): """GET /api/v1/analytics/topics/ returns 200 with stats.""" from mqttui.analytics import get_analytics analytics = get_analytics() now = time.time() analytics.record("home/sensor", '{"temp": 22}', now) resp = auth_client.get('/api/v1/analytics/topics/home/sensor') assert resp.status_code == 200 data = resp.get_json() assert data["status"] == "success" assert data["data"]["topic"] == "home/sensor" def test_get_unknown_topic_returns_404(self, auth_client): """GET /api/v1/analytics/topics/ returns 404.""" resp = auth_client.get('/api/v1/analytics/topics/nonexistent/topic') assert resp.status_code == 404 data = resp.get_json() assert data["status"] == "error" assert data["error"]["code"] == "NOT_FOUND" def test_unauthenticated_request_rejected(self, client): """Unauthenticated request to analytics API is rejected.""" resp = client.get('/api/v1/analytics/topics') # Flask-Login redirects to login page (302) or returns 401 assert resp.status_code in (302, 401) class TestAnalyticsPartial: """Tests for analytics partial template route.""" def test_analytics_partial_returns_200(self, auth_client): """GET /partials/analytics returns 200 with analyticsWidget component.""" resp = auth_client.get('/partials/analytics') assert resp.status_code == 200 assert b'analyticsWidget' in resp.data def test_analytics_partial_contains_rate_per_min(self, auth_client): """Analytics partial contains rate_per_min display binding.""" resp = auth_client.get('/partials/analytics') assert resp.status_code == 200 assert b'rate_per_min' in resp.data def test_analytics_partial_requires_auth(self, client): """Unauthenticated request to analytics partial is rejected.""" resp = client.get('/partials/analytics') assert resp.status_code in (302, 401) ================================================ FILE: tests/test_api_v1.py ================================================ """Tests for API v1 endpoints.""" import json def test_json_envelope_success(auth_client): """API responses follow success envelope format.""" r = auth_client.get('/api/v1/version') data = json.loads(r.data) assert 'status' in data assert 'data' in data assert 'error' in data assert data['status'] == 'success' assert data['error'] is None def test_json_envelope_error(auth_client): """API error responses follow error envelope format.""" r = auth_client.post('/api/v1/publish', json={}) if r.status_code >= 400: data = json.loads(r.data) assert data['status'] == 'error' assert data['error'] is not None assert 'code' in data['error'] assert 'message' in data['error'] def test_messages_endpoint(auth_client): """GET /api/v1/messages returns envelope with messages array.""" r = auth_client.get('/api/v1/messages') assert r.status_code == 200 data = json.loads(r.data) assert data['status'] == 'success' assert 'messages' in data['data'] def test_topics_endpoint(auth_client): """GET /api/v1/topics returns envelope with topics array.""" r = auth_client.get('/api/v1/topics') assert r.status_code == 200 data = json.loads(r.data) assert data['status'] == 'success' assert 'topics' in data['data'] def test_version_public(client): """GET /api/v1/version is public (no auth required).""" r = client.get('/api/v1/version') assert r.status_code == 200 data = json.loads(r.data) assert data['status'] == 'success' def test_docs_endpoint(client): """GET /api/v1/docs returns Swagger UI.""" r = client.get('/api/v1/docs') assert r.status_code == 200 assert b'swagger-ui' in r.data def test_openapi_spec(client): """GET /api/v1/openapi.json returns valid OpenAPI spec.""" r = client.get('/api/v1/openapi.json') assert r.status_code == 200 data = json.loads(r.data) assert 'openapi' in data or 'swagger' in data assert 'info' in data assert 'paths' in data def test_cors_headers(client): """API responses include CORS headers.""" r = client.options('/api/v1/version', headers={ 'Origin': 'http://example.com', 'Access-Control-Request-Method': 'GET' }) # Flask-CORS should add Access-Control-Allow-Origin assert r.status_code in (200, 204) def test_publish_requires_auth(client): """POST /api/v1/publish without auth is rejected.""" r = client.post('/api/v1/publish', json={'topic': 'test', 'message': 'hello'}) assert r.status_code in (302, 401) def test_stats_requires_auth(client): """GET /api/v1/stats without auth is rejected.""" r = client.get('/api/v1/stats') assert r.status_code in (302, 401) def test_messages_requires_auth(client): """GET /api/v1/messages without auth is rejected.""" r = client.get('/api/v1/messages') assert r.status_code in (302, 401) def test_rate_limit_publish(auth_client, app): """POST /api/v1/publish is rate limited to 30/minute with Retry-After header.""" from unittest.mock import patch with patch('mqttui.mqtt_client.publish'): hit_429 = False for i in range(35): r = auth_client.post('/api/v1/publish', json={'topic': 'test/rate', 'message': f'msg-{i}'}) if r.status_code == 429: hit_429 = True assert 'Retry-After' in r.headers data = json.loads(r.data) assert data['status'] == 'error' assert data['error']['code'] == 'RATE_LIMIT_EXCEEDED' break # Rate limiter should have kicked in after 30 requests assert hit_429, "Expected 429 after exceeding rate limit" ================================================ FILE: tests/test_app_factory.py ================================================ def test_create_app(app): assert app is not None assert app.testing is True def test_create_app_has_blueprints(app): blueprint_names = list(app.blueprints.keys()) assert 'main' in blueprint_names assert 'api' in blueprint_names assert 'debug' in blueprint_names def test_create_app_custom_config(tmp_path): from unittest.mock import patch with patch('mqttui.mqtt_client.init_mqtt'): from mqttui.app import create_app app = create_app({ 'SECRET_KEY': 'custom-key', 'DB_ENABLED': False, 'MQTT_BROKER': '127.0.0.1', 'MQTT_PORT': 1883, 'MQTT_VERSION': '3.1.1', 'MQTT_TOPICS': '#', 'MQTT_USERNAME': None, 'MQTT_PASSWORD': None, 'MQTT_KEEPALIVE': 60, }) assert app.config['SECRET_KEY'] == 'custom-key' def test_socketio_async_mode(app): from mqttui.extensions import socketio assert socketio.async_mode == 'gevent' ================================================ FILE: tests/test_auth.py ================================================ """Tests for authentication flow.""" def test_login_page_loads(client): """GET /login returns 200 with login form.""" r = client.get('/login') assert r.status_code == 200 assert b'username' in r.data assert b'password' in r.data def test_unauthenticated_redirect(client): """Unauthenticated GET / redirects to /login.""" r = client.get('/') assert r.status_code == 302 assert '/login' in r.headers['Location'] def test_login_valid_credentials(client, app): """POST /login with valid creds redirects to /.""" with app.app_context(): from mqttui.models import User from mqttui.extensions import sa user = User.query.filter_by(username='admin').first() if not user: user = User(username='admin') user.set_password('testpass') sa.session.add(user) sa.session.commit() r = client.post('/login', data={'username': 'admin', 'password': 'admin'}, follow_redirects=False) assert r.status_code == 302 assert r.headers['Location'] in ('/', 'http://localhost/') def test_login_invalid_credentials(client, app): """POST /login with bad creds returns login page with error.""" r = client.post('/login', data={'username': 'admin', 'password': 'wrong'}) assert r.status_code == 200 assert b'Invalid' in r.data or b'invalid' in r.data def test_logout(auth_client): """GET /logout clears session and redirects.""" r = auth_client.get('/logout', follow_redirects=False) assert r.status_code == 302 assert '/login' in r.headers['Location'] def test_api_token_auth(client, api_token): """X-API-Key header authenticates API requests.""" r = client.get('/api/v1/stats', headers={'X-API-Key': api_token}) assert r.status_code == 200 import json data = json.loads(r.data) assert data['status'] == 'success' def test_api_token_invalid(client): """Invalid X-API-Key returns 401/redirect.""" r = client.get('/api/v1/stats', headers={'X-API-Key': 'bad-token'}) # Should redirect to login or return 401 assert r.status_code in (302, 401) def test_secret_key_guard(): """App refuses to start with insecure SECRET_KEY in production.""" import os old_env = os.environ.get('FLASK_ENV') os.environ['FLASK_ENV'] = 'production' try: from unittest.mock import patch import pytest with patch('mqttui.mqtt_client.init_mqtt'): from mqttui.app import create_app with pytest.raises(RuntimeError, match="SECRET_KEY"): create_app({'SECRET_KEY': 'dev', 'DB_ENABLED': False, 'SQLALCHEMY_DATABASE_URI': 'sqlite://'}) finally: if old_env is None: os.environ.pop('FLASK_ENV', None) else: os.environ['FLASK_ENV'] = old_env def test_token_get(auth_client, app): """GET /api/v1/auth/token returns current token.""" r = auth_client.get('/api/v1/auth/token') assert r.status_code == 200 import json data = json.loads(r.data) assert data['status'] == 'success' assert 'api_token' in data['data'] def test_token_regenerate(auth_client, app): """POST /api/v1/auth/token regenerates token.""" r = auth_client.post('/api/v1/auth/token') assert r.status_code == 200 import json data = json.loads(r.data) assert data['status'] == 'success' assert data['data']['api_token'] is not None def test_token_revoke(auth_client, app): """DELETE /api/v1/auth/token revokes token.""" r = auth_client.delete('/api/v1/auth/token') assert r.status_code == 200 import json data = json.loads(r.data) assert data['data']['message'] == 'API token revoked' ================================================ FILE: tests/test_cooldown.py ================================================ """Tests for CooldownTracker and webhook cooldown integration.""" import time from unittest.mock import patch, MagicMock import pytest from mqttui.rules.cooldown import CooldownTracker, cooldown_tracker # --------------------------------------------------------------------------- # CooldownTracker unit tests # --------------------------------------------------------------------------- class TestCooldownTracker: """Tests for the per-rule in-memory cooldown tracker.""" def test_cooldown_allows_first_alert(self): """First call for a rule_id should always be allowed.""" tracker = CooldownTracker() assert tracker.check(rule_id=1) is True def test_cooldown_blocks_during_window(self): """Subsequent calls within the cooldown window should be blocked.""" tracker = CooldownTracker() assert tracker.check(rule_id=1) is True assert tracker.check(rule_id=1) is False def test_cooldown_allows_after_expiry(self): """After cooldown expires, alert should be allowed again.""" tracker = CooldownTracker(default_seconds=1) assert tracker.check(rule_id=1) is True # Mock time to advance past cooldown window with patch('mqttui.rules.cooldown.time') as mock_time: mock_time.monotonic.return_value = time.monotonic() + 2 assert tracker.check(rule_id=1) is True def test_cooldown_different_rules_independent(self): """Cooldown for one rule should not affect another.""" tracker = CooldownTracker() assert tracker.check(rule_id=1) is True assert tracker.check(rule_id=2) is True # rule_id=1 still blocked assert tracker.check(rule_id=1) is False # rule_id=2 also blocked assert tracker.check(rule_id=2) is False def test_cooldown_custom_window(self): """CooldownTracker should respect custom default_seconds.""" tracker = CooldownTracker(default_seconds=60) assert tracker.check(rule_id=1) is True # Even 50 seconds later, still blocked (60s window) with patch('mqttui.rules.cooldown.time') as mock_time: mock_time.monotonic.return_value = time.monotonic() + 50 assert tracker.check(rule_id=1) is False # After 61 seconds, allowed again with patch('mqttui.rules.cooldown.time') as mock_time: mock_time.monotonic.return_value = time.monotonic() + 61 assert tracker.check(rule_id=1) is True def test_cooldown_suppressed_count(self): """Suppressed count should track how many alerts were blocked.""" tracker = CooldownTracker() assert tracker.check(rule_id=1) is True assert tracker.get_suppressed_count(rule_id=1) == 0 # Suppress 3 alerts tracker.check(rule_id=1) tracker.check(rule_id=1) tracker.check(rule_id=1) assert tracker.get_suppressed_count(rule_id=1) == 3 def test_cooldown_suppressed_count_resets_on_new_fire(self): """Suppressed count resets when a new alert is allowed.""" tracker = CooldownTracker(default_seconds=1) assert tracker.check(rule_id=1) is True tracker.check(rule_id=1) # suppressed tracker.check(rule_id=1) # suppressed assert tracker.get_suppressed_count(rule_id=1) == 2 # After cooldown expires, count resets with patch('mqttui.rules.cooldown.time') as mock_time: mock_time.monotonic.return_value = time.monotonic() + 2 assert tracker.check(rule_id=1) is True assert tracker.get_suppressed_count(rule_id=1) == 0 # --------------------------------------------------------------------------- # Webhook cooldown integration test # --------------------------------------------------------------------------- class TestWebhookCooldownIntegration: """Test that _execute_webhook respects cooldown.""" def test_webhook_skipped_during_cooldown(self, app): """Webhook should return cooldown detail when blocked.""" from mqttui.rules.cooldown import cooldown_tracker as ct from mqttui.rules.actions import _execute_webhook with app.app_context(): context = { 'rule_id': 99, 'rule_name': 'test-rule', 'topic': 'sensors/temp', 'payload': '{"temp": 42}', } action = {'type': 'webhook', 'url': 'https://example.com/hook'} # Reset the module-level tracker for clean test ct._last_fired.clear() ct._suppressed.clear() # First call -- allowed (submitted to thread pool) result1 = _execute_webhook(action, context) assert result1['success'] is True assert 'cooldown' not in result1['detail'].lower() # Second call -- blocked by cooldown result2 = _execute_webhook(action, context) assert result2['success'] is True assert 'cooldown' in result2['detail'].lower() or 'suppressed' in result2['detail'].lower() ================================================ FILE: tests/test_database.py ================================================ from datetime import datetime def test_wal_mode(test_db): conn = test_db.get_connection() mode = conn.execute('PRAGMA journal_mode').fetchone()[0] assert mode == 'wal' def test_busy_timeout(test_db): conn = test_db.get_connection() timeout = conn.execute('PRAGMA busy_timeout').fetchone()[0] assert timeout == 5000 def test_store_and_retrieve(test_db): now = datetime.now() test_db.store_message( topic='test/topic', payload='hello world', timestamp=now, qos=0, retain=False ) messages = test_db.get_messages(limit=10) assert len(messages) >= 1 assert messages[0]['topic'] == 'test/topic' assert messages[0]['payload'] == 'hello world' def test_get_topics(test_db): test_db.store_message(topic='sensor/temp', payload='22.5', timestamp=datetime.now()) test_db.store_message(topic='sensor/humidity', payload='45', timestamp=datetime.now()) topics = test_db.get_topics() topic_names = [t['topic'] for t in topics] assert 'sensor/temp' in topic_names assert 'sensor/humidity' in topic_names def test_message_count(test_db): for i in range(5): test_db.store_message(topic='count/test', payload=str(i), timestamp=datetime.now()) count = test_db.get_message_count(topic_filter='count/test') assert count == 5 ================================================ FILE: tests/test_frontend.py ================================================ """Tests for Phase 5 frontend: partials, Alpine.js, htmx, batch emitter.""" import pytest from unittest.mock import MagicMock, patch class TestPartialRoutes: """Test htmx partial template routes.""" def test_rules_list_partial_returns_html(self, auth_client): resp = auth_client.get('/partials/rules') assert resp.status_code == 200 assert b'rules-list' in resp.data def test_alerts_list_partial_returns_html(self, auth_client): resp = auth_client.get('/partials/alerts') assert resp.status_code == 200 assert b'alerts-list' in resp.data def test_rule_form_partial_returns_html(self, auth_client): resp = auth_client.get('/partials/rules/form') assert resp.status_code == 200 assert b'name' in resp.data def test_partials_require_auth(self, client): """Unauthenticated requests should redirect to login.""" for url in ['/partials/rules', '/partials/alerts', '/partials/rules/form']: resp = client.get(url) assert resp.status_code in (302, 401), f"{url} should require auth" class TestIndexTemplate: """Test main index template includes Alpine.js, htmx, and tabs.""" def test_index_contains_alpine(self, auth_client): resp = auth_client.get('/') assert resp.status_code == 200 assert b'alpinejs' in resp.data or b'alpine' in resp.data.lower() def test_index_contains_htmx(self, auth_client): resp = auth_client.get('/') assert b'htmx' in resp.data def test_index_contains_tabs(self, auth_client): resp = auth_client.get('/') html = resp.data.decode() assert 'Messages' in html assert 'Rules' in html assert 'Alerts' in html def test_index_contains_alpine_xdata(self, auth_client): resp = auth_client.get('/') assert b'x-data' in resp.data class TestBatchEmitter: """Test server-side Socket.IO batch emitter.""" def test_enqueue_adds_to_buffer(self): from mqttui.socketio_batch import BatchEmitter mock_sio = MagicMock() emitter = BatchEmitter(mock_sio, interval_ms=100) emitter.enqueue({'topic': 'test', 'payload': 'hello'}) assert len(emitter._buffer) == 1 def test_flush_emits_batch_and_clears(self): from mqttui.socketio_batch import BatchEmitter mock_sio = MagicMock() emitter = BatchEmitter(mock_sio, interval_ms=100) emitter.enqueue({'topic': 't1', 'payload': 'p1'}) emitter.enqueue({'topic': 't2', 'payload': 'p2'}) emitter._flush() mock_sio.emit.assert_called_once_with('mqtt_messages_batch', [ {'topic': 't1', 'payload': 'p1'}, {'topic': 't2', 'payload': 'p2'}, ]) assert len(emitter._buffer) == 0 def test_flush_noop_when_empty(self): from mqttui.socketio_batch import BatchEmitter mock_sio = MagicMock() emitter = BatchEmitter(mock_sio, interval_ms=100) emitter._flush() mock_sio.emit.assert_not_called() ================================================ FILE: tests/test_observability.py ================================================ """Tests for structured logging (structlog) and Prometheus metrics endpoint.""" import pytest from unittest.mock import patch class TestStructlogConfiguration: """Tests for mqttui.logging_config.configure_logging.""" def test_configure_logging_debug_uses_console_renderer(self): """configure_logging(debug=True) sets up colored console renderer.""" import structlog from mqttui.logging_config import configure_logging configure_logging(debug=True) # structlog should be configured -- get a logger and verify it works logger = structlog.get_logger("test") assert logger is not None # In debug mode, the formatter should use ConsoleRenderer (not JSON) import logging root = logging.getLogger() assert len(root.handlers) > 0 formatter = root.handlers[0].formatter # ProcessorFormatter stores processors list assert hasattr(formatter, 'processors') processor_names = [type(p).__name__ for p in formatter.processors] assert 'ConsoleRenderer' in processor_names def test_configure_logging_production_uses_json_renderer(self): """configure_logging(debug=False) sets up JSON renderer.""" import structlog from mqttui.logging_config import configure_logging configure_logging(debug=False) import logging root = logging.getLogger() assert len(root.handlers) > 0 formatter = root.handlers[0].formatter assert hasattr(formatter, 'processors') processor_names = [type(p).__name__ for p in formatter.processors] assert 'JSONRenderer' in processor_names def test_structlog_get_logger_returns_bound_logger(self): """structlog.get_logger() returns a logger with .info() and .error().""" import structlog from mqttui.logging_config import configure_logging configure_logging(debug=True) logger = structlog.get_logger("test.bound") assert callable(getattr(logger, 'info', None)) assert callable(getattr(logger, 'error', None)) class TestPrometheusMetricsEndpoint: """Tests for the /metrics Prometheus scrape endpoint.""" def test_metrics_returns_200(self, client): """GET /metrics returns 200 with text/plain content type.""" response = client.get('/metrics') assert response.status_code == 200 assert 'text/plain' in response.content_type def test_metrics_contains_mqtt_messages_total(self, client): """The /metrics response contains mqtt_messages_total counter.""" response = client.get('/metrics') assert b'mqtt_messages_total' in response.data def test_metrics_contains_mqtt_connected_gauge(self, client): """The /metrics response contains mqtt_connected gauge.""" response = client.get('/metrics') assert b'mqtt_connected' in response.data def test_metrics_contains_rule_firings_total(self, client): """The /metrics response contains rule_firings_total counter.""" response = client.get('/metrics') assert b'rule_firings_total' in response.data def test_metrics_no_auth_required(self, client): """The /metrics endpoint does NOT require authentication.""" # client is unauthenticated -- should still get 200, not 302/401 response = client.get('/metrics') assert response.status_code == 200 ================================================ FILE: tests/test_plugin_registry.py ================================================ """Tests for plugin hookspec, model, and registry.""" import pytest from unittest.mock import patch, MagicMock @pytest.fixture(autouse=True) def _ensure_plugin_table(app): """Ensure plugin_configs table exists (before Task 2 wires it into create_app).""" with app.app_context(): from mqttui.plugins.models import PluginConfig # noqa: F401 from mqttui.extensions import sa sa.create_all() class TestMQTTUIPluginHookspec: """Test that MQTTUIPlugin defines the expected hookspec methods.""" def test_has_on_message_hookspec(self): from mqttui.plugins.hookspec import MQTTUIPlugin assert hasattr(MQTTUIPlugin, 'on_message') def test_has_on_connect_hookspec(self): from mqttui.plugins.hookspec import MQTTUIPlugin assert hasattr(MQTTUIPlugin, 'on_connect') def test_has_on_rule_trigger_hookspec(self): from mqttui.plugins.hookspec import MQTTUIPlugin assert hasattr(MQTTUIPlugin, 'on_rule_trigger') def test_hookspec_markers_applied(self): from mqttui.plugins.hookspec import MQTTUIPlugin # pluggy marks hookspec methods with a special attribute assert hasattr(MQTTUIPlugin.on_message, 'mqttui_spec') assert hasattr(MQTTUIPlugin.on_connect, 'mqttui_spec') assert hasattr(MQTTUIPlugin.on_rule_trigger, 'mqttui_spec') def test_hookimpl_marker_exported(self): from mqttui.plugins.hookspec import hookimpl assert hookimpl is not None class TestPluginConfigModel: """Test PluginConfig SQLAlchemy model CRUD.""" def test_create_plugin_config(self, app): with app.app_context(): from mqttui.plugins.models import PluginConfig from mqttui.extensions import sa pc = PluginConfig( name='test-plugin', entry_point='test_plugin:TestPlugin', enabled=False, config_json='{"key": "value"}', ) sa.session.add(pc) sa.session.commit() found = PluginConfig.query.filter_by(name='test-plugin').first() assert found is not None assert found.name == 'test-plugin' assert found.entry_point == 'test_plugin:TestPlugin' assert found.enabled is False assert found.config_json == '{"key": "value"}' def test_update_enabled_state(self, app): with app.app_context(): from mqttui.plugins.models import PluginConfig from mqttui.extensions import sa pc = PluginConfig( name='toggle-plugin', entry_point='toggle:Plugin', enabled=False, ) sa.session.add(pc) sa.session.commit() pc.enabled = True sa.session.commit() found = PluginConfig.query.filter_by(name='toggle-plugin').first() assert found.enabled is True def test_to_dict(self, app): with app.app_context(): from mqttui.plugins.models import PluginConfig from mqttui.extensions import sa pc = PluginConfig( name='dict-plugin', entry_point='dict:Plugin', enabled=True, config_json='{}', version='1.0.0', description='A test plugin', ) sa.session.add(pc) sa.session.commit() d = pc.to_dict() assert d['name'] == 'dict-plugin' assert d['entry_point'] == 'dict:Plugin' assert d['enabled'] is True assert d['version'] == '1.0.0' assert d['description'] == 'A test plugin' assert 'installed_at' in d class TestPluginRegistry: """Test PluginRegistry discover, list, enable, disable.""" def test_discover_with_no_entry_points(self, app): with app.app_context(): from mqttui.plugins.registry import PluginRegistry registry = PluginRegistry(app) with patch('mqttui.plugins.registry.entry_points', return_value=[]): result = registry.discover() assert result == [] def test_discover_with_mock_entry_point(self, app): with app.app_context(): from mqttui.plugins.registry import PluginRegistry from mqttui.extensions import sa registry = PluginRegistry(app) mock_ep = MagicMock() mock_ep.name = 'sample-plugin' mock_ep.value = 'sample_plugin:SamplePlugin' mock_ep.dist = MagicMock() mock_ep.dist.version = '0.1.0' mock_ep.dist.metadata = {'Summary': 'A sample plugin'} with patch('mqttui.plugins.registry.entry_points', return_value=[mock_ep]): result = registry.discover() assert len(result) == 1 assert result[0]['name'] == 'sample-plugin' def test_list_plugins(self, app): with app.app_context(): from mqttui.plugins.models import PluginConfig from mqttui.plugins.registry import PluginRegistry from mqttui.extensions import sa pc = PluginConfig( name='list-plugin', entry_point='list:Plugin', enabled=True, ) sa.session.add(pc) sa.session.commit() registry = PluginRegistry(app) plugins = registry.list_plugins() names = [p['name'] for p in plugins] assert 'list-plugin' in names def test_enable_plugin(self, app): with app.app_context(): from mqttui.plugins.models import PluginConfig from mqttui.plugins.registry import PluginRegistry from mqttui.extensions import sa pc = PluginConfig( name='enable-me', entry_point='enable:Plugin', enabled=False, ) sa.session.add(pc) sa.session.commit() registry = PluginRegistry(app) result = registry.enable('enable-me') assert result is True found = PluginConfig.query.filter_by(name='enable-me').first() assert found.enabled is True def test_disable_plugin(self, app): with app.app_context(): from mqttui.plugins.models import PluginConfig from mqttui.plugins.registry import PluginRegistry from mqttui.extensions import sa pc = PluginConfig( name='disable-me', entry_point='disable:Plugin', enabled=True, ) sa.session.add(pc) sa.session.commit() registry = PluginRegistry(app) result = registry.disable('disable-me') assert result is True found = PluginConfig.query.filter_by(name='disable-me').first() assert found.enabled is False def test_enable_nonexistent_returns_false(self, app): with app.app_context(): from mqttui.plugins.registry import PluginRegistry registry = PluginRegistry(app) assert registry.enable('no-such-plugin') is False def test_disable_nonexistent_returns_false(self, app): with app.app_context(): from mqttui.plugins.registry import PluginRegistry registry = PluginRegistry(app) assert registry.disable('no-such-plugin') is False def test_get_enabled_plugins(self, app): with app.app_context(): from mqttui.plugins.models import PluginConfig from mqttui.plugins.registry import PluginRegistry from mqttui.extensions import sa sa.session.add(PluginConfig(name='on-plugin', entry_point='on:P', enabled=True)) sa.session.add(PluginConfig(name='off-plugin', entry_point='off:P', enabled=False)) sa.session.commit() registry = PluginRegistry(app) enabled = registry.get_enabled_plugins() names = [p.name for p in enabled] assert 'on-plugin' in names assert 'off-plugin' not in names ================================================ FILE: tests/test_plugin_runner.py ================================================ """Tests for PluginRunner subprocess isolation and JSON protocol.""" import json import subprocess from unittest.mock import patch, MagicMock, PropertyMock import pytest from mqttui.plugins.runner import PluginRunner, PLUGIN_TIMEOUT @pytest.fixture def runner(): """Create a PluginRunner instance without app context.""" return PluginRunner(app=None) @pytest.fixture def mock_plugin(): """Create a mock PluginConfig.""" plugin = MagicMock() plugin.name = "test-plugin" plugin.entry_point = "test_plugin.main" plugin.enabled = True plugin.config_json = "{}" return plugin class TestCallPlugin: """Tests for call_plugin subprocess execution.""" def test_spawns_subprocess_with_correct_args(self, runner, mock_plugin): """call_plugin spawns subprocess with python -m entry_point.""" with patch("mqttui.plugins.runner.subprocess.Popen") as mock_popen: mock_proc = MagicMock() mock_proc.communicate.return_value = ('{"actions": []}', "") mock_proc.returncode = 0 mock_popen.return_value = mock_proc runner.call_plugin(mock_plugin, "on_message", {"topic": "t", "payload": "p"}) mock_popen.assert_called_once() call_args = mock_popen.call_args import sys assert call_args[0][0] == [sys.executable, "-m", "test_plugin.main"] def test_sends_json_on_stdin(self, runner, mock_plugin): """Subprocess receives JSON with event type and data on stdin.""" with patch("mqttui.plugins.runner.subprocess.Popen") as mock_popen: mock_proc = MagicMock() mock_proc.communicate.return_value = ('{"actions": []}', "") mock_proc.returncode = 0 mock_popen.return_value = mock_proc data = {"topic": "home/temp", "payload": "22.5"} runner.call_plugin(mock_plugin, "on_message", data) input_json = mock_proc.communicate.call_args[1].get("input") or mock_proc.communicate.call_args[0][0] parsed = json.loads(input_json) assert parsed["event"] == "on_message" assert parsed["data"] == data def test_parses_valid_json_response(self, runner, mock_plugin): """Subprocess response with actions list is parsed and returned.""" actions = [{"type": "publish", "topic": "out/t", "payload": "hello"}] with patch("mqttui.plugins.runner.subprocess.Popen") as mock_popen: mock_proc = MagicMock() mock_proc.communicate.return_value = (json.dumps({"actions": actions}), "") mock_proc.returncode = 0 mock_popen.return_value = mock_proc result = runner.call_plugin(mock_plugin, "on_message", {"topic": "t", "payload": "p"}) assert result == actions def test_timeout_kills_subprocess(self, runner, mock_plugin): """Subprocess exceeding timeout is killed and returns empty actions.""" with patch("mqttui.plugins.runner.subprocess.Popen") as mock_popen: mock_proc = MagicMock() mock_proc.communicate.side_effect = subprocess.TimeoutExpired(cmd="test", timeout=5) mock_popen.return_value = mock_proc result = runner.call_plugin(mock_plugin, "on_message", {"topic": "t", "payload": "p"}) mock_proc.kill.assert_called_once() assert result == [] def test_invalid_json_returns_empty(self, runner, mock_plugin): """Invalid JSON from subprocess is handled gracefully.""" with patch("mqttui.plugins.runner.subprocess.Popen") as mock_popen: mock_proc = MagicMock() mock_proc.communicate.return_value = ("not valid json!", "") mock_proc.returncode = 0 mock_popen.return_value = mock_proc result = runner.call_plugin(mock_plugin, "on_message", {"topic": "t", "payload": "p"}) assert result == [] def test_empty_env_for_security_isolation(self, runner, mock_plugin): """Subprocess is spawned with empty env dict for isolation.""" with patch("mqttui.plugins.runner.subprocess.Popen") as mock_popen: mock_proc = MagicMock() mock_proc.communicate.return_value = ('{"actions": []}', "") mock_proc.returncode = 0 mock_popen.return_value = mock_proc runner.call_plugin(mock_plugin, "on_message", {"topic": "t", "payload": "p"}) call_kwargs = mock_popen.call_args[1] assert call_kwargs["env"] == {} class TestDispatchMessage: """Tests for dispatch_message calling all enabled plugins.""" def test_calls_all_enabled_plugins(self, runner): """dispatch_message calls call_plugin for each enabled plugin.""" plugin1 = MagicMock(name="p1", entry_point="p1.main", enabled=True) plugin2 = MagicMock(name="p2", entry_point="p2.main", enabled=True) mock_registry = MagicMock() mock_registry.get_enabled_plugins.return_value = [plugin1, plugin2] with patch("mqttui.plugins.runner.get_plugin_registry", return_value=mock_registry): with patch.object(runner, "call_plugin", return_value=[]) as mock_call: runner.dispatch_message("test/topic", "payload") assert mock_call.call_count == 2 mock_call.assert_any_call(plugin1, "on_message", {"topic": "test/topic", "payload": "payload"}) mock_call.assert_any_call(plugin2, "on_message", {"topic": "test/topic", "payload": "payload"}) def test_collects_all_actions(self, runner): """dispatch_message aggregates actions from all plugins.""" plugin1 = MagicMock() plugin2 = MagicMock() mock_registry = MagicMock() mock_registry.get_enabled_plugins.return_value = [plugin1, plugin2] actions1 = [{"type": "publish", "topic": "a", "payload": "1"}] actions2 = [{"type": "log", "message": "hello"}] with patch("mqttui.plugins.runner.get_plugin_registry", return_value=mock_registry): with patch.object(runner, "call_plugin", side_effect=[actions1, actions2]): result = runner.dispatch_message("t", "p") assert len(result) == 2 assert actions1[0] in result assert actions2[0] in result class TestDispatchActions: """Tests for dispatch_actions handling different action types.""" def test_publish_action(self, runner): """Publish actions call mqtt_client.publish.""" actions = [{"type": "publish", "topic": "out/topic", "payload": "hello"}] with patch("mqttui.plugins.runner.mqttui.mqtt_client") as mock_mqtt: runner.dispatch_actions(actions) mock_mqtt.publish.assert_called_once_with("out/topic", "hello") def test_log_action(self, runner): """Log actions are logged without errors.""" actions = [{"type": "log", "message": "test log message"}] # Should not raise runner.dispatch_actions(actions) def test_unknown_action_type(self, runner): """Unknown action types are logged and skipped without crash.""" actions = [{"type": "unknown_action", "data": "something"}] # Should not raise runner.dispatch_actions(actions) class TestPluginTimeout: """Tests for timeout configuration.""" def test_timeout_is_5_seconds(self): """PLUGIN_TIMEOUT constant is 5 seconds.""" assert PLUGIN_TIMEOUT == 5 ================================================ FILE: tests/test_plugins_api.py ================================================ """Tests for plugin management REST API and example plugins.""" import json import subprocess import sys import pytest # ── API Tests ────────────────────────────────────────────────────── class TestPluginsAPI: """Test plugin management REST API endpoints.""" @pytest.fixture(autouse=True) def seed_plugin(self, app): """Seed a test plugin into the database.""" with app.app_context(): from mqttui.plugins.models import PluginConfig from mqttui.extensions import sa pc = PluginConfig( name="test-plugin", entry_point="mqttui.plugins.examples.json_formatter", enabled=False, version="1.0.0", description="A test plugin", ) sa.session.add(pc) sa.session.commit() def test_list_plugins(self, auth_client): """GET /api/v1/plugins returns JSON list of plugins.""" resp = auth_client.get("/api/v1/plugins") assert resp.status_code == 200 data = resp.get_json() assert data["status"] == "success" plugins = data["data"]["plugins"] assert isinstance(plugins, list) assert any(p["name"] == "test-plugin" for p in plugins) def test_enable_plugin(self, auth_client, app): """POST /api/v1/plugins/test-plugin/enable enables the plugin.""" resp = auth_client.post("/api/v1/plugins/test-plugin/enable") assert resp.status_code == 200 data = resp.get_json() assert data["status"] == "success" # Verify plugin is now enabled with app.app_context(): from mqttui.plugins.models import PluginConfig pc = PluginConfig.query.filter_by(name="test-plugin").first() assert pc.enabled is True def test_disable_plugin(self, auth_client, app): """POST /api/v1/plugins/test-plugin/disable disables the plugin.""" # Enable first auth_client.post("/api/v1/plugins/test-plugin/enable") # Then disable resp = auth_client.post("/api/v1/plugins/test-plugin/disable") assert resp.status_code == 200 data = resp.get_json() assert data["status"] == "success" with app.app_context(): from mqttui.plugins.models import PluginConfig pc = PluginConfig.query.filter_by(name="test-plugin").first() assert pc.enabled is False def test_enable_nonexistent_returns_404(self, auth_client): """POST /api/v1/plugins/nonexistent/enable returns 404.""" resp = auth_client.post("/api/v1/plugins/nonexistent/enable") assert resp.status_code == 404 data = resp.get_json() assert data["status"] == "error" def test_disable_nonexistent_returns_404(self, auth_client): """POST /api/v1/plugins/nonexistent/disable returns 404.""" resp = auth_client.post("/api/v1/plugins/nonexistent/disable") assert resp.status_code == 404 data = resp.get_json() assert data["status"] == "error" def test_partials_plugins_returns_html(self, auth_client): """GET /partials/plugins returns HTML partial.""" resp = auth_client.get("/partials/plugins") assert resp.status_code == 200 assert b"test-plugin" in resp.data # ── Example Plugin Tests ─────────────────────────────────────────── class TestJsonFormatterPlugin: """Test the json_formatter example plugin via subprocess.""" def test_formats_json_payload(self): """JSON payload is pretty-printed.""" input_data = { "event": "on_message", "data": {"topic": "test/topic", "payload": '{"a": 1, "b": 2}'}, } result = subprocess.run( [sys.executable, "-m", "mqttui.plugins.examples.json_formatter"], input=json.dumps(input_data) + "\n", capture_output=True, text=True, timeout=5, ) output = json.loads(result.stdout) assert "actions" in output assert len(output["actions"]) == 1 assert output["actions"][0]["type"] == "transform" # Verify it's pretty-printed (has newlines) assert "\n" in output["actions"][0]["result"] def test_non_json_payload_returns_empty_actions(self): """Non-JSON payload returns empty actions list.""" input_data = { "event": "on_message", "data": {"topic": "test/topic", "payload": "not json"}, } result = subprocess.run( [sys.executable, "-m", "mqttui.plugins.examples.json_formatter"], input=json.dumps(input_data) + "\n", capture_output=True, text=True, timeout=5, ) output = json.loads(result.stdout) assert output["actions"] == [] def test_non_message_event_returns_empty_actions(self): """Non on_message events return empty actions.""" input_data = { "event": "on_rule_trigger", "data": {"rule_name": "test"}, } result = subprocess.run( [sys.executable, "-m", "mqttui.plugins.examples.json_formatter"], input=json.dumps(input_data) + "\n", capture_output=True, text=True, timeout=5, ) output = json.loads(result.stdout) assert output["actions"] == [] class TestTopicLoggerPlugin: """Test the topic_logger example plugin via subprocess.""" def test_logs_message(self): """on_message event produces a log action.""" input_data = { "event": "on_message", "data": {"topic": "home/sensor/temp", "payload": "22.5"}, } result = subprocess.run( [sys.executable, "-m", "mqttui.plugins.examples.topic_logger"], input=json.dumps(input_data) + "\n", capture_output=True, text=True, timeout=5, ) output = json.loads(result.stdout) assert len(output["actions"]) == 1 assert output["actions"][0]["type"] == "log" assert "home/sensor/temp" in output["actions"][0]["message"] assert "22.5" in output["actions"][0]["message"] def test_logs_to_stderr(self): """Plugin logs to stderr (not stdout which is protocol).""" input_data = { "event": "on_message", "data": {"topic": "home/sensor/temp", "payload": "22.5"}, } result = subprocess.run( [sys.executable, "-m", "mqttui.plugins.examples.topic_logger"], input=json.dumps(input_data) + "\n", capture_output=True, text=True, timeout=5, ) assert "home/sensor/temp" in result.stderr ================================================ FILE: tests/test_routes.py ================================================ def test_index_redirects_unauthenticated(client): """Unauthenticated access to / should redirect to /login.""" response = client.get('/') assert response.status_code == 302 assert '/login' in response.headers['Location'] def test_index_returns_200_authenticated(client, app): """Authenticated access to / should return 200.""" # Login first client.post('/login', data={'username': 'admin', 'password': 'admin'}) response = client.get('/') assert response.status_code == 200 def test_stats_returns_json(client): response = client.get('/stats') assert response.status_code == 200 data = response.get_json() assert 'connection_count' in data assert 'topic_count' in data assert 'message_count' in data assert 'errors' in data def test_version_returns_json(client): response = client.get('/version') assert response.status_code == 200 data = response.get_json() assert 'version' in data def test_api_messages_returns_json(client): response = client.get('/api/messages') assert response.status_code == 200 data = response.get_json() assert 'messages' in data def test_api_topics_returns_json(client): response = client.get('/api/topics') assert response.status_code == 200 data = response.get_json() assert 'topics' in data def test_database_stats(client): response = client.get('/api/database/stats') assert response.status_code == 200 data = response.get_json() assert 'enabled' in data ================================================ FILE: tests/test_rules_api.py ================================================ """Integration tests for rules REST API endpoints.""" import json # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _create_rule(auth_client, **overrides): """POST a new rule with sensible defaults. Returns response object.""" data = { "name": "Test Rule", "trigger_topic": "sensors/+/temp", "condition": {"path": "temp", "op": "gt", "value": 30}, "action": {"type": "publish", "topic": "alerts/temp", "payload": '{"alert": true}'}, } data.update(overrides) return auth_client.post('/api/v1/rules/', json=data) # --------------------------------------------------------------------------- # List # --------------------------------------------------------------------------- def test_list_rules_empty(auth_client): """GET /api/v1/rules/ returns empty list when no rules exist.""" r = auth_client.get('/api/v1/rules/') assert r.status_code == 200 body = r.get_json() assert body['status'] == 'success' assert body['data']['rules'] == [] def test_list_rules_with_data(auth_client): """GET /api/v1/rules/ returns created rules.""" _create_rule(auth_client, name="Rule A") _create_rule(auth_client, name="Rule B") r = auth_client.get('/api/v1/rules/') assert r.status_code == 200 rules = r.get_json()['data']['rules'] assert len(rules) == 2 # --------------------------------------------------------------------------- # Create # --------------------------------------------------------------------------- def test_create_rule(auth_client): """POST /api/v1/rules/ with valid data returns 201 with rule data.""" r = _create_rule(auth_client) assert r.status_code == 201 body = r.get_json() assert body['status'] == 'success' rule = body['data'] assert rule['name'] == 'Test Rule' assert rule['trigger_topic'] == 'sensors/+/temp' assert rule['enabled'] is True assert rule['id'] is not None assert rule['condition'] == {"path": "temp", "op": "gt", "value": 30} assert rule['action']['type'] == 'publish' def test_create_rule_missing_name(auth_client): """POST without name returns 400 VALIDATION_ERROR.""" r = auth_client.post('/api/v1/rules/', json={ "trigger_topic": "test/+", "action": {"type": "publish", "topic": "out", "payload": "x"}, }) assert r.status_code == 400 body = r.get_json() assert body['status'] == 'error' assert body['error']['code'] == 'VALIDATION_ERROR' def test_create_rule_missing_action(auth_client): """POST without action returns 400 VALIDATION_ERROR.""" r = auth_client.post('/api/v1/rules/', json={ "name": "No Action", "trigger_topic": "test/+", }) assert r.status_code == 400 body = r.get_json() assert body['status'] == 'error' assert body['error']['code'] == 'VALIDATION_ERROR' def test_create_rule_missing_trigger_topic(auth_client): """POST without trigger_topic returns 400 VALIDATION_ERROR.""" r = auth_client.post('/api/v1/rules/', json={ "name": "No Topic", "action": {"type": "publish", "topic": "out", "payload": "x"}, }) assert r.status_code == 400 body = r.get_json() assert body['error']['code'] == 'VALIDATION_ERROR' # --------------------------------------------------------------------------- # Get single # --------------------------------------------------------------------------- def test_get_rule(auth_client): """GET /api/v1/rules/ returns rule.""" cr = _create_rule(auth_client) rule_id = cr.get_json()['data']['id'] r = auth_client.get(f'/api/v1/rules/{rule_id}') assert r.status_code == 200 assert r.get_json()['data']['id'] == rule_id def test_get_rule_not_found(auth_client): """GET /api/v1/rules/999 returns 404 NOT_FOUND.""" r = auth_client.get('/api/v1/rules/999') assert r.status_code == 404 body = r.get_json() assert body['error']['code'] == 'NOT_FOUND' # --------------------------------------------------------------------------- # Update # --------------------------------------------------------------------------- def test_update_rule(auth_client): """PUT /api/v1/rules/ updates specified fields only.""" cr = _create_rule(auth_client) rule_id = cr.get_json()['data']['id'] r = auth_client.put(f'/api/v1/rules/{rule_id}', json={ "name": "Updated Rule", "description": "Now with description", }) assert r.status_code == 200 rule = r.get_json()['data'] assert rule['name'] == 'Updated Rule' assert rule['description'] == 'Now with description' # Unchanged fields preserved assert rule['trigger_topic'] == 'sensors/+/temp' def test_update_rule_not_found(auth_client): """PUT /api/v1/rules/999 returns 404.""" r = auth_client.put('/api/v1/rules/999', json={"name": "x"}) assert r.status_code == 404 # --------------------------------------------------------------------------- # Delete # --------------------------------------------------------------------------- def test_delete_rule(auth_client): """DELETE /api/v1/rules/ returns 200, subsequent GET returns 404.""" cr = _create_rule(auth_client) rule_id = cr.get_json()['data']['id'] r = auth_client.delete(f'/api/v1/rules/{rule_id}') assert r.status_code == 200 assert r.get_json()['status'] == 'success' # Verify deleted r2 = auth_client.get(f'/api/v1/rules/{rule_id}') assert r2.status_code == 404 def test_delete_rule_not_found(auth_client): """DELETE /api/v1/rules/999 returns 404.""" r = auth_client.delete('/api/v1/rules/999') assert r.status_code == 404 # --------------------------------------------------------------------------- # Enable / Disable # --------------------------------------------------------------------------- def test_enable_disable(auth_client): """POST enable sets enabled=True, POST disable sets enabled=False.""" cr = _create_rule(auth_client, enabled=True) rule_id = cr.get_json()['data']['id'] # Disable r = auth_client.post(f'/api/v1/rules/{rule_id}/disable') assert r.status_code == 200 assert r.get_json()['data']['enabled'] is False # Enable r = auth_client.post(f'/api/v1/rules/{rule_id}/enable') assert r.status_code == 200 assert r.get_json()['data']['enabled'] is True def test_enable_not_found(auth_client): """POST /api/v1/rules/999/enable returns 404.""" r = auth_client.post('/api/v1/rules/999/enable') assert r.status_code == 404 # --------------------------------------------------------------------------- # Dry-run / Test # --------------------------------------------------------------------------- def test_dry_run_match(auth_client): """POST /api/v1/rules//test with matching topic+payload returns match=True.""" cr = _create_rule(auth_client) rule_id = cr.get_json()['data']['id'] r = auth_client.post(f'/api/v1/rules/{rule_id}/test', json={ "topic": "sensors/living-room/temp", "payload": '{"temp": 35}', }) assert r.status_code == 200 body = r.get_json()['data'] assert body['match'] is True assert body['topic_match'] is True assert body['condition_match'] is True assert len(body['actions']) == 1 assert body['actions'][0]['type'] == 'publish' def test_dry_run_no_match_condition(auth_client): """POST /test with non-matching payload returns match=False.""" cr = _create_rule(auth_client) rule_id = cr.get_json()['data']['id'] r = auth_client.post(f'/api/v1/rules/{rule_id}/test', json={ "topic": "sensors/living-room/temp", "payload": '{"temp": 20}', }) assert r.status_code == 200 body = r.get_json()['data'] assert body['match'] is False assert body['topic_match'] is True assert body['condition_match'] is False assert body['actions'] == [] def test_dry_run_no_match_topic(auth_client): """POST /test with non-matching topic returns match=False.""" cr = _create_rule(auth_client) rule_id = cr.get_json()['data']['id'] r = auth_client.post(f'/api/v1/rules/{rule_id}/test', json={ "topic": "other/topic", "payload": '{"temp": 35}', }) assert r.status_code == 200 body = r.get_json()['data'] assert body['match'] is False assert body['topic_match'] is False def test_dry_run_dict_payload(auth_client): """POST /test with dict payload (not string) also works.""" cr = _create_rule(auth_client) rule_id = cr.get_json()['data']['id'] r = auth_client.post(f'/api/v1/rules/{rule_id}/test', json={ "topic": "sensors/bedroom/temp", "payload": {"temp": 40}, }) assert r.status_code == 200 assert r.get_json()['data']['match'] is True def test_dry_run_missing_topic(auth_client): """POST /test without topic returns 400.""" cr = _create_rule(auth_client) rule_id = cr.get_json()['data']['id'] r = auth_client.post(f'/api/v1/rules/{rule_id}/test', json={ "payload": '{"temp": 35}', }) assert r.status_code == 400 # --------------------------------------------------------------------------- # Authentication # --------------------------------------------------------------------------- def test_unauthenticated_access(client): """GET /api/v1/rules/ without auth returns 401 or redirect to login.""" r = client.get('/api/v1/rules/') # Flask-Login redirects to /login (302) or returns 401 assert r.status_code in (302, 401) ================================================ FILE: tests/test_rules_engine.py ================================================ """Tests for the RuleEngine class -- cache, matcher, rate limiter, loop prevention.""" import json import time from collections import deque from datetime import datetime from unittest.mock import patch, MagicMock, call import pytest from mqttui.events import mqtt_message_received, rule_fired, rule_changed # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def _create_rule(app, **overrides): """Create a Rule record inside app context and return its id.""" with app.app_context(): from mqttui.rules.models import Rule from mqttui.extensions import sa defaults = { 'name': 'Test Rule', 'trigger_topic': 'sensors/+/temp', 'condition_json': '{}', 'action_json': json.dumps({'type': 'log'}), 'enabled': True, 'rate_limit_per_min': 10, } defaults.update(overrides) rule = Rule(**defaults) sa.session.add(rule) sa.session.commit() return rule.id def _send_message(topic, payload_dict): """Fire mqtt_message_received signal with given topic and JSON payload.""" mqtt_message_received.send( 'mqtt_client', topic=topic, payload=json.dumps(payload_dict), timestamp=datetime.utcnow(), qos=0, retain=False, ) @pytest.fixture def engine(app): """Create a RuleEngine with mocked scheduler, disconnect after test.""" from mqttui.rules.engine import RuleEngine eng = RuleEngine(app) # Prevent real GeventScheduler from starting in tests eng._scheduler = MagicMock() eng._scheduler.get_jobs.return_value = [] yield eng eng.disconnect() @pytest.fixture def fire_log(): """Collect rule_fired signals. Auto-disconnects after test.""" fired = [] def _handler(sender, **kw): fired.append(kw) rule_fired.connect(_handler) yield fired rule_fired.disconnect(_handler) # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- class TestLoopPrevention: """Messages with __source: mqttui-automation must be skipped.""" def test_loop_prevention_skips_automation_messages(self, app, engine, fire_log): _create_rule(app, trigger_topic='sensors/#') engine.connect() with app.app_context(): _send_message('sensors/outdoor/temp', { '__source': 'mqttui-automation', 'temp': 35, }) assert len(fire_log) == 0, "Rule must not fire on automation messages" class TestTopicMatching: """Rules should match based on MQTT wildcard patterns.""" def test_wildcard_topic_matching(self, app, engine, fire_log): _create_rule(app, trigger_topic='sensors/+/temp', action_json=json.dumps({'type': 'log'})) engine.connect() with app.app_context(): _send_message('sensors/outdoor/temp', {'temp': 25}) assert len(fire_log) == 1 assert fire_log[0]['topic'] == 'sensors/outdoor/temp' def test_non_matching_topic(self, app, engine, fire_log): _create_rule(app, trigger_topic='sensors/+/temp') engine.connect() with app.app_context(): _send_message('actuators/fan/speed', {'speed': 50}) assert len(fire_log) == 0 class TestConditionEvaluation: """Rules should evaluate conditions before firing.""" def test_matching_condition_fires(self, app, engine, fire_log): _create_rule( app, trigger_topic='sensors/+/temp', condition_json=json.dumps({'path': 'temp', 'op': 'gt', 'value': 30}), action_json=json.dumps({'type': 'log'}), ) engine.connect() with app.app_context(): _send_message('sensors/outdoor/temp', {'temp': 35}) assert len(fire_log) == 1 def test_non_matching_condition_does_not_fire(self, app, engine, fire_log): _create_rule( app, trigger_topic='sensors/+/temp', condition_json=json.dumps({'path': 'temp', 'op': 'gt', 'value': 30}), ) engine.connect() with app.app_context(): _send_message('sensors/outdoor/temp', {'temp': 20}) assert len(fire_log) == 0 class TestDisabledRule: """Disabled rules must not be in cache and never fire.""" def test_disabled_rule_not_in_cache(self, app, engine, fire_log): _create_rule(app, trigger_topic='sensors/#', enabled=False) engine.connect() with app.app_context(): _send_message('sensors/outdoor/temp', {'temp': 35}) assert len(fire_log) == 0 class TestPerRuleRateLimit: """Per-rule rate limit blocks excessive firings within the window.""" def test_rate_limit_blocks_excess(self, app, engine, fire_log): _create_rule( app, trigger_topic='sensors/#', rate_limit_per_min=2, action_json=json.dumps({'type': 'log'}), ) engine.connect() with app.app_context(): for _ in range(3): _send_message('sensors/outdoor/temp', {'temp': 35}) assert len(fire_log) == 2, "Third firing should be rate-limited" class TestGlobalCircuitBreaker: """Global circuit breaker blocks all firings after threshold.""" def test_global_limit_blocks_excess(self, app, engine, fire_log): # Create rules that will all fire for i in range(5): _create_rule( app, name=f'Rule {i}', trigger_topic='sensors/#', rate_limit_per_min=100, action_json=json.dumps({'type': 'log'}), ) engine._GLOBAL_LIMIT = 3 # Low limit for testing engine.connect() with app.app_context(): _send_message('sensors/outdoor/temp', {'temp': 35}) # 5 rules match but global limit is 3 assert len(fire_log) == 3, "Global circuit breaker should limit to 3 firings" class TestPublishActionSourceMarker: """Publish action must inject __source marker.""" def test_publish_injects_source_marker(self, app): from mqttui.rules.actions import execute_action with app.app_context(): with patch('mqttui.mqtt_client.publish') as mock_pub: result = execute_action( {'type': 'publish', 'topic': 'output/fan', 'payload': '{"speed": 100}'}, {'rule_id': 1, 'rule_name': 'Test', 'topic': 'sensors/temp', 'payload': '{}'}, ) assert result['success'] is True call_args = mock_pub.call_args published_payload = json.loads(call_args[0][1]) assert published_payload['__source'] == 'mqttui-automation' assert published_payload['speed'] == 100 class TestLogActionCreatesAlert: """Log action must create an AlertHistory record.""" def test_log_creates_alert_history(self, app): from mqttui.rules.actions import execute_action from mqttui.rules.models import AlertHistory with app.app_context(): result = execute_action( {'type': 'log', 'severity': 'warning', 'message': 'Temperature high'}, {'rule_id': 1, 'rule_name': 'Temp Alert', 'topic': 'sensors/temp', 'payload': '{}'}, ) assert result['success'] is True alerts = AlertHistory.query.all() assert len(alerts) == 1 assert alerts[0].severity == 'warning' assert alerts[0].rule_name == 'Temp Alert' class TestRuleFiredSignal: """rule_fired signal must be emitted with correct kwargs.""" def test_signal_emitted_on_fire(self, app, engine, fire_log): rule_id = _create_rule( app, name='Signal Test Rule', trigger_topic='sensors/#', action_json=json.dumps({'type': 'log'}), ) engine.connect() with app.app_context(): _send_message('sensors/outdoor/temp', {'temp': 25}) assert len(fire_log) == 1 sig = fire_log[0] assert sig['rule_id'] == rule_id assert sig['rule_name'] == 'Signal Test Rule' assert sig['topic'] == 'sensors/outdoor/temp' # --------------------------------------------------------------------------- # Scheduler & Hot-Reload Tests (Plan 03-04) # --------------------------------------------------------------------------- class TestHotReload: """rule_changed signal should trigger cache reload.""" def test_hot_reload_on_rule_changed(self, app, engine): """Cache updates when rule_changed signal fires.""" rule_id = _create_rule(app, trigger_topic='sensors/#') engine.connect() # Rule should be in cache assert rule_id in engine._rules # Delete the rule from DB with app.app_context(): from mqttui.rules.models import Rule from mqttui.extensions import sa rule = sa.session.get(Rule, rule_id) sa.session.delete(rule) sa.session.commit() # Fire rule_changed signal -- should trigger cache reload rule_changed.send('test', action='deleted', rule_id=rule_id) # Rule should no longer be in cache assert rule_id not in engine._rules class TestCronScheduleSync: """sync_scheduled_jobs should manage APScheduler jobs for cron rules.""" def test_cron_schedule_sync(self, app, engine): """A rule with schedule_cron gets a scheduler job with replace_existing.""" rule_id = _create_rule( app, trigger_topic='sensors/#', schedule_cron='*/5 * * * *', ) engine.connect() # Verify add_job was called with correct arguments engine._scheduler.add_job.assert_called() call_kwargs = engine._scheduler.add_job.call_args assert call_kwargs[1]['replace_existing'] is True assert call_kwargs[1]['id'] == f'rule_{rule_id}' def test_removed_rule_job_cleaned_up(self, app, engine): """Jobs for deleted rules are removed from scheduler.""" # Simulate a stale job from a previously deleted rule stale_job = MagicMock() stale_job.id = 'rule_999' engine._scheduler.get_jobs.return_value = [stale_job] # Create a rule without cron (no new job expected) _create_rule(app, trigger_topic='sensors/#') engine.connect() # Stale job should be removed engine._scheduler.remove_job.assert_any_call('rule_999') class TestFireScheduledRule: """fire_scheduled_rule should execute the action and log to AlertHistory.""" def test_fire_scheduled_rule(self, app, engine): """Calling fire_scheduled_rule creates an AlertHistory record.""" from mqttui.rules.models import AlertHistory rule_id = _create_rule( app, name='Heartbeat Rule', trigger_topic='__scheduled__', action_json=json.dumps({'type': 'log', 'message': 'heartbeat'}), ) engine.connect() with app.app_context(): engine.fire_scheduled_rule(rule_id) alerts = AlertHistory.query.all() assert len(alerts) == 1 assert alerts[0].message == 'heartbeat' assert alerts[0].rule_name == 'Heartbeat Rule' def test_fire_scheduled_rule_missing_id(self, app, engine): """fire_scheduled_rule silently returns for non-existent rule.""" engine.connect() with app.app_context(): # Should not raise engine.fire_scheduled_rule(99999) class TestAppCreatesRuleEngine: """create_app should register the rules blueprint.""" def test_rules_blueprint_registered(self, app): """The rules blueprint should be in app.blueprints.""" assert 'rules' in app.blueprints ================================================ FILE: tests/test_rules_evaluator.py ================================================ """Tests for the condition evaluator pure function.""" import pytest from mqttui.rules.evaluator import evaluate, ConditionError def test_simple_gt_true(): """Greater-than returns True when actual > value.""" assert evaluate({"path": "temperature", "op": "gt", "value": 30}, {"temperature": 35}) is True def test_simple_gt_false(): """Greater-than returns False when actual < value.""" assert evaluate({"path": "temperature", "op": "gt", "value": 30}, {"temperature": 25}) is False def test_nested_path_eq(): """Dot-notation path resolves nested dicts.""" condition = {"path": "sensors.outdoor.temp", "op": "eq", "value": 22} payload = {"sensors": {"outdoor": {"temp": 22}}} assert evaluate(condition, payload) is True def test_compound_all_true(): """Compound 'all' returns True when all sub-conditions match.""" condition = {"all": [ {"path": "temp", "op": "gt", "value": 20}, {"path": "temp", "op": "lt", "value": 40}, ]} assert evaluate(condition, {"temp": 30}) is True def test_compound_all_false(): """Compound 'all' returns False when any sub-condition fails.""" condition = {"all": [ {"path": "temp", "op": "gt", "value": 20}, {"path": "temp", "op": "lt", "value": 25}, ]} assert evaluate(condition, {"temp": 30}) is False def test_compound_any_true(): """Compound 'any' returns True when at least one sub-condition matches.""" condition = {"any": [ {"path": "status", "op": "eq", "value": "on"}, {"path": "status", "op": "eq", "value": "active"}, ]} assert evaluate(condition, {"status": "active"}) is True def test_compound_any_false(): """Compound 'any' returns False when no sub-condition matches.""" condition = {"any": [ {"path": "status", "op": "eq", "value": "on"}, {"path": "status", "op": "eq", "value": "active"}, ]} assert evaluate(condition, {"status": "off"}) is False def test_contains(): """Contains checks substring in string value.""" assert evaluate({"path": "name", "op": "contains", "value": "sensor"}, {"name": "temp_sensor_1"}) is True def test_not_contains(): """Not-contains checks absence of substring.""" assert evaluate({"path": "name", "op": "not_contains", "value": "xyz"}, {"name": "abc"}) is True def test_regex_match(): """Regex operator matches patterns.""" assert evaluate({"path": "topic", "op": "regex", "value": r"^sensor_\d+"}, {"topic": "sensor_42"}) is True def test_regex_no_match(): """Regex returns False when pattern doesn't match.""" assert evaluate({"path": "topic", "op": "regex", "value": r"^sensor_\d+"}, {"topic": "device_42"}) is False def test_exists_true(): """Exists returns True when path is present (even if value is falsy).""" assert evaluate({"path": "temp", "op": "exists"}, {"temp": 0}) is True def test_exists_false(): """Exists returns False when path is missing.""" assert evaluate({"path": "missing", "op": "exists"}, {"temp": 0}) is False def test_not_exists_true(): """Not-exists returns True when path is missing.""" assert evaluate({"path": "missing", "op": "not_exists"}, {"temp": 0}) is True def test_not_exists_false(): """Not-exists returns False when path is present.""" assert evaluate({"path": "temp", "op": "not_exists"}, {"temp": 0}) is False def test_non_json_payload_returns_false(): """Non-dict payload returns False for path-based conditions.""" assert evaluate({"path": "temp", "op": "gt", "value": 30}, None) is False def test_numeric_coercion(): """String numbers are coerced for numeric comparisons.""" assert evaluate({"path": "temp", "op": "gt", "value": 30}, {"temp": "35"}) is True def test_numeric_coercion_lt(): """String numbers coerced for less-than.""" assert evaluate({"path": "temp", "op": "lt", "value": 30}, {"temp": "25"}) is True def test_empty_condition_always_true(): """Empty condition dict means always match.""" assert evaluate({}, {"temp": 30}) is True def test_none_condition_always_true(): """None condition means always match.""" assert evaluate(None, {"temp": 30}) is True def test_unknown_operator_raises(): """Unknown operator raises ConditionError.""" with pytest.raises(ConditionError, match="Unknown operator"): evaluate({"path": "x", "op": "unknown_op", "value": 1}, {"x": 1}) def test_eq_operator(): """Equality check.""" assert evaluate({"path": "x", "op": "eq", "value": 5}, {"x": 5}) is True assert evaluate({"path": "x", "op": "eq", "value": 5}, {"x": 6}) is False def test_ne_operator(): """Not-equal check.""" assert evaluate({"path": "x", "op": "ne", "value": 5}, {"x": 6}) is True assert evaluate({"path": "x", "op": "ne", "value": 5}, {"x": 5}) is False def test_gte_operator(): """Greater-than-or-equal check.""" assert evaluate({"path": "x", "op": "gte", "value": 5}, {"x": 5}) is True assert evaluate({"path": "x", "op": "gte", "value": 5}, {"x": 4}) is False def test_lte_operator(): """Less-than-or-equal check.""" assert evaluate({"path": "x", "op": "lte", "value": 5}, {"x": 5}) is True assert evaluate({"path": "x", "op": "lte", "value": 5}, {"x": 6}) is False def test_missing_path_returns_false(): """Missing path in payload returns False (not an error).""" assert evaluate({"path": "missing.deep.path", "op": "eq", "value": 1}, {"x": 1}) is False ================================================ FILE: tests/test_rules_models.py ================================================ """Tests for Rule and AlertHistory models.""" import json from datetime import datetime def test_rule_model_columns(app): """Rule model has all required columns.""" with app.app_context(): from mqttui.rules.models import Rule cols = [c.name for c in Rule.__table__.columns] expected = [ 'id', 'name', 'description', 'trigger_topic', 'condition_json', 'action_json', 'enabled', 'rate_limit_per_min', 'schedule_cron', 'last_fired', 'fire_count', 'created_at', 'updated_at', ] for col in expected: assert col in cols, f"Missing column: {col}" def test_rule_tablename(app): """Rule model uses 'rules' table.""" with app.app_context(): from mqttui.rules.models import Rule assert Rule.__tablename__ == 'rules' def test_rule_to_dict(app): """Rule.to_dict() returns dict with parsed JSON fields.""" with app.app_context(): from mqttui.rules.models import Rule from mqttui.extensions import sa rule = Rule( name='Test Rule', description='A test rule', trigger_topic='sensors/temp', condition_json=json.dumps({"path": "temp", "op": "gt", "value": 30}), action_json=json.dumps({"type": "log", "message": "Hot!"}), enabled=True, rate_limit_per_min=5, ) sa.session.add(rule) sa.session.commit() d = rule.to_dict() assert d['name'] == 'Test Rule' assert d['trigger_topic'] == 'sensors/temp' assert d['condition'] == {"path": "temp", "op": "gt", "value": 30} assert d['action'] == {"type": "log", "message": "Hot!"} assert d['enabled'] is True assert d['rate_limit_per_min'] == 5 assert d['fire_count'] == 0 assert d['id'] is not None def test_alert_history_model_columns(app): """AlertHistory model has all required columns.""" with app.app_context(): from mqttui.rules.models import AlertHistory cols = [c.name for c in AlertHistory.__table__.columns] expected = ['id', 'rule_id', 'rule_name', 'topic', 'severity', 'message', 'fired_at'] for col in expected: assert col in cols, f"Missing column: {col}" def test_alert_history_tablename(app): """AlertHistory model uses 'alert_history' table.""" with app.app_context(): from mqttui.rules.models import AlertHistory assert AlertHistory.__tablename__ == 'alert_history' def test_alert_history_to_dict(app): """AlertHistory.to_dict() returns dict with ISO datetime.""" with app.app_context(): from mqttui.rules.models import AlertHistory from mqttui.extensions import sa ah = AlertHistory( rule_id=1, rule_name='Test Rule', topic='sensors/temp', severity='warning', message='Temperature too high', ) sa.session.add(ah) sa.session.commit() d = ah.to_dict() assert d['rule_name'] == 'Test Rule' assert d['severity'] == 'warning' assert d['topic'] == 'sensors/temp' assert d['fired_at'] is not None ================================================ FILE: tests/test_ux_features.py ================================================ """Tests for UX features: topic favorites/bookmarks and retained message indicator.""" import json class TestTopicFavorites: """Tests for topic bookmark/favorite functionality.""" def test_bookmark_creates_favorite(self, auth_client, app): """POST /api/v1/topics//bookmark creates a favorite (201).""" r = auth_client.post('/api/v1/topics/home%2Fsensor/bookmark') assert r.status_code == 201 data = json.loads(r.data) assert data['status'] == 'success' assert data['data']['bookmarked'] is True def test_bookmark_toggle_removes_favorite(self, auth_client, app): """POST /api/v1/topics//bookmark again removes the favorite (200).""" # First call creates auth_client.post('/api/v1/topics/home%2Fsensor/bookmark') # Second call removes (toggle) r = auth_client.post('/api/v1/topics/home%2Fsensor/bookmark') assert r.status_code == 200 data = json.loads(r.data) assert data['status'] == 'success' assert data['data']['bookmarked'] is False def test_get_favorites_returns_bookmarked_topics(self, auth_client, app): """GET /api/v1/topics/favorites returns list of bookmarked topics.""" # Bookmark two topics auth_client.post('/api/v1/topics/home%2Fsensor/bookmark') auth_client.post('/api/v1/topics/home%2Flight/bookmark') r = auth_client.get('/api/v1/topics/favorites') assert r.status_code == 200 data = json.loads(r.data) assert data['status'] == 'success' favorites = data['data']['favorites'] assert len(favorites) == 2 topics = [f['topic'] for f in favorites] assert 'home/sensor' in topics assert 'home/light' in topics def test_topics_includes_is_favorite(self, auth_client, app): """GET /api/v1/topics includes is_favorite=true for bookmarked topics.""" # Bookmark a topic auth_client.post('/api/v1/topics/home%2Fsensor/bookmark') r = auth_client.get('/api/v1/topics') assert r.status_code == 200 data = json.loads(r.data) assert data['status'] == 'success' # The topics list should have is_favorite field topics = data['data']['topics'] for t in topics: assert 'is_favorite' in t def test_unauthenticated_bookmark_rejected(self, client): """Unauthenticated bookmark request returns 401 or redirect.""" r = client.post('/api/v1/topics/home%2Fsensor/bookmark') # Flask-Login redirects to login page (302) by default assert r.status_code in (302, 401) ================================================ FILE: tests/test_webhook.py ================================================ """Tests for webhook delivery system: SSRF validation, webhook execution, retry logic.""" import json import pytest import socket from unittest.mock import patch, MagicMock, call from datetime import datetime # --------------------------------------------------------------------------- # SSRF Validator Tests # --------------------------------------------------------------------------- class TestSSRFValidator: """Test SSRF URL validation blocks private/reserved IPs.""" def test_ssrf_blocks_private_10(self): from mqttui.rules.ssrf import is_ssrf_safe safe, reason = is_ssrf_safe("http://10.0.0.1/hook") assert safe is False assert "private" in reason.lower() or "blocked" in reason.lower() def test_ssrf_blocks_private_172(self): from mqttui.rules.ssrf import is_ssrf_safe safe, reason = is_ssrf_safe("http://172.16.0.1/hook") assert safe is False def test_ssrf_blocks_private_192(self): from mqttui.rules.ssrf import is_ssrf_safe safe, reason = is_ssrf_safe("http://192.168.1.1/hook") assert safe is False def test_ssrf_blocks_localhost(self): from mqttui.rules.ssrf import is_ssrf_safe safe, reason = is_ssrf_safe("http://127.0.0.1/hook") assert safe is False def test_ssrf_blocks_link_local(self): from mqttui.rules.ssrf import is_ssrf_safe safe, reason = is_ssrf_safe("http://169.254.1.1/hook") assert safe is False @patch('socket.getaddrinfo', return_value=[ (socket.AF_INET, socket.SOCK_STREAM, 0, '', ('93.184.216.34', 443)) ]) def test_ssrf_allows_public(self, mock_dns): from mqttui.rules.ssrf import is_ssrf_safe safe, reason = is_ssrf_safe("https://hooks.example.com/hook") assert safe is True def test_ssrf_blocks_ipv6_localhost(self): from mqttui.rules.ssrf import is_ssrf_safe safe, reason = is_ssrf_safe("http://[::1]/hook") assert safe is False # --------------------------------------------------------------------------- # Webhook Delivery Tests # --------------------------------------------------------------------------- class TestWebhookDelivery: """Test webhook HTTP delivery, retry logic, and AlertHistory integration.""" @patch('httpx.post') def test_webhook_success(self, mock_post, app): """Successful webhook creates AlertHistory record with http_status=200.""" mock_response = MagicMock() mock_response.status_code = 200 mock_response.text = "OK" mock_post.return_value = mock_response from mqttui.rules.actions import _deliver_webhook with app.app_context(): from mqttui.rules.models import AlertHistory from mqttui.extensions import sa result = _deliver_webhook( url="https://hooks.example.com/hook", payload_json={"topic": "test/topic", "payload": "hello"}, rule_id=1, rule_name="Test Rule", topic="test/topic", ) assert result["success"] is True record = AlertHistory.query.filter_by(rule_id=1).order_by(AlertHistory.id.desc()).first() assert record is not None assert record.http_status == 200 assert record.webhook_url == "https://hooks.example.com/hook" @patch('httpx.post') def test_webhook_retry_on_5xx(self, mock_post, app): """5xx responses trigger up to 3 retries.""" mock_response = MagicMock() mock_response.status_code = 500 mock_response.text = "Internal Server Error" mock_post.return_value = mock_response from mqttui.rules.actions import _deliver_webhook with app.app_context(): from mqttui.rules.models import AlertHistory result = _deliver_webhook( url="https://hooks.example.com/hook", payload_json={"topic": "test/topic"}, rule_id=2, rule_name="Retry Rule", topic="test/topic", _sleep_fn=lambda x: None, # Skip actual sleep in tests ) assert result["success"] is False # 1 initial + 3 retries = 4 total calls assert mock_post.call_count == 4 record = AlertHistory.query.filter_by(rule_id=2).order_by(AlertHistory.id.desc()).first() assert record is not None assert record.retry_count == 3 assert record.http_status == 500 @patch('httpx.post') def test_webhook_no_retry_on_4xx(self, mock_post, app): """4xx responses fail immediately without retry.""" mock_response = MagicMock() mock_response.status_code = 400 mock_response.text = "Bad Request" mock_post.return_value = mock_response from mqttui.rules.actions import _deliver_webhook with app.app_context(): from mqttui.rules.models import AlertHistory result = _deliver_webhook( url="https://hooks.example.com/hook", payload_json={"topic": "test/topic"}, rule_id=3, rule_name="NoRetry Rule", topic="test/topic", ) assert result["success"] is False assert mock_post.call_count == 1 record = AlertHistory.query.filter_by(rule_id=3).order_by(AlertHistory.id.desc()).first() assert record is not None assert record.retry_count == 0 assert record.http_status == 400 def test_webhook_payload_template(self, app): """Webhook with template substitutes {{topic}}, {{rule_name}} context values.""" from mqttui.rules.actions import _build_webhook_payload context = { 'rule_id': 1, 'rule_name': 'Temp Alert', 'topic': 'sensors/temp', 'payload': '{"temp": 42}', } template = '{"alert": "{{rule_name}}", "on": "{{topic}}"}' result = _build_webhook_payload(template, context) assert result["alert"] == "Temp Alert" assert result["on"] == "sensors/temp" @patch('httpx.post') def test_webhook_runs_in_thread(self, mock_post, app): """Webhook delivery via execute_action returns immediately (async via thread pool).""" mock_response = MagicMock() mock_response.status_code = 200 mock_response.text = "OK" mock_post.return_value = mock_response from mqttui.rules.actions import execute_action with app.app_context(): action = {"type": "webhook", "url": "https://hooks.example.com/hook"} context = { "rule_id": 1, "rule_name": "Thread Rule", "topic": "test/topic", "payload": "hello", } result = execute_action(action, context) # Should return immediately with submission confirmation assert result["success"] is True assert "submitted" in result["detail"].lower() or "delivery" in result["detail"].lower() # --------------------------------------------------------------------------- # SSRF Validation in Rule Endpoints # --------------------------------------------------------------------------- class TestSSRFEndpoints: """Test SSRF validation in rule create/update API endpoints.""" def test_create_rule_ssrf_blocked(self, auth_client, app): """POST /api/v1/rules/ with private webhook URL returns 400 SSRF_BLOCKED.""" with app.app_context(): resp = auth_client.post('/api/v1/rules/', json={ 'name': 'SSRF Test Rule', 'trigger_topic': 'test/#', 'action': { 'type': 'webhook', 'url': 'http://192.168.1.1/hook', }, }) assert resp.status_code == 400 data = resp.get_json() assert data['error']['code'] == 'SSRF_BLOCKED' @patch('socket.getaddrinfo', return_value=[ (socket.AF_INET, socket.SOCK_STREAM, 0, '', ('93.184.216.34', 443)) ]) def test_create_rule_ssrf_allowed(self, mock_dns, auth_client, app): """POST /api/v1/rules/ with public webhook URL succeeds.""" with app.app_context(): resp = auth_client.post('/api/v1/rules/', json={ 'name': 'Public Webhook Rule', 'trigger_topic': 'test/#', 'action': { 'type': 'webhook', 'url': 'https://hooks.example.com/hook', }, }) assert resp.status_code == 201 def test_update_rule_ssrf_blocked(self, auth_client, app): """PUT /api/v1/rules/ with private webhook URL returns 400.""" with app.app_context(): from mqttui.rules.models import Rule from mqttui.extensions import sa import json # Create a rule first rule = Rule( name='Updatable Rule', trigger_topic='test/#', action_json=json.dumps({'type': 'log', 'severity': 'info'}), ) sa.session.add(rule) sa.session.commit() rule_id = rule.id # Try to update with private webhook URL resp = auth_client.put(f'/api/v1/rules/{rule_id}', json={ 'action': { 'type': 'webhook', 'url': 'http://10.0.0.1/hook', }, }) assert resp.status_code == 400 data = resp.get_json() assert data['error']['code'] == 'SSRF_BLOCKED' ================================================ FILE: wsgi.py ================================================ from dotenv import load_dotenv load_dotenv() from mqttui.app import create_app from mqttui.extensions import socketio app = create_app() if __name__ == '__main__': socketio.run(app, host=app.config['HOST'], port=app.config['PORT'], debug=app.config['DEBUG'])