[
{
"path": ".github/workflows/docker-publish.yml",
"content": "name: Docker Build & Publish\n\n# This tells GitHub: \"Run this every time I push code to the main branch\"\non:\n push:\n branches: [ \"main\" ]\n # Also run if I create a Release tag (e.g., v1.0)\n tags: [ 'v*.*.*' ]\n pull_request:\n branches: [ \"main\" ]\n\nenv:\n # Use GitHub's built-in registry (ghcr.io)\n REGISTRY: ghcr.io\n # Use the repository name as the image name\n IMAGE_NAME: ${{ github.repository }}\n\njobs:\n build:\n runs-on: ubuntu-latest\n permissions:\n contents: read\n packages: write # Needed to push the image to GHCR\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v3\n\n # Set up Docker Buildx (The builder engine)\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v2\n\n # Login to GitHub Container Registry using the automatic GitHub Token\n - name: Log into registry ${{ env.REGISTRY }}\n if: github.event_name != 'pull_request'\n uses: docker/login-action@v2\n with:\n registry: ${{ env.REGISTRY }}\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n # Generate tags (e.g., :latest, :v1.0, :main)\n - name: Extract Docker metadata\n id: meta\n uses: docker/metadata-action@v4\n with:\n images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}\n tags: |\n type=raw,value=latest,enable={{is_default_branch}}\n type=ref,event=branch\n type=semver,pattern={{version}}\n\n # Build the image and push it to the registry\n - name: Build and push Docker image\n uses: docker/build-push-action@v4\n with:\n context: .\n push: ${{ github.event_name != 'pull_request' }}\n tags: ${{ steps.meta.outputs.tags }}\n labels: ${{ steps.meta.outputs.labels }}\n"
},
{
"path": ".gitignore",
"content": "data/\n__pycache__/\n.env\n"
},
{
"path": "Dockerfile",
"content": "FROM python:3.11-slim\n\n# Set environment variables\nENV PYTHONUNBUFFERED=1\nENV PYTHONDONTWRITEBYTECODE=1\n\nWORKDIR /app\n\n# Install system dependencies (for some Python packages)\nRUN apt-get update && apt-get install -y --no-install-recommends \\\n curl \\\n ffmpeg \\\n && rm -rf /var/lib/apt/lists/*\n\n# Copy requirements first for better caching\nCOPY requirements.txt .\nRUN pip install --no-cache-dir -r requirements.txt\n\n# Copy all source code\nCOPY main.py .\nCOPY config.py .\nCOPY analytics/ ./analytics/\nCOPY alerts/ ./alerts/\nCOPY dashboard/ ./dashboard/\nCOPY export/ ./export/\nCOPY scheduler/ ./scheduler/\nCOPY scraper/ ./scraper/\nCOPY search/ ./search/\nCOPY plugins/ ./plugins/\nCOPY api/ ./api/\nCOPY docs/ ./docs/\n\n# Create data directory with subdirectories\nRUN mkdir -p data/backups data/parquet\n\n# Expose ports\n# 8501 = Streamlit Dashboard\n# 8000 = REST API\nEXPOSE 8501 8000\n\n# Health check for API mode\nHEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \\\n CMD curl -f http://localhost:8000/health || exit 1\n\n# Default: show help\nENTRYPOINT [\"python\", \"main.py\"]\n"
},
{
"path": "LICENSE",
"content": "MIT License\n\nCopyright (c) 2024\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
},
{
"path": "README.md",
"content": "# ๐ค Universal Reddit Scraper Suite\n\n[](https://github.com/ksanjeev284/reddit-universal-scraper/actions/workflows/docker-publish.yml)\n\nA **full-featured** Reddit scraper with analytics dashboard, REST API, scheduled scraping, plugins, and more. **No API keys required!**\n\n![\"image\"](\"https://github.com/user-attachments/assets/180b89ce-db02-4cd2-922d-aa3d1b8eeda7\")
\n\n## โจ Features\n\n| Feature | Description |\n|---------|-------------|\n| ๐ **Full Scraping** | Posts, comments, images, videos, galleries |\n| ๐ **Web Dashboard** | Beautiful Streamlit UI with 7 tabs |\n| ๐ **REST API** | Connect Metabase, Grafana, DuckDB |\n| ๐ **Plugin System** | Extensible post-processing (sentiment, dedupe, keywords) |\n| ๐ **Job Tracking** | Full history with status, duration, errors |\n| ๐งช **Dry Run Mode** | Test scrape rules without saving data |\n| ๐ฆ **Parquet Export** | Analytics-ready format for DuckDB/warehouses |\n| ๐ **Sentiment Analysis** | Analyze post/comment sentiment |\n| ๐
**Scheduled Scraping** | Cron-style job scheduling |\n| ๐ง **Notifications** | Discord & Telegram alerts |\n| ๐๏ธ **SQLite Database** | Structured storage with auto-backup |\n\n---\n\n## ๐ Quick Start\n\n```bash\n# Install dependencies\npip install -r requirements.txt\n\n# Scrape a subreddit\npython main.py python --mode full --limit 100\n\n# Launch dashboard\npython main.py --dashboard\n# Opens at http://localhost:8501\n```\n\n### ๐ Requirements\n\n- **Python 3.8+**\n- **ffmpeg** (optional, for video with audio)\n\n```bash\n# Windows (via chocolatey)\nchoco install ffmpeg\n\n# macOS\nbrew install ffmpeg\n\n# Ubuntu/Debian\nsudo apt install ffmpeg\n```\n\n---\n\n## ๐ All Commands\n\n### ๐ Scraping\n\n```bash\n# Full scrape (posts + media + comments)\npython main.py delhi --mode full --limit 100\n\n# Fast history-only (no media/comments)\npython main.py delhi --mode history --limit 500\n\n# Live monitor (checks every 5 min)\npython main.py delhi --mode monitor\n\n# Scrape a user's posts\npython main.py spez --user --mode full --limit 50\n\n# Skip media or comments\npython main.py delhi --no-media --limit 200\npython main.py delhi --no-comments --limit 200\n```\n\n### ๐งช Dry Run Mode\n\nTest scrape rules without saving any data:\n\n```bash\npython main.py python --mode full --limit 50 --dry-run\n```\n\nOutput:\n```\n๐งช DRY RUN MODE - No data will be saved\n๐งช DRY RUN COMPLETE!\n ๐ Would scrape: 100 posts\n ๐ฌ Would scrape: 245 comments\n```\n\n### ๐ Plugins\n\nEnable post-processing plugins:\n\n```bash\n# List available plugins\npython main.py --list-plugins\n\n# Run with plugins enabled\npython main.py python --mode full --plugins\n```\n\n**Built-in Plugins:**\n| Plugin | Description |\n|--------|-------------|\n| `sentiment_tagger` | Adds sentiment scores to posts |\n| `deduplicator` | Removes duplicate posts |\n| `keyword_extractor` | Extracts top keywords |\n\nCreate custom plugins in `plugins/` folder.\n\n### ๐ Dashboard\n\n```bash\npython main.py --dashboard\n# Opens at http://localhost:8501\n```\n\n**Dashboard Tabs:**\n- ๐ Overview - Stats & charts\n- ๐ Analytics - Sentiment & keywords\n- ๐ Search - Query scraped data\n- ๐ฌ Comments - Comment analysis\n- โ๏ธ Scraper - Start new scrapes\n- ๐ Job History - View all jobs\n- ๐ Integrations - API, export, plugins\n\n### ๐ REST API\n\n```bash\npython main.py --api\n# API at http://localhost:8000\n# Docs at http://localhost:8000/docs\n```\n\n**Endpoints:**\n| Endpoint | Description |\n|----------|-------------|\n| `GET /posts` | List posts with filters |\n| `GET /comments` | List comments |\n| `GET /subreddits` | All scraped subreddits |\n| `GET /jobs` | Job history |\n| `GET /query?sql=...` | Raw SQL queries |\n| `GET /grafana/query` | Grafana time-series |\n\n### ๐ฆ Export & Maintenance\n\n```bash\n# Export to Parquet (for DuckDB/warehouses)\npython main.py --export-parquet python\n\n# View job history\npython main.py --job-history\n\n# Backup database\npython main.py --backup\n\n# Optimize database\npython main.py --vacuum\n```\n\n### ๐
Scheduled Scraping\n\n```bash\n# Scrape every 60 minutes\npython main.py --schedule delhi --every 60\n\n# With options\npython main.py --schedule delhi --every 30 --mode full --limit 50\n```\n\n### ๐ Search & Analytics\n\n```bash\n# Search scraped data\npython main.py --search \"credit card\" --min-score 100\n\n# Run sentiment analysis\npython main.py --analyze delhi --sentiment\n\n# Extract keywords\npython main.py --analyze delhi --keywords\n```\n\n---\n\n## ๐ณ Docker\n\n### Quick Start\n\n```bash\n# Build\ndocker build -t reddit-scraper .\n\n# Run scrape\ndocker run -v ./data:/app/data reddit-scraper python --limit 100\n\n# Run with plugins\ndocker run -v ./data:/app/data reddit-scraper python --plugins\n```\n\n### Docker Compose (Full Stack)\n\n```bash\n# Start API + Dashboard\ndocker-compose up -d\n\n# Access:\n# Dashboard: http://localhost:8501\n# API: http://localhost:8000/docs\n```\n\n### Deploy to AWS/VPS\n\n```bash\n# SSH into your server\nssh user@your-server-ip\n\n# Clone repo\ngit clone https://github.com/ksanjeev284/reddit-universal-scraper.git\ncd reddit-universal-scraper\n\n# Start services\ndocker-compose up -d\n\n# Open firewall ports\nsudo ufw allow 8000\nsudo ufw allow 8501\n```\n\nAccess:\n- `http://your-server-ip:8501` โ Dashboard\n- `http://your-server-ip:8000/docs` โ API\n\n---\n\n## ๐ External Integrations\n\n### Metabase\n\n1. Start API: `python main.py --api`\n2. Add HTTP datasource: `http://localhost:8000`\n3. Query: `/posts?subreddit=python&limit=100`\n\n### Grafana\n\n1. Install \"JSON API\" or \"Infinity\" plugin\n2. Add datasource: `http://localhost:8000`\n3. Use `/grafana/query` for time-series\n\n### DuckDB\n\n```python\nimport duckdb\n\n# Export to Parquet first\n# python main.py --export-parquet python\n\n# Query directly\nduckdb.query(\"SELECT * FROM 'data/parquet/*.parquet'\").df()\n```\n\n---\n\n## ๐ Project Structure\n\n```\nreddit-scraper/\nโโโ main.py # CLI entry point\nโโโ config.py # Settings\nโโโ analytics/ # Sentiment & keywords\nโโโ alerts/ # Discord/Telegram\nโโโ api/ # REST API server\nโโโ dashboard/ # Streamlit UI\nโโโ export/ # Database & exports\nโโโ plugins/ # Post-processing plugins\nโโโ scheduler/ # Cron scheduling\nโโโ search/ # Search engine\nโโโ data/\n โโโ r_subreddit/ # Scraped data\n โโโ backups/ # DB backups\n โโโ parquet/ # Parquet exports\n```\n\n---\n\n## ๐ Data Output\n\n### posts.csv\n| Column | Description |\n|--------|-------------|\n| id | Reddit post ID |\n| title | Post title |\n| author | Username |\n| score | Net upvotes |\n| num_comments | Comment count |\n| post_type | text/image/video/gallery |\n| selftext | Post body |\n| sentiment_score | -1.0 to 1.0 (with plugins) |\n\n### comments.csv\n| Column | Description |\n|--------|-------------|\n| comment_id | Comment ID |\n| post_permalink | Parent post |\n| author | Username |\n| body | Comment text |\n| score | Upvotes |\n\n---\n\n## โ๏ธ Environment Variables\n\n```bash\n# Notifications\nexport DISCORD_WEBHOOK_URL=\"https://discord.com/api/webhooks/...\"\nexport TELEGRAM_BOT_TOKEN=\"123456:ABC...\"\nexport TELEGRAM_CHAT_ID=\"987654321\"\n```\n\n---\n\n## ๐ License\n\nMIT License - Feel free to use, modify, and distribute.\n\n## ๐ค Contributing\n\nPull requests welcome! For major changes, please open an issue first.\n"
},
{
"path": "alerts/__init__.py",
"content": "# Alerts module\nfrom .notifications import *\n"
},
{
"path": "alerts/notifications.py",
"content": "\"\"\"\nNotification module - Discord & Telegram alerts\n\"\"\"\nimport requests\nimport json\nfrom datetime import datetime, timezone\n\ndef send_discord_alert(webhook_url, title, message, posts=None, color=0x5865F2):\n \"\"\"\n Send alert to Discord via webhook.\n \n Args:\n webhook_url: Discord webhook URL\n title: Alert title\n message: Alert message\n posts: Optional list of posts to include\n color: Embed color (default: Discord blue)\n \"\"\"\n if not webhook_url:\n print(\"โ ๏ธ Discord webhook URL not configured\")\n return False\n \n embeds = [{\n \"title\": f\"๐ค {title}\",\n \"description\": message,\n \"color\": color,\n \"timestamp\": datetime.now(timezone.utc).isoformat(),\n \"footer\": {\"text\": \"Reddit Scraper Alert\"}\n }]\n \n # Add post previews\n if posts:\n fields = []\n for post in posts[:5]: # Max 5 posts\n fields.append({\n \"name\": post.get('title', 'No Title')[:100],\n \"value\": f\"Score: {post.get('score', 0)} | Comments: {post.get('num_comments', 0)}\\n[View Post](https://reddit.com{post.get('permalink', '')})\",\n \"inline\": False\n })\n embeds[0][\"fields\"] = fields\n \n payload = {\"embeds\": embeds}\n \n try:\n response = requests.post(\n webhook_url,\n json=payload,\n headers={\"Content-Type\": \"application/json\"},\n timeout=10\n )\n if response.status_code == 204:\n print(\"โ
Discord alert sent!\")\n return True\n else:\n print(f\"โ Discord error: {response.status_code}\")\n return False\n except Exception as e:\n print(f\"โ Discord error: {e}\")\n return False\n\ndef send_telegram_alert(bot_token, chat_id, title, message, posts=None):\n \"\"\"\n Send alert to Telegram via bot.\n \n Args:\n bot_token: Telegram bot token\n chat_id: Chat/Channel ID to send to\n title: Alert title\n message: Alert message\n posts: Optional list of posts to include\n \"\"\"\n if not bot_token or not chat_id:\n print(\"โ ๏ธ Telegram credentials not configured\")\n return False\n \n # Build message\n text = f\"๐ค *{title}*\\n\\n{message}\"\n \n if posts:\n text += \"\\n\\n๐ *New Posts:*\\n\"\n for post in posts[:5]:\n title_text = post.get('title', 'No Title')[:80]\n score = post.get('score', 0)\n permalink = post.get('permalink', '')\n text += f\"\\nโข [{title_text}](https://reddit.com{permalink}) (โฌ๏ธ {score})\"\n \n url = f\"https://api.telegram.org/bot{bot_token}/sendMessage\"\n payload = {\n \"chat_id\": chat_id,\n \"text\": text,\n \"parse_mode\": \"Markdown\",\n \"disable_web_page_preview\": True\n }\n \n try:\n response = requests.post(url, json=payload, timeout=10)\n if response.status_code == 200:\n print(\"โ
Telegram alert sent!\")\n return True\n else:\n print(f\"โ Telegram error: {response.json()}\")\n return False\n except Exception as e:\n print(f\"โ Telegram error: {e}\")\n return False\n\ndef check_keyword_alerts(posts, keywords, webhook_url=None, telegram_token=None, telegram_chat=None):\n \"\"\"\n Check posts for keyword matches and send alerts.\n \n Args:\n posts: List of posts to check\n keywords: List of keywords to monitor\n webhook_url: Discord webhook URL\n telegram_token: Telegram bot token\n telegram_chat: Telegram chat ID\n \n Returns:\n List of matching posts\n \"\"\"\n if not keywords:\n return []\n \n keywords_lower = [k.lower() for k in keywords]\n matching_posts = []\n \n for post in posts:\n text = f\"{post.get('title', '')} {post.get('selftext', '')}\".lower()\n \n matched_keywords = []\n for keyword in keywords_lower:\n if keyword in text:\n matched_keywords.append(keyword)\n \n if matched_keywords:\n post['matched_keywords'] = matched_keywords\n matching_posts.append(post)\n \n if matching_posts:\n title = f\"Keyword Alert: {len(matching_posts)} matches!\"\n message = f\"Found posts matching: {', '.join(set(k for p in matching_posts for k in p.get('matched_keywords', [])))}\"\n \n if webhook_url:\n send_discord_alert(webhook_url, title, message, matching_posts, color=0xFF6B6B)\n \n if telegram_token and telegram_chat:\n send_telegram_alert(telegram_token, telegram_chat, title, message, matching_posts)\n \n return matching_posts\n\ndef send_scrape_summary(subreddit, stats, webhook_url=None, telegram_token=None, telegram_chat=None):\n \"\"\"\n Send a summary after scraping completes.\n \n Args:\n subreddit: Subreddit name\n stats: Dictionary with scrape statistics\n webhook_url: Discord webhook URL\n telegram_token: Telegram bot token\n telegram_chat: Telegram chat ID\n \"\"\"\n title = f\"Scrape Complete: r/{subreddit}\"\n message = f\"\"\"\n๐ **Statistics:**\nโข Posts: {stats.get('posts', 0)}\nโข Comments: {stats.get('comments', 0)}\nโข Images: {stats.get('images', 0)}\nโข Videos: {stats.get('videos', 0)}\nโข Duration: {stats.get('duration', 'N/A')}\n \"\"\".strip()\n \n if webhook_url:\n send_discord_alert(webhook_url, title, message, color=0x00D166)\n \n if telegram_token and telegram_chat:\n send_telegram_alert(telegram_token, telegram_chat, title, message)\n\nclass AlertMonitor:\n \"\"\"Monitor for keyword-based alerts.\"\"\"\n \n def __init__(self, keywords, discord_webhook=None, telegram_token=None, telegram_chat=None):\n self.keywords = keywords\n self.discord_webhook = discord_webhook\n self.telegram_token = telegram_token\n self.telegram_chat = telegram_chat\n self.seen_posts = set()\n \n def check_posts(self, posts):\n \"\"\"Check new posts for keyword matches.\"\"\"\n new_posts = [p for p in posts if p.get('id') not in self.seen_posts]\n \n if not new_posts:\n return []\n \n # Mark as seen\n for p in new_posts:\n self.seen_posts.add(p.get('id'))\n \n # Check for keywords\n matches = check_keyword_alerts(\n new_posts, \n self.keywords,\n self.discord_webhook,\n self.telegram_token,\n self.telegram_chat\n )\n \n return matches\n"
},
{
"path": "analytics/__init__.py",
"content": "# Analytics module\nfrom .sentiment import *\n"
},
{
"path": "analytics/sentiment.py",
"content": "\"\"\"\nAnalytics module - Sentiment Analysis, Word Clouds, Statistics\n\"\"\"\nimport re\nfrom collections import Counter\nfrom pathlib import Path\nimport sys\n\n# Simple sentiment analysis without external dependencies\nPOSITIVE_WORDS = {\n 'good', 'great', 'awesome', 'excellent', 'amazing', 'love', 'best', 'perfect',\n 'nice', 'wonderful', 'fantastic', 'brilliant', 'superb', 'outstanding', 'happy',\n 'beautiful', 'helpful', 'thanks', 'thank', 'appreciate', 'recommend', 'interesting',\n 'useful', 'cool', 'fun', 'enjoy', 'like', 'loved', 'impressive', 'incredible'\n}\n\nNEGATIVE_WORDS = {\n 'bad', 'terrible', 'awful', 'horrible', 'hate', 'worst', 'poor', 'disappointing',\n 'useless', 'waste', 'annoying', 'boring', 'ugly', 'stupid', 'dumb', 'fail',\n 'wrong', 'broken', 'sad', 'angry', 'frustrated', 'scam', 'fake', 'trash',\n 'pathetic', 'ridiculous', 'disgusting', 'overpriced', 'avoid', 'never'\n}\n\nINTENSIFIERS = {'very', 'really', 'extremely', 'absolutely', 'totally', 'completely'}\n\ndef analyze_sentiment(text):\n \"\"\"\n Simple sentiment analysis.\n Returns: (score, label)\n - score: -1.0 to 1.0\n - label: 'positive', 'negative', or 'neutral'\n \"\"\"\n if not text:\n return 0.0, 'neutral'\n \n # Clean and tokenize\n words = re.findall(r'\\b[a-z]+\\b', text.lower())\n \n if not words:\n return 0.0, 'neutral'\n \n positive_count = 0\n negative_count = 0\n intensifier_next = False\n \n for word in words:\n multiplier = 1.5 if intensifier_next else 1.0\n \n if word in POSITIVE_WORDS:\n positive_count += multiplier\n elif word in NEGATIVE_WORDS:\n negative_count += multiplier\n \n intensifier_next = word in INTENSIFIERS\n \n total = positive_count + negative_count\n if total == 0:\n return 0.0, 'neutral'\n \n score = (positive_count - negative_count) / len(words)\n score = max(-1.0, min(1.0, score * 5)) # Normalize\n \n if score > 0.1:\n label = 'positive'\n elif score < -0.1:\n label = 'negative'\n else:\n label = 'neutral'\n \n return round(score, 3), label\n\ndef analyze_posts_sentiment(posts):\n \"\"\"Analyze sentiment for a list of posts.\"\"\"\n results = []\n sentiment_counts = {'positive': 0, 'negative': 0, 'neutral': 0}\n \n for post in posts:\n text = f\"{post.get('title', '')} {post.get('selftext', '')}\"\n score, label = analyze_sentiment(text)\n post['sentiment_score'] = score\n post['sentiment_label'] = label\n sentiment_counts[label] += 1\n results.append(post)\n \n return results, sentiment_counts\n\ndef analyze_comments_sentiment(comments):\n \"\"\"Analyze sentiment for comments.\"\"\"\n results = []\n sentiment_counts = {'positive': 0, 'negative': 0, 'neutral': 0}\n \n for comment in comments:\n score, label = analyze_sentiment(comment.get('body', ''))\n comment['sentiment_score'] = score\n comment['sentiment_label'] = label\n sentiment_counts[label] += 1\n results.append(comment)\n \n return results, sentiment_counts\n\ndef extract_keywords(texts, top_n=50):\n \"\"\"Extract most common keywords from texts.\"\"\"\n # Stopwords\n stopwords = {\n 'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',\n 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',\n 'should', 'may', 'might', 'must', 'shall', 'can', 'to', 'of', 'in',\n 'for', 'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through',\n 'during', 'before', 'after', 'above', 'below', 'between', 'under',\n 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where',\n 'why', 'how', 'all', 'each', 'few', 'more', 'most', 'other', 'some',\n 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than',\n 'too', 'very', 'just', 'and', 'but', 'if', 'or', 'because', 'until',\n 'while', 'this', 'that', 'these', 'those', 'i', 'me', 'my', 'myself',\n 'we', 'our', 'you', 'your', 'he', 'she', 'it', 'they', 'them', 'what',\n 'which', 'who', 'whom', 'its', 'his', 'her', 'their', 'our', 'up',\n 'out', 'about', 'any', 'also', 'get', 'got', 'like', 'one', 'two',\n 'know', 'even', 'new', 'want', 'way', 'people', 'time', 'year', 'think',\n 'amp', 'http', 'https', 'www', 'com', 'reddit', 'deleted', 'removed', 'nan'\n }\n \n all_words = []\n for text in texts:\n if text:\n words = re.findall(r'\\b[a-z]{3,}\\b', text.lower())\n all_words.extend([w for w in words if w not in stopwords])\n \n return Counter(all_words).most_common(top_n)\n\ndef generate_wordcloud_data(texts, top_n=100):\n \"\"\"Generate word frequency data for word cloud visualization.\"\"\"\n keywords = extract_keywords(texts, top_n)\n \n if not keywords:\n return []\n \n max_count = keywords[0][1]\n \n return [\n {\"text\": word, \"value\": count, \"size\": int(10 + (count / max_count) * 90)}\n for word, count in keywords\n ]\n\ndef calculate_engagement_metrics(posts):\n \"\"\"Calculate engagement metrics for posts.\"\"\"\n if not posts:\n return {}\n \n total_posts = len(posts)\n total_score = sum(p.get('score', 0) for p in posts)\n total_comments = sum(p.get('num_comments', 0) for p in posts)\n total_awards = sum(p.get('total_awards', 0) for p in posts)\n \n # Posts with engagement\n engaged_posts = [p for p in posts if p.get('score', 0) > 0 or p.get('num_comments', 0) > 0]\n \n # Top performers\n top_by_score = sorted(posts, key=lambda x: x.get('score', 0), reverse=True)[:10]\n top_by_comments = sorted(posts, key=lambda x: x.get('num_comments', 0), reverse=True)[:10]\n \n # Post type performance\n type_performance = {}\n for post in posts:\n ptype = post.get('post_type', 'unknown')\n if ptype not in type_performance:\n type_performance[ptype] = {'count': 0, 'total_score': 0, 'total_comments': 0}\n type_performance[ptype]['count'] += 1\n type_performance[ptype]['total_score'] += post.get('score', 0)\n type_performance[ptype]['total_comments'] += post.get('num_comments', 0)\n \n for ptype in type_performance:\n count = type_performance[ptype]['count']\n type_performance[ptype]['avg_score'] = type_performance[ptype]['total_score'] / count\n type_performance[ptype]['avg_comments'] = type_performance[ptype]['total_comments'] / count\n \n return {\n 'total_posts': total_posts,\n 'total_score': total_score,\n 'total_comments': total_comments,\n 'total_awards': total_awards,\n 'avg_score': total_score / total_posts if total_posts else 0,\n 'avg_comments': total_comments / total_posts if total_posts else 0,\n 'engagement_rate': len(engaged_posts) / total_posts if total_posts else 0,\n 'top_by_score': top_by_score,\n 'top_by_comments': top_by_comments,\n 'type_performance': type_performance\n }\n\ndef find_best_posting_times(posts):\n \"\"\"Analyze best times to post based on engagement.\"\"\"\n hourly_stats = {}\n daily_stats = {}\n \n for post in posts:\n created = post.get('created_utc', '')\n if not created:\n continue\n \n try:\n # Parse ISO format\n from datetime import datetime\n dt = datetime.fromisoformat(created.replace('Z', '+00:00'))\n hour = dt.hour\n day = dt.strftime('%A')\n \n # Hourly\n if hour not in hourly_stats:\n hourly_stats[hour] = {'count': 0, 'total_score': 0}\n hourly_stats[hour]['count'] += 1\n hourly_stats[hour]['total_score'] += post.get('score', 0)\n \n # Daily\n if day not in daily_stats:\n daily_stats[day] = {'count': 0, 'total_score': 0}\n daily_stats[day]['count'] += 1\n daily_stats[day]['total_score'] += post.get('score', 0)\n except:\n continue\n \n # Calculate averages\n for hour in hourly_stats:\n hourly_stats[hour]['avg_score'] = hourly_stats[hour]['total_score'] / hourly_stats[hour]['count']\n \n for day in daily_stats:\n daily_stats[day]['avg_score'] = daily_stats[day]['total_score'] / daily_stats[day]['count']\n \n # Find best times\n best_hours = sorted(hourly_stats.items(), key=lambda x: x[1]['avg_score'], reverse=True)[:5]\n best_days = sorted(daily_stats.items(), key=lambda x: x[1]['avg_score'], reverse=True)[:3]\n \n return {\n 'hourly_stats': hourly_stats,\n 'daily_stats': daily_stats,\n 'best_hours': [(h, s['avg_score']) for h, s in best_hours],\n 'best_days': [(d, s['avg_score']) for d, s in best_days]\n }\n"
},
{
"path": "analytics/subreddit_stats.py",
"content": "\"\"\"\nSubreddit Statistics - Subscribers, rules, mods, and metadata\n\"\"\"\nimport requests\nfrom datetime import datetime\nimport json\n\nUSER_AGENT = \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\"\n\ndef get_subreddit_about(subreddit):\n \"\"\"\n Fetch subreddit metadata (subscribers, description, rules, etc.)\n \n Args:\n subreddit: Subreddit name (without r/)\n \n Returns:\n Dictionary with subreddit info\n \"\"\"\n url = f\"https://old.reddit.com/r/{subreddit}/about.json\"\n \n try:\n response = requests.get(url, headers={\"User-Agent\": USER_AGENT}, timeout=15)\n \n if response.status_code != 200:\n print(f\"โ Failed to fetch r/{subreddit} info: {response.status_code}\")\n return None\n \n data = response.json()['data']\n \n return {\n \"name\": data.get('display_name'),\n \"title\": data.get('title'),\n \"description\": data.get('public_description'),\n \"subscribers\": data.get('subscribers', 0),\n \"active_users\": data.get('accounts_active', 0),\n \"created_utc\": datetime.fromtimestamp(data.get('created_utc', 0)).isoformat(),\n \"over_18\": data.get('over18', False),\n \"subreddit_type\": data.get('subreddit_type'), # public, private, restricted\n \"lang\": data.get('lang'),\n \"icon_url\": data.get('icon_img', '').split('?')[0] if data.get('icon_img') else None,\n \"banner_url\": data.get('banner_img', '').split('?')[0] if data.get('banner_img') else None,\n \"header_url\": data.get('header_img'),\n \"community_icon\": data.get('community_icon', '').split('?')[0] if data.get('community_icon') else None,\n \"wiki_enabled\": data.get('wiki_enabled', False),\n \"spoilers_enabled\": data.get('spoilers_enabled', False),\n \"allow_videos\": data.get('allow_videos', False),\n \"allow_images\": data.get('allow_images', False),\n \"allow_polls\": data.get('allow_polls', False),\n }\n except Exception as e:\n print(f\"โ Error fetching subreddit info: {e}\")\n return None\n\ndef get_subreddit_rules(subreddit):\n \"\"\"\n Fetch subreddit rules.\n \n Args:\n subreddit: Subreddit name\n \n Returns:\n List of rule dictionaries\n \"\"\"\n url = f\"https://old.reddit.com/r/{subreddit}/about/rules.json\"\n \n try:\n response = requests.get(url, headers={\"User-Agent\": USER_AGENT}, timeout=15)\n \n if response.status_code != 200:\n return []\n \n data = response.json()\n rules = []\n \n for rule in data.get('rules', []):\n rules.append({\n \"short_name\": rule.get('short_name'),\n \"description\": rule.get('description'),\n \"priority\": rule.get('priority'),\n \"kind\": rule.get('kind'), # link, comment, all\n \"created_utc\": datetime.fromtimestamp(rule.get('created_utc', 0)).isoformat()\n })\n \n return rules\n except Exception as e:\n print(f\"โ Error fetching rules: {e}\")\n return []\n\ndef get_subreddit_mods(subreddit):\n \"\"\"\n Fetch subreddit moderators.\n \n Args:\n subreddit: Subreddit name\n \n Returns:\n List of moderator usernames\n \"\"\"\n url = f\"https://old.reddit.com/r/{subreddit}/about/moderators.json\"\n \n try:\n response = requests.get(url, headers={\"User-Agent\": USER_AGENT}, timeout=15)\n \n if response.status_code != 200:\n return []\n \n data = response.json()\n mods = []\n \n for mod in data.get('data', {}).get('children', []):\n mods.append({\n \"name\": mod.get('name'),\n \"permissions\": mod.get('mod_permissions', []),\n \"added_utc\": datetime.fromtimestamp(mod.get('date', 0)).isoformat() if mod.get('date') else None\n })\n \n return mods\n except Exception as e:\n print(f\"โ Error fetching mods: {e}\")\n return []\n\ndef get_subreddit_flairs(subreddit):\n \"\"\"\n Fetch available post flairs.\n \n Args:\n subreddit: Subreddit name\n \n Returns:\n List of flair options\n \"\"\"\n url = f\"https://old.reddit.com/r/{subreddit}/api/link_flair_v2.json\"\n \n try:\n response = requests.get(url, headers={\"User-Agent\": USER_AGENT}, timeout=15)\n \n if response.status_code != 200:\n return []\n \n flairs = []\n for flair in response.json():\n flairs.append({\n \"text\": flair.get('text'),\n \"id\": flair.get('id'),\n \"background_color\": flair.get('background_color'),\n \"text_color\": flair.get('text_color'),\n \"type\": flair.get('type')\n })\n \n return flairs\n except Exception as e:\n return []\n\ndef get_full_subreddit_stats(subreddit):\n \"\"\"\n Get comprehensive subreddit statistics.\n \n Args:\n subreddit: Subreddit name\n \n Returns:\n Dictionary with all stats\n \"\"\"\n print(f\"๐ Fetching stats for r/{subreddit}...\")\n \n about = get_subreddit_about(subreddit)\n \n if not about:\n return None\n \n rules = get_subreddit_rules(subreddit)\n mods = get_subreddit_mods(subreddit)\n flairs = get_subreddit_flairs(subreddit)\n \n stats = {\n **about,\n \"rules\": rules,\n \"rules_count\": len(rules),\n \"moderators\": mods,\n \"moderator_count\": len(mods),\n \"flairs\": flairs,\n \"flair_count\": len(flairs),\n \"fetched_at\": datetime.now().isoformat()\n }\n \n # Print summary\n print(f\"\\n๐ r/{subreddit} Statistics:\")\n print(f\" ๐ฅ Subscribers: {stats['subscribers']:,}\")\n print(f\" ๐ข Active Users: {stats['active_users']:,}\")\n print(f\" ๐ Rules: {stats['rules_count']}\")\n print(f\" ๐ฎ Moderators: {stats['moderator_count']}\")\n print(f\" ๐ท๏ธ Flairs: {stats['flair_count']}\")\n print(f\" ๐
Created: {stats['created_utc'][:10]}\")\n print(f\" ๐ NSFW: {stats['over_18']}\")\n \n return stats\n\ndef save_subreddit_stats(subreddit, output_dir=\"data\"):\n \"\"\"\n Fetch and save subreddit stats to JSON.\n \n Args:\n subreddit: Subreddit name\n output_dir: Output directory\n \n Returns:\n Path to saved file\n \"\"\"\n import os\n \n stats = get_full_subreddit_stats(subreddit)\n \n if not stats:\n return None\n \n save_dir = f\"{output_dir}/r_{subreddit}\"\n os.makedirs(save_dir, exist_ok=True)\n \n filepath = f\"{save_dir}/subreddit_stats.json\"\n \n with open(filepath, 'w', encoding='utf-8') as f:\n json.dump(stats, f, indent=2, ensure_ascii=False)\n \n print(f\"\\n๐พ Saved to {filepath}\")\n return filepath\n\n# CLI for testing\nif __name__ == \"__main__\":\n import argparse\n \n parser = argparse.ArgumentParser(description=\"Subreddit Statistics\")\n parser.add_argument(\"subreddit\", help=\"Subreddit name\")\n parser.add_argument(\"--save\", action=\"store_true\", help=\"Save to JSON\")\n \n args = parser.parse_args()\n \n if args.save:\n save_subreddit_stats(args.subreddit)\n else:\n stats = get_full_subreddit_stats(args.subreddit)\n if stats:\n print(f\"\\n๐ Description: {stats['description'][:200]}...\" if stats['description'] else \"\")\n"
},
{
"path": "api/__init__.py",
"content": "\"\"\"Reddit Scraper REST API\"\"\"\nfrom .server import app\n"
},
{
"path": "api/server.py",
"content": "\"\"\"\nREST API Module - Expose Reddit Scraper data as a REST API\nFor integration with Metabase, Grafana, DreamFactory, and other tools.\n\nStart with: python api/server.py\nOr: uvicorn api.server:app --reload --port 8000\n\"\"\"\nfrom fastapi import FastAPI, Query, HTTPException\nfrom fastapi.middleware.cors import CORSMiddleware\nfrom typing import Optional, List\nimport sys\nfrom pathlib import Path\n\n# Add parent to path\nsys.path.insert(0, str(Path(__file__).parent.parent))\n\nfrom export.database import (\n get_connection, search_posts, search_comments,\n get_subreddit_stats, get_all_subreddits,\n get_job_history, get_job_stats, get_database_info\n)\n\n# Create FastAPI app\napp = FastAPI(\n title=\"Reddit Scraper API\",\n description=\"REST API for Reddit Scraper data. Use with Metabase, Grafana, or any tool.\",\n version=\"1.0.0\",\n docs_url=\"/docs\",\n redoc_url=\"/redoc\"\n)\n\n# Enable CORS for external tools\napp.add_middleware(\n CORSMiddleware,\n allow_origins=[\"*\"], # Allow all origins for local tools\n allow_credentials=True,\n allow_methods=[\"*\"],\n allow_headers=[\"*\"],\n)\n\n\n# --- HEALTH & INFO ---\n\n@app.get(\"/\", tags=[\"Info\"])\ndef root():\n \"\"\"API root - basic info.\"\"\"\n return {\n \"name\": \"Reddit Scraper API\",\n \"version\": \"1.0.0\",\n \"docs\": \"/docs\",\n \"endpoints\": [\"/posts\", \"/comments\", \"/subreddits\", \"/jobs\", \"/stats\"]\n }\n\n\n@app.get(\"/health\", tags=[\"Info\"])\ndef health_check():\n \"\"\"Health check endpoint.\"\"\"\n try:\n info = get_database_info()\n return {\"status\": \"healthy\", \"database\": info}\n except Exception as e:\n return {\"status\": \"unhealthy\", \"error\": str(e)}\n\n\n@app.get(\"/info\", tags=[\"Info\"])\ndef database_info():\n \"\"\"Get database info and table counts.\"\"\"\n return get_database_info()\n\n\n# --- POSTS ---\n\n@app.get(\"/posts\", tags=[\"Posts\"])\ndef list_posts(\n q: Optional[str] = Query(None, description=\"Search query\"),\n subreddit: Optional[str] = Query(None, description=\"Filter by subreddit\"),\n author: Optional[str] = Query(None, description=\"Filter by author\"),\n min_score: Optional[int] = Query(None, description=\"Minimum score\"),\n post_type: Optional[str] = Query(None, description=\"Post type filter\"),\n limit: int = Query(100, ge=1, le=1000, description=\"Max results\")\n):\n \"\"\"\n Get posts with optional filters.\n \n Use for Grafana dashboards, Metabase queries, or custom integrations.\n \"\"\"\n return search_posts(\n query=q,\n subreddit=subreddit,\n author=author,\n min_score=min_score,\n post_type=post_type,\n limit=limit\n )\n\n\n@app.get(\"/posts/{post_id}\", tags=[\"Posts\"])\ndef get_post(post_id: str):\n \"\"\"Get a single post by ID.\"\"\"\n conn = get_connection()\n cursor = conn.cursor()\n cursor.execute(\"SELECT * FROM posts WHERE id = ?\", (post_id,))\n row = cursor.fetchone()\n conn.close()\n \n if not row:\n raise HTTPException(status_code=404, detail=\"Post not found\")\n return dict(row)\n\n\n# --- COMMENTS ---\n\n@app.get(\"/comments\", tags=[\"Comments\"])\ndef list_comments(\n q: Optional[str] = Query(None, description=\"Search in comment body\"),\n post_id: Optional[str] = Query(None, description=\"Filter by post ID\"),\n author: Optional[str] = Query(None, description=\"Filter by author\"),\n min_score: Optional[int] = Query(None, description=\"Minimum score\"),\n limit: int = Query(100, ge=1, le=1000, description=\"Max results\")\n):\n \"\"\"Get comments with optional filters.\"\"\"\n return search_comments(\n query=q,\n post_id=post_id,\n author=author,\n min_score=min_score,\n limit=limit\n )\n\n\n# --- SUBREDDITS ---\n\n@app.get(\"/subreddits\", tags=[\"Subreddits\"])\ndef list_subreddits():\n \"\"\"Get all scraped subreddits with post counts.\"\"\"\n return get_all_subreddits()\n\n\n@app.get(\"/subreddits/{subreddit}/stats\", tags=[\"Subreddits\"])\ndef subreddit_stats(subreddit: str):\n \"\"\"Get detailed statistics for a subreddit.\"\"\"\n stats = get_subreddit_stats(subreddit)\n if not stats.get('total_posts'):\n raise HTTPException(status_code=404, detail=f\"No data for r/{subreddit}\")\n return stats\n\n\n# --- JOBS ---\n\n@app.get(\"/jobs\", tags=[\"Jobs\"])\ndef list_jobs(\n status: Optional[str] = Query(None, description=\"Filter by status\"),\n target: Optional[str] = Query(None, description=\"Filter by target\"),\n limit: int = Query(50, ge=1, le=200)\n):\n \"\"\"Get job history.\"\"\"\n return get_job_history(limit=limit, target=target, status=status)\n\n\n@app.get(\"/jobs/stats\", tags=[\"Jobs\"])\ndef job_stats():\n \"\"\"Get aggregated job statistics.\"\"\"\n return get_job_stats()\n\n\n# --- RAW SQL (for advanced users) ---\n\n@app.get(\"/query\", tags=[\"Advanced\"])\ndef raw_query(\n sql: str = Query(..., description=\"SQL SELECT query\"),\n limit: int = Query(100, ge=1, le=1000)\n):\n \"\"\"\n Execute a raw SQL SELECT query.\n \n โ ๏ธ Only SELECT queries allowed. Use for custom Grafana/Metabase queries.\n \n Example: /query?sql=SELECT title, score FROM posts ORDER BY score DESC\n \"\"\"\n # Security: Only allow SELECT\n if not sql.strip().upper().startswith(\"SELECT\"):\n raise HTTPException(status_code=400, detail=\"Only SELECT queries allowed\")\n \n # Add limit if not present\n if \"LIMIT\" not in sql.upper():\n sql = f\"{sql} LIMIT {limit}\"\n \n try:\n conn = get_connection()\n cursor = conn.cursor()\n cursor.execute(sql)\n results = [dict(row) for row in cursor.fetchall()]\n conn.close()\n return {\"query\": sql, \"count\": len(results), \"results\": results}\n except Exception as e:\n raise HTTPException(status_code=400, detail=f\"Query error: {e}\")\n\n\n# --- GRAFANA COMPATIBLE ENDPOINTS ---\n\n@app.get(\"/grafana/search\", tags=[\"Grafana\"])\ndef grafana_search():\n \"\"\"Grafana SimpleJSON datasource - search endpoint.\"\"\"\n subs = get_all_subreddits()\n return [s['subreddit'] for s in subs]\n\n\n@app.post(\"/grafana/query\", tags=[\"Grafana\"])\ndef grafana_query(body: dict):\n \"\"\"Grafana SimpleJSON datasource - query endpoint.\"\"\"\n # Return time series data for Grafana\n results = []\n \n for target in body.get('targets', []):\n subreddit = target.get('target')\n if subreddit:\n conn = get_connection()\n cursor = conn.cursor()\n cursor.execute(\"\"\"\n SELECT date(created_utc) as time, COUNT(*) as value\n FROM posts WHERE subreddit = ?\n GROUP BY date(created_utc)\n ORDER BY time\n \"\"\", (subreddit,))\n \n datapoints = [[row['value'], row['time']] for row in cursor.fetchall()]\n conn.close()\n \n results.append({\n \"target\": subreddit,\n \"datapoints\": datapoints\n })\n \n return results\n\n\n# --- CLI ---\n\nif __name__ == \"__main__\":\n import uvicorn\n print(\"๐ Starting Reddit Scraper API...\")\n print(\" ๐ Docs: http://localhost:8000/docs\")\n print(\" ๐ Use with Metabase, Grafana, or any REST client\")\n uvicorn.run(app, host=\"0.0.0.0\", port=8000)\n"
},
{
"path": "config.py",
"content": "\"\"\"\nReddit Scraper Suite - Configuration\n\"\"\"\nimport os\nfrom pathlib import Path\n\n# --- PATHS ---\nBASE_DIR = Path(__file__).parent\nDATA_DIR = BASE_DIR / \"data\"\nDB_PATH = DATA_DIR / \"reddit_scraper.db\"\n\n# --- SCRAPER SETTINGS ---\nUSER_AGENT = \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\"\n\n# Sources: old.reddit.com for residential IPs, mirrors for data centers\nMIRRORS = [\n \"https://old.reddit.com\",\n \"https://redlib.catsarch.com\",\n \"https://redlib.vsls.cz\",\n \"https://r.nf\",\n \"https://libreddit.northboot.xyz\",\n \"https://redlib.tux.pizza\"\n]\n\n# Rate limiting\nREQUEST_TIMEOUT = 15\nCOOLDOWN_SECONDS = 3\nRETRY_WAIT = 30\n\n# Media settings\nMAX_IMAGES_PER_POST = 10\nMAX_VIDEOS_PER_POST = 2\nMAX_GALLERY_IMAGES = 15\n\n# Comment settings\nMAX_COMMENT_DEPTH = 5\n\n# --- ASYNC SETTINGS ---\nASYNC_MAX_CONCURRENT = 10\nASYNC_BATCH_SIZE = 50\n\n# --- NOTIFICATION SETTINGS ---\nDISCORD_WEBHOOK_URL = os.getenv(\"DISCORD_WEBHOOK_URL\", \"\")\nTELEGRAM_BOT_TOKEN = os.getenv(\"TELEGRAM_BOT_TOKEN\", \"\")\nTELEGRAM_CHAT_ID = os.getenv(\"TELEGRAM_CHAT_ID\", \"\")\n\n# --- DASHBOARD SETTINGS ---\nDASHBOARD_HOST = \"0.0.0.0\"\nDASHBOARD_PORT = 8501\n\n# --- SCHEDULER SETTINGS ---\nSCHEDULER_TIMEZONE = \"Asia/Kolkata\"\n\n# --- DATABASE SETTINGS ---\nDATABASE_URL = os.getenv(\"DATABASE_URL\", f\"sqlite:///{DB_PATH}\")\n\n# Ensure data directory exists\nDATA_DIR.mkdir(exist_ok=True)\n"
},
{
"path": "dashboard/__init__.py",
"content": "# Dashboard module\n"
},
{
"path": "dashboard/app.py",
"content": "\"\"\"\nReddit Scraper Dashboard - Streamlit Web UI\nRun with: streamlit run dashboard/app.py\n\"\"\"\nimport streamlit as st\nimport pandas as pd\nfrom pathlib import Path\nimport sys\nfrom datetime import datetime\nimport time\nimport os\nimport json\nimport signal\n\n# Add parent to path\nsys.path.insert(0, str(Path(__file__).parent.parent))\n\nfrom analytics.sentiment import (\n analyze_posts_sentiment, extract_keywords, \n calculate_engagement_metrics, find_best_posting_times\n)\nfrom search.query import search_all_data, advanced_search, get_top_posts\n\n# Page config\nst.set_page_config(\n page_title=\"Reddit Scraper Dashboard\",\n page_icon=\"๐ค\",\n layout=\"wide\",\n initial_sidebar_state=\"expanded\"\n)\n\n# Custom CSS\nst.markdown(\"\"\"\n\n\"\"\", unsafe_allow_html=True)\n\ndef load_subreddit_data(subreddit_path):\n \"\"\"Load all data for a subreddit.\"\"\"\n data = {}\n \n posts_file = subreddit_path / 'posts.csv'\n if posts_file.exists():\n data['posts'] = pd.read_csv(posts_file)\n \n comments_file = subreddit_path / 'comments.csv'\n if comments_file.exists():\n data['comments'] = pd.read_csv(comments_file)\n \n return data\n\ndef get_available_data():\n \"\"\"Get list of scraped subreddits and users.\"\"\"\n data_dir = Path(__file__).parent.parent / 'data'\n data = {'subreddits': [], 'users': []}\n \n if data_dir.exists():\n for sub_dir in data_dir.iterdir():\n if sub_dir.is_dir():\n # Check for r_ or u_ prefix (standard scraper format)\n # We allow folders even without posts.csv so users can see empty scrapes\n if sub_dir.name.startswith('u_'):\n data['users'].append(sub_dir.name)\n elif sub_dir.name.startswith('r_'):\n data['subreddits'].append(sub_dir.name)\n elif (sub_dir / 'posts.csv').exists():\n # Fallback for old/other folders that have data\n data['subreddits'].append(sub_dir.name)\n \n # Sort lists\n data['subreddits'].sort()\n data['users'].sort()\n return data\n\ndef main():\n # Header\n st.markdown('๐ค Reddit Scraper Dashboard
', unsafe_allow_html=True)\n \n # Sidebar\n st.sidebar.title(\"๐ Navigation\")\n \n if st.sidebar.button(\"๐ Refresh List\"):\n st.rerun()\n \n # Get available data\n available_data = get_available_data()\n \n # Source Selector\n source_type = st.sidebar.radio(\n \"Source Type\",\n [\"Subreddits\", \"Users\"],\n horizontal=True\n )\n \n # Filter list based on type\n if source_type == \"Users\":\n options = available_data['users']\n prefix_len = 2 # 'u_'\n empty_msg = \"No scraped users found.\"\n icon = \"๐ค\"\n else:\n options = available_data['subreddits']\n prefix_len = 2 # 'r_' is 2 chars, but some might not have it if legacy? \n # Actually standard scraper uses r_.\n empty_msg = \"No scraped subreddits found.\"\n icon = \"๐\"\n \n selected_sub = None\n \n if not options:\n st.sidebar.warning(empty_msg)\n if source_type == \"Subreddits\":\n st.sidebar.info(\"Go to 'โ๏ธ Scraper' tab to start scraping.\")\n else:\n st.sidebar.info(\"Go to 'โ๏ธ Scraper' tab to start scraping users.\")\n else:\n # Selector\n selected_sub = st.sidebar.selectbox(\n f\"Select {source_type[:-1]}\", # \"Select Subreddit\" or \"Select User\"\n options,\n format_func=lambda x: f\"{icon} {x[2:] if x.startswith(('r_', 'u_')) else x}\"\n )\n \n # Load data if selected\n posts_df = pd.DataFrame()\n comments_df = pd.DataFrame()\n data_loaded = False\n \n if selected_sub:\n data_dir = Path(__file__).parent.parent / 'data'\n sub_path = data_dir / selected_sub\n data = load_subreddit_data(sub_path)\n \n if 'posts' in data:\n posts_df = data['posts']\n comments_df = data.get('comments', pd.DataFrame())\n data_loaded = True\n else:\n st.error(\"No posts data found for selected item!\")\n \n # Define Tabs\n # Data tabs only if data loaded\n tab_list = []\n if data_loaded:\n tab_list.extend([\"๐ Overview\", \"๐ Analytics\", \"๐ Search\", \"๐ฌ Comments\"])\n \n # Always present tabs\n tab_list.extend([\"โ๏ธ Scraper\", \"๐ Job History\", \"๐ Integrations\"])\n \n # Create tabs\n tabs = st.tabs(tab_list)\n \n # Map tabs to variables for easy access\n tab_map = {name: tabs[i] for i, name in enumerate(tab_list)}\n \n # --- RENDER TABS ---\n \n if data_loaded:\n with tab_map[\"๐ Overview\"]:\n st.header(f\"๐ Overview: {selected_sub}\")\n \n # Metrics row\n col1, col2, col3, col4, col5 = st.columns(5)\n \n with col1:\n st.metric(\"Total Posts\", len(posts_df))\n with col2:\n st.metric(\"Total Comments\", len(comments_df))\n with col3:\n total_score = posts_df['score'].sum() if 'score' in posts_df else 0\n st.metric(\"Total Score\", f\"{total_score:,}\")\n with col4:\n avg_score = posts_df['score'].mean() if 'score' in posts_df else 0\n st.metric(\"Avg Score\", f\"{avg_score:.1f}\")\n with col5:\n media_count = posts_df['has_media'].sum() if 'has_media' in posts_df else 0\n st.metric(\"Media Posts\", int(media_count))\n \n st.divider()\n \n # Post type distribution\n col1, col2 = st.columns(2)\n \n with col1:\n st.subheader(\"๐ Post Types\")\n if 'post_type' in posts_df:\n type_counts = posts_df['post_type'].value_counts()\n st.bar_chart(type_counts)\n \n with col2:\n st.subheader(\"๐
Posts Over Time\")\n if 'created_utc' in posts_df:\n posts_df['date'] = pd.to_datetime(posts_df['created_utc']).dt.date\n daily = posts_df.groupby('date').size()\n st.line_chart(daily)\n \n st.divider()\n \n # Top posts\n st.subheader(\"๐ฅ Top Posts by Score\")\n if 'score' in posts_df:\n top_posts = posts_df.nlargest(10, 'score')[['title', 'score', 'num_comments', 'post_type', 'created_utc']]\n st.dataframe(top_posts)\n\n with tab_map[\"๐ Analytics\"]:\n st.header(\"๐ Analytics\")\n \n # Sentiment Analysis\n st.subheader(\"๐ Sentiment Analysis\")\n \n if st.button(\"Run Sentiment Analysis\"):\n with st.spinner(\"Analyzing sentiment...\"):\n posts_list = posts_df.to_dict('records')\n analyzed_posts, sentiment_counts = analyze_posts_sentiment(posts_list)\n \n col1, col2, col3 = st.columns(3)\n col1.metric(\"Positive\", sentiment_counts['positive'], delta=None)\n col2.metric(\"Neutral\", sentiment_counts['neutral'], delta=None)\n col3.metric(\"Negative\", sentiment_counts['negative'], delta=None)\n \n # Pie chart\n sentiment_df = pd.DataFrame({\n 'Sentiment': ['Positive', 'Neutral', 'Negative'],\n 'Count': [sentiment_counts['positive'], sentiment_counts['neutral'], sentiment_counts['negative']]\n })\n st.bar_chart(sentiment_df.set_index('Sentiment'))\n \n st.divider()\n \n # Keywords\n st.subheader(\"โ๏ธ Top Keywords\")\n texts = posts_df['title'].tolist()\n if 'selftext' in posts_df:\n texts.extend(posts_df['selftext'].dropna().tolist())\n \n keywords = extract_keywords(texts, top_n=30)\n \n if keywords:\n kw_df = pd.DataFrame(keywords, columns=['Word', 'Count'])\n st.bar_chart(kw_df.set_index('Word').head(20))\n \n st.divider()\n \n # Best posting times\n st.subheader(\"โฐ Best Posting Times\")\n \n if 'created_utc' in posts_df:\n timing_data = find_best_posting_times(posts_df.to_dict('records'))\n \n if timing_data['best_hours']:\n st.write(\"**Best Hours to Post:**\")\n for hour, avg_score in timing_data['best_hours']:\n st.write(f\"โข {hour}:00 - Avg Score: {avg_score:.1f}\")\n \n if timing_data['best_days']:\n st.write(\"**Best Days to Post:**\")\n for day, avg_score in timing_data['best_days']:\n st.write(f\"โข {day} - Avg Score: {avg_score:.1f}\")\n\n with tab_map[\"๐ Search\"]:\n st.header(\"๐ Search Posts\")\n \n # Search form\n col1, col2 = st.columns([3, 1])\n \n with col1:\n search_query = st.text_input(\"Search query\", placeholder=\"Enter keywords...\")\n \n with col2:\n min_score = st.number_input(\"Min Score\", min_value=0, value=0)\n \n col3, col4, col5 = st.columns(3)\n \n with col3:\n if 'post_type' in posts_df:\n post_types = ['All'] + posts_df['post_type'].dropna().unique().tolist()\n selected_type = st.selectbox(\"Post Type\", post_types)\n \n with col4:\n if 'author' in posts_df:\n authors = ['All'] + posts_df['author'].dropna().unique().tolist()[:50]\n selected_author = st.selectbox(\"Author\", authors)\n \n with col5:\n sort_by = st.selectbox(\"Sort by\", ['score', 'num_comments', 'created_utc'])\n \n # Search button\n if st.button(\"๐ Search\"):\n filtered = posts_df.copy()\n \n if search_query:\n mask = filtered['title'].str.contains(search_query, case=False, na=False)\n if 'selftext' in filtered:\n mask |= filtered['selftext'].str.contains(search_query, case=False, na=False)\n filtered = filtered[mask]\n \n if min_score > 0:\n filtered = filtered[filtered['score'] >= min_score]\n \n if selected_type != 'All' and 'post_type' in filtered:\n filtered = filtered[filtered['post_type'] == selected_type]\n \n if selected_author != 'All' and 'author' in filtered:\n filtered = filtered[filtered['author'] == selected_author]\n \n filtered = filtered.sort_values(sort_by, ascending=False)\n \n st.write(f\"Found {len(filtered)} results\")\n st.dataframe(filtered[['title', 'score', 'num_comments', 'post_type', 'author', 'created_utc']].head(50))\n\n with tab_map[\"๐ฌ Comments\"]:\n st.header(\"๐ฌ Comments Analysis\")\n \n if len(comments_df) == 0:\n st.warning(\"No comments data found for this subreddit\")\n else:\n col1, col2, col3 = st.columns(3)\n \n with col1:\n st.metric(\"Total Comments\", len(comments_df))\n with col2:\n avg_score = comments_df['score'].mean() if 'score' in comments_df else 0\n st.metric(\"Avg Score\", f\"{avg_score:.1f}\")\n with col3:\n unique_authors = comments_df['author'].nunique() if 'author' in comments_df else 0\n st.metric(\"Unique Commenters\", unique_authors)\n \n st.divider()\n \n # Top comments\n st.subheader(\"๐ฅ Top Comments by Score\")\n if 'score' in comments_df:\n top_comments = comments_df.nlargest(10, 'score')[['body', 'score', 'author', 'created_utc']]\n for _, row in top_comments.iterrows():\n with st.expander(f\"โฌ๏ธ {row['score']} - by u/{row['author']}\"):\n st.write(row['body'][:500])\n \n st.divider()\n \n # Top commenters\n st.subheader(\"๐ฅ Top Commenters\")\n if 'author' in comments_df:\n top_authors = comments_df['author'].value_counts().head(10)\n st.bar_chart(top_authors)\n\n # Scraper Tab (Always visible)\n with tab_map[\"โ๏ธ Scraper\"]:\n\n st.header(\"โ๏ธ Scraper Controls\")\n \n # Persistence logic\n import json\n import signal\n \n JOB_FILE = Path(\"active_job.json\")\n LOG_DIR = Path(\"logs\")\n LOG_DIR.mkdir(exist_ok=True)\n\n def get_active_job():\n if JOB_FILE.exists():\n try:\n with open(JOB_FILE, \"r\") as f:\n return json.load(f)\n except:\n return None\n return None\n\n # Check for active job\n active_job = get_active_job()\n \n # Auto-detect if process is dead\n if active_job:\n try:\n import psutil\n if not psutil.pid_exists(active_job['pid']):\n # Process is dead\n if JOB_FILE.exists():\n JOB_FILE.unlink()\n active_job = None\n st.rerun()\n except ImportError:\n # Fallback for systems without psutil\n try:\n os.kill(active_job['pid'], 0)\n except OSError:\n # PID doesn't exist (Process dead)\n if JOB_FILE.exists():\n JOB_FILE.unlink()\n active_job = None\n st.rerun()\n \n # Monitor Section (Always visible if job exists)\n if active_job:\n st.info(f\"๐ **Scraping in Progress**: {active_job.get('target', 'Unknown')} (PID: {active_job.get('pid')})\")\n \n # Stop button\n if st.button(\"๐ Stop Scraping\"):\n try:\n import signal\n os.kill(active_job['pid'], signal.SIGTERM)\n st.warning(\"Stopped process.\")\n except:\n st.warning(\"Process already stopped.\")\n \n if JOB_FILE.exists():\n JOB_FILE.unlink()\n st.rerun()\n \n # Read logs\n log_file = Path(active_job['log_file'])\n if log_file.exists():\n with open(log_file, \"r\", encoding=\"utf-8\", errors=\"replace\") as f:\n lines = f.readlines()\n \n # Parse metrics from lines\n posts_saved = 0\n comments_count = 0\n images_count = 0\n videos_count = 0\n found_posts = 0\n processed_posts = 0\n \n for line in lines:\n import re\n # Progress: X/Y (Saved posts)\n m = re.search(r'Progress: (\\d+)/(\\d+)', line)\n if m: posts_saved = int(m.group(1))\n \n # Saved X posts\n m = re.search(r'Saved (\\d+)', line)\n if m: posts_saved += int(m.group(1))\n \n # Found X posts\n m = re.search(r'Found (\\d+) posts', line)\n if m: found_posts += int(m.group(1))\n \n # Processed posts (Fetching comments)\n if \"Fetching comments for:\" in line: \n processed_posts += 1\n \n # Comments: X (Summary)\n m = re.search(r'Comments:\\s*(\\d+)', line)\n if m: \n comments_count = int(m.group(1))\n else:\n # Incremental comments\n m = re.search(r'\\+ Scraped (\\d+) comments', line)\n if m: comments_count += int(m.group(1))\n \n # Images/Videos (Summary line)\n m = re.search(r'Images:\\s*(\\d+).*Videos:\\s*(\\d+)', line)\n if m:\n images_count = int(m.group(1))\n videos_count = int(m.group(2))\n \n # Images/Videos (Real-time line)\n m = re.search(r'\\+ Downloaded: (\\d+) images, (\\d+) videos', line)\n if m:\n images_count += int(m.group(1))\n videos_count += int(m.group(2))\n\n # Display Metrics\n col1, col2, col3, col4 = st.columns(4)\n \n # Posts Metric Logic\n if posts_saved > 0:\n col1.metric(\"๐ Posts\", f\"{posts_saved} (Found {found_posts})\")\n elif found_posts > 0:\n col1.metric(\"๐ Posts\", f\"Processing: {processed_posts}/{found_posts}\")\n else:\n col1.metric(\"๐ Posts\", \"0\")\n \n col2.metric(\"๐ฌ Comments\", comments_count)\n col3.metric(\"๐ผ๏ธ Images\", images_count)\n col4.metric(\"๐ฌ Videos\", videos_count)\n \n # Show latest logs\n st.code(\"\".join(lines[-20:]), language=\"text\")\n \n # Auto-refresh\n time.sleep(1)\n st.rerun()\n else:\n st.warning(\"Log file not found.\")\n \n else:\n # Start New Scrape UI\n st.subheader(\"๐ Start New Scrape\")\n \n col1, col2 = st.columns(2)\n with col1:\n new_sub = st.text_input(\"Subreddit/User name\", placeholder=\"e.g. python\")\n is_user = st.checkbox(\"Is a User (not subreddit)\")\n \n with col2:\n limit = st.number_input(\"Post Limit\", min_value=10, max_value=5000, value=100)\n mode = st.selectbox(\"Mode\", ['full', 'history'])\n \n no_media = st.checkbox(\"Skip media download\")\n no_comments = st.checkbox(\"Skip comments\")\n \n if st.button(\"๐ Start Scraping\"):\n if not new_sub:\n st.error(\"Please enter a subreddit/user name!\")\n else:\n target_cmd = [\"python\", \"-u\", \"main.py\", new_sub, \"--mode\", mode, \"--limit\", str(limit)]\n if is_user: target_cmd.append(\"--user\")\n if no_media: target_cmd.append(\"--no-media\")\n if no_comments: target_cmd.append(\"--no-comments\")\n \n # Start background process\n import subprocess\n\n job_id = f\"job_{int(time.time())}\"\n log_file = LOG_DIR / f\"{job_id}.log\"\n \n try:\n with open(log_file, \"w\", encoding=\"utf-8\") as f:\n env = os.environ.copy()\n env['PYTHONIOENCODING'] = 'utf-8'\n env['PYTHONUNBUFFERED'] = '1'\n \n process = subprocess.Popen(\n target_cmd,\n stdout=f,\n stderr=subprocess.STDOUT,\n cwd=str(Path(__file__).parent.parent),\n env=env\n )\n \n # Save job state\n job_info = {\n \"job_id\": job_id,\n \"pid\": process.pid,\n \"target\": new_sub,\n \"log_file\": str(log_file.absolute()),\n \"start_time\": time.time()\n }\n \n with open(JOB_FILE, \"w\") as f:\n json.dump(job_info, f)\n \n st.success(f\"Started job {job_id}!\")\n st.rerun()\n \n except Exception as e:\n st.error(f\"Failed to start: {e}\")\n \n st.divider()\n \n if selected_sub:\n # Export options\n st.subheader(\"๐ค Export Data\")\n \n export_format = st.selectbox(\"Format\", ['CSV', 'JSON', 'Excel'])\n \n if st.button(\"๐ฅ Download Posts\"):\n if export_format == 'CSV':\n csv = posts_df.to_csv(index=False)\n st.download_button(\n \"Download CSV\",\n csv,\n f\"{selected_sub}_posts.csv\",\n \"text/csv\"\n )\n elif export_format == 'JSON':\n json_data = posts_df.to_json(orient='records', indent=2)\n st.download_button(\n \"Download JSON\",\n json_data,\n f\"{selected_sub}_posts.json\",\n \"application/json\"\n )\n \n st.divider()\n \n # Media Export\n st.subheader(\"๐ผ๏ธ Media Export\")\n \n media_dir = Path(f\"data/{selected_sub}/media\")\n if media_dir.exists():\n images_dir = media_dir / \"images\"\n videos_dir = media_dir / \"videos\"\n \n images = list(images_dir.glob(\"*\")) if images_dir.exists() else []\n videos = list(videos_dir.glob(\"*\")) if videos_dir.exists() else []\n \n col1, col2, col3 = st.columns(3)\n with col1:\n st.metric(\"๐ท Images\", len(images))\n with col2:\n st.metric(\"๐ฌ Videos\", len(videos))\n with col3:\n total_size = sum(f.stat().st_size for f in images + videos) / (1024 * 1024)\n st.metric(\"๐พ Total Size\", f\"{total_size:.1f} MB\")\n \n if images or videos:\n if st.button(\"๐ฆ Download All Media (ZIP)\"):\n import zipfile\n import io\n \n zip_buffer = io.BytesIO()\n with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zf:\n for img in images:\n zf.write(img, f\"images/{img.name}\")\n for vid in videos:\n zf.write(vid, f\"videos/{vid.name}\")\n \n st.download_button(\n \"๐พ Download ZIP\",\n zip_buffer.getvalue(),\n f\"{selected_sub}_media.zip\",\n \"application/zip\"\n )\n st.success(f\"โ
ZIP ready: {len(images)} images, {len(videos)} videos\")\n \n # Preview recent images\n if images:\n st.write(\"**Recent Images:**\")\n preview_cols = st.columns(min(5, len(images)))\n for i, img in enumerate(images[:5]):\n with preview_cols[i]:\n try:\n st.image(str(img), width=100)\n except:\n st.text(img.name[:15])\n else:\n st.info(f\"No media found for {selected_sub}. Run with `--mode full` to download media.\")\n \n with tab_map[\"๐ Job History\"]:\n st.header(\"๐ Job History\")\n \n try:\n from export.database import get_job_history, get_job_stats\n \n # Job stats\n stats = get_job_stats()\n \n col1, col2, col3, col4 = st.columns(4)\n with col1:\n st.metric(\"Total Jobs\", stats.get('total_jobs', 0))\n with col2:\n st.metric(\"Completed\", stats.get('completed', 0))\n with col3:\n st.metric(\"Failed\", stats.get('failed', 0))\n with col4:\n avg_dur = stats.get('avg_duration')\n st.metric(\"Avg Duration\", f\"{avg_dur:.1f}s\" if avg_dur else \"-\")\n \n st.divider()\n \n # Job history table\n st.subheader(\"Recent Jobs\")\n \n col1, col2 = st.columns(2)\n with col1:\n filter_status = st.selectbox(\"Filter by Status\", ['All', 'completed', 'failed', 'running'])\n with col2:\n limit = st.number_input(\"Show last N jobs\", min_value=10, max_value=100, value=20)\n \n status_filter = None if filter_status == 'All' else filter_status\n jobs = get_job_history(limit=limit, status=status_filter)\n \n if jobs:\n jobs_df = pd.DataFrame(jobs)\n # Format for display\n display_cols = ['job_id', 'target', 'mode', 'status', 'posts_scraped', \n 'comments_scraped', 'duration_seconds', 'started_at', 'dry_run']\n display_cols = [c for c in display_cols if c in jobs_df.columns]\n st.dataframe(jobs_df[display_cols])\n \n # Success rate chart\n st.subheader(\"Success Rate\")\n if 'status' in jobs_df.columns:\n status_counts = jobs_df['status'].value_counts()\n st.bar_chart(status_counts)\n else:\n st.info(\"No job history found. Run some scrapes first!\")\n \n except Exception as e:\n st.error(f\"Failed to load job history: {e}\")\n st.info(\"Make sure the database is initialized.\")\n \n with tab_map[\"๐ Integrations\"]:\n st.header(\"๐ Integrations & Settings\")\n \n # REST API Section\n st.subheader(\"๐ REST API\")\n \n col1, col2, col3 = st.columns(3)\n with col1:\n api_port = st.number_input(\"API Port\", value=8000, min_value=1000, max_value=65535)\n \n with col2:\n if st.button(\"๐ Start API Server\"):\n st.info(\"Starting API server in background...\")\n import subprocess\n try:\n # Start API in background (non-blocking)\n subprocess.Popen(\n [\"python\", \"main.py\", \"--api\"],\n cwd=str(Path(__file__).parent.parent),\n creationflags=subprocess.CREATE_NEW_CONSOLE if hasattr(subprocess, 'CREATE_NEW_CONSOLE') else 0\n )\n st.success(f\"โ
API server starting on port {api_port}!\")\n st.markdown(f\"**Open:** [http://localhost:{api_port}/docs](http://localhost:{api_port}/docs)\")\n except Exception as e:\n st.error(f\"โ Failed to start API: {e}\")\n \n with col3:\n # Check if API is running\n import requests\n try:\n resp = requests.get(f\"http://localhost:{api_port}/health\", timeout=1)\n if resp.status_code == 200:\n st.success(\"๐ข API is running\")\n else:\n st.warning(\"๐ก API responded but not healthy\")\n except:\n st.info(\"๐ด API not running\")\n \n st.markdown(\"\"\"\n **Available Endpoints:**\n | Endpoint | Description |\n |----------|-------------|\n | `/posts` | List posts with filters |\n | `/comments` | List comments |\n | `/subreddits` | All scraped subreddits |\n | `/jobs` | Job history |\n | `/query?sql=...` | Raw SQL queries |\n | `/docs` | Interactive Swagger UI |\n \"\"\")\n \n st.divider()\n \n # External Tools\n st.subheader(\"๐ External Tools Integration\")\n \n tool_tabs = st.tabs([\"๐ Metabase\", \"๐ Grafana\", \"๐ DreamFactory\", \"๐งฆ DuckDB\"])\n \n with tool_tabs[0]:\n st.markdown(\"\"\"\n **Metabase Setup:**\n 1. Start API: `python main.py --api`\n 2. In Metabase: New Question โ Native Query\n 3. Use HTTP datasource with `http://localhost:8000`\n 4. Query: `/posts?subreddit=python&limit=100`\n \n **Or use raw SQL:**\n ```\n /query?sql=SELECT title, score FROM posts ORDER BY score DESC\n ```\n \"\"\")\n \n with tool_tabs[1]:\n st.markdown(\"\"\"\n **Grafana Setup:**\n 1. Install \"JSON API\" or \"Infinity\" plugin\n 2. Add datasource: `http://localhost:8000`\n 3. Use `/grafana/query` for time-series\n \n **Example Panel Query:**\n ```sql\n SELECT date(created_utc) as time, COUNT(*) as posts \n FROM posts GROUP BY date(created_utc)\n ```\n \"\"\")\n \n with tool_tabs[2]:\n st.markdown(\"\"\"\n **DreamFactory Setup:**\n 1. Point to SQLite file: `data/reddit_scraper.db`\n 2. Or use REST API: `http://localhost:8000`\n 3. Auto-generates API for all tables\n \"\"\")\n \n with tool_tabs[3]:\n st.markdown(\"\"\"\n **DuckDB (Analytics):**\n 1. Export to Parquet first (see below)\n 2. Query directly:\n ```python\n import duckdb\n duckdb.query(\"SELECT * FROM 'data/parquet/*.parquet'\").df()\n ```\n \"\"\")\n \n st.divider()\n \n # Parquet Export\n st.subheader(\"๐ฆ Parquet Export\")\n \n all_targets = available_data['subreddits'] + available_data['users']\n \n col1, col2 = st.columns(2)\n with col1:\n export_sub = st.selectbox(\"Select target to export\", all_targets, key=\"parquet_export\")\n with col2:\n if st.button(\"๐ฆ Export to Parquet\"):\n if export_sub:\n target_name = export_sub.replace('r_', '').replace('u_', '')\n with st.spinner(f\"Exporting {target_name} to Parquet...\"):\n import subprocess\n result = subprocess.run(\n [\"python\", \"main.py\", \"--export-parquet\", target_name],\n capture_output=True,\n text=True,\n cwd=str(Path(__file__).parent.parent)\n )\n if result.returncode == 0:\n st.success(f\"โ
Exported {target_name} to Parquet!\")\n st.code(result.stdout[-500:] if len(result.stdout) > 500 else result.stdout)\n else:\n st.error(f\"โ Export failed: {result.stderr}\")\n else:\n st.error(\"Select a target first\")\n \n # List existing parquet files\n parquet_dir = Path(\"data/parquet\")\n if parquet_dir.exists():\n parquet_files = list(parquet_dir.glob(\"*.parquet\"))\n if parquet_files:\n st.write(\"**Existing Parquet files:**\")\n for f in parquet_files[:10]:\n size_mb = f.stat().st_size / (1024 * 1024)\n st.text(f\" โข {f.name} ({size_mb:.2f} MB)\")\n \n st.divider()\n \n # Database Maintenance\n st.subheader(\"๐ ๏ธ Database Maintenance\")\n \n col1, col2, col3 = st.columns(3)\n \n with col1:\n if st.button(\"๐พ Backup Database\"):\n with st.spinner(\"Creating backup...\"):\n import subprocess\n result = subprocess.run(\n [\"python\", \"main.py\", \"--backup\"],\n capture_output=True,\n text=True,\n cwd=str(Path(__file__).parent.parent)\n )\n if result.returncode == 0:\n st.success(\"โ
Database backed up!\")\n st.code(result.stdout[-300:] if len(result.stdout) > 300 else result.stdout)\n else:\n st.error(f\"โ Backup failed: {result.stderr}\")\n \n with col2:\n if st.button(\"๐งน Vacuum/Optimize\"):\n with st.spinner(\"Optimizing database...\"):\n import subprocess\n result = subprocess.run(\n [\"python\", \"main.py\", \"--vacuum\"],\n capture_output=True,\n text=True,\n cwd=str(Path(__file__).parent.parent)\n )\n if result.returncode == 0:\n st.success(\"โ
Database optimized!\")\n st.code(result.stdout[-300:] if len(result.stdout) > 300 else result.stdout)\n else:\n st.error(f\"โ Vacuum failed: {result.stderr}\")\n \n with col3:\n try:\n from export.database import get_database_info\n db_info = get_database_info()\n st.metric(\"DB Size\", f\"{db_info.get('size_mb', 0):.2f} MB\")\n except:\n st.metric(\"DB Size\", \"N/A\")\n \n # Show backup files\n backup_dir = Path(\"data/backups\")\n if backup_dir.exists():\n backups = sorted(backup_dir.glob(\"*.db\"), reverse=True)[:5]\n if backups:\n st.write(\"**Recent Backups:**\")\n for b in backups:\n size_mb = b.stat().st_size / (1024 * 1024)\n st.text(f\" โข {b.name} ({size_mb:.2f} MB)\")\n \n st.divider()\n \n # Plugin Configuration\n st.subheader(\"๐ Plugins\")\n \n try:\n from plugins import load_plugins\n plugins = load_plugins()\n \n if plugins:\n st.write(\"**Available Plugins:**\")\n for plugin in plugins:\n status = \"โ
\" if plugin.enabled else \"โ\"\n st.markdown(f\"{status} **{plugin.name}** - {plugin.description}\")\n \n st.info(\"๐ก Enable plugins when scraping: `python main.py --plugins`\")\n else:\n st.warning(\"No plugins found in plugins/ directory\")\n except Exception as e:\n st.error(f\"Plugin loading error: {e}\")\n \n st.divider()\n \n # Quick Commands Reference\n st.subheader(\"๐ Quick Commands\")\n st.code(\"\"\"\n# Start REST API\npython main.py --api\n\n# Export to Parquet\npython main.py --export-parquet \n\n# Backup database\npython main.py --backup\n\n# Scrape with plugins\npython main.py --plugins\n\n# Dry run (test without saving)\npython main.py --dry-run\n \"\"\", language=\"bash\")\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "docker-compose.yml",
"content": "version: '3.8'\n\n# Reddit Scraper Suite - Full Stack\n# Start with: docker-compose up -d\n\nservices:\n # Main Scraper (run scrape jobs)\n scraper:\n build: .\n volumes:\n - ./data:/app/data # Persist scraped data\n command: [\"--help\"] # Override with your scrape command\n profiles: [\"scrape\"] # Only run when explicitly requested\n\n # REST API Server (for Metabase/Grafana integration)\n api:\n build: .\n ports:\n - \"8000:8000\"\n volumes:\n - ./data:/app/data\n command: [\"--api\"]\n restart: unless-stopped\n healthcheck:\n test: [\"CMD\", \"curl\", \"-f\", \"http://localhost:8000/health\"]\n interval: 30s\n timeout: 10s\n retries: 3\n\n # Streamlit Dashboard\n dashboard:\n build: .\n ports:\n - \"8501:8501\"\n volumes:\n - ./data:/app/data\n command: [\"--dashboard\"]\n restart: unless-stopped\n depends_on:\n - api\n\n # Scheduled Scraper (optional - uncomment and configure)\n # scheduler:\n # build: .\n # volumes:\n # - ./data:/app/data\n # command: [\"--schedule\", \"python\", \"--every\", \"60\"]\n # restart: unless-stopped\n\n# Optional: Add Metabase for data visualization\n# Uncomment to enable\n#\n# metabase:\n# image: metabase/metabase:latest\n# ports:\n# - \"3000:3000\"\n# environment:\n# MB_DB_TYPE: h2\n# volumes:\n# - metabase-data:/metabase-data\n# depends_on:\n# - api\n\n# ===========================================\n# PRODUCTION DEPLOYMENT (AWS/VPS)\n# ===========================================\n# Uncomment the nginx service below for:\n# - HTTPS/SSL termination\n# - Basic authentication\n# - Single port exposure (80/443)\n\n# nginx:\n# image: nginx:alpine\n# ports:\n# - \"80:80\"\n# - \"443:443\"\n# volumes:\n# - ./nginx.conf:/etc/nginx/nginx.conf:ro\n# - ./ssl:/etc/nginx/ssl:ro # Add your SSL certs\n# depends_on:\n# - api\n# - dashboard\n\n# volumes:\n# metabase-data:\n\n# ===========================================\n# QUICK DEPLOY TO AWS/VPS:\n# ===========================================\n# 1. SSH into your server\n# 2. git clone \n# 3. docker-compose up -d\n# 4. Open firewall: ports 8000, 8501\n#\n# Access:\n# http://:8501 (Dashboard)\n# http://:8000 (API)\n# ===========================================\n"
},
{
"path": "docs/BLOG.md",
"content": "# Building the Ultimate Reddit Scraper: A Full-Featured, API-Free Data Collection Suite\n\n\n\n\n\n**December 2024** | By Sanjeev Kumar\n\n---\n\n## TL;DR\n\nI built a **complete Reddit scraper suite** that requires **zero API keys**. It comes with a beautiful Streamlit dashboard, REST API for integration with tools like Grafana and Metabase, plugin system for post-processing, scheduled scraping, notifications, and much more. Best of allโit's completely open source.\n\n๐ **GitHub**: [reddit-universal-scraper](https://github.com/ksanjeev284/reddit-universal-scraper)\n\n---\n\n## The Problem\n\nIf you've ever tried to scrape Reddit data for analysis, research, or just personal projects, you know the pain:\n\n1. **Reddit's API is heavily rate-limited** (especially after the 2023 API changes)\n2. **API keys require approval** and are increasingly restricted\n3. **Existing scrapers are often single-purpose** - scrape posts OR comments, not both\n4. **No easy way to visualize or analyze the data** after scraping\n5. **Running scrapes manually is tedious** - you want automation\n\nI decided to solve all of these problems at once.\n\n---\n\n## The Solution: Universal Reddit Scraper Suite\n\nAfter weeks of development, I created a full-featured scraper that:\n\n| Feature | What It Does |\n|---------|--------------|\n| ๐ **Full Scraping** | Posts, comments, images, videos, galleriesโeverything |\n| ๐ซ **No API Keys** | Uses Reddit's public JSON endpoints and mirrors |\n| ๐ **Web Dashboard** | Beautiful 7-tab Streamlit UI for analysis |\n| ๐ **REST API** | Connect Metabase, Grafana, DuckDB, and more |\n| ๐ **Plugin System** | Extensible post-processing (sentiment analysis, deduplication, keywords) |\n| ๐
**Scheduled Scraping** | Cron-style automation |\n| ๐ง **Notifications** | Discord & Telegram alerts when scrapes complete |\n| ๐ณ **Docker Ready** | One command to deploy anywhere |\n\n---\n\n## Architecture Deep Dive\n\n### How It Works Without API Keys\n\nThe secret sauce is in the approach. Instead of using Reddit's official (and restricted) API, I leverage:\n\n1. **Reddit's public JSON endpoints**: Every Reddit page has a `.json` suffix that returns structured data\n2. **Multiple mirror fallbacks**: When one source is rate-limited, the scraper automatically rotates through alternatives like Redlib instances\n3. **Smart rate limiting**: Built-in delays and cool-down periods to stay under the radar\n\n```python\nMIRRORS = [\n \"https://old.reddit.com\",\n \"https://redlib.catsarch.com\",\n \"https://redlib.vsls.cz\",\n \"https://r.nf\",\n \"https://libreddit.northboot.xyz\",\n \"https://redlib.tux.pizza\"\n]\n```\n\nWhen one source fails, it automatically tries the next. No manual intervention needed.\n\n### The Core Scraping Engine\n\nThe scraper operates in three modes:\n\n**1. Full Mode** - The complete package\n```bash\npython main.py python --mode full --limit 100\n```\nThis scrapes posts, downloads all media (images, videos, galleries), and fetches comments with their full thread hierarchy.\n\n**2. History Mode** - Fast metadata-only\n```bash\npython main.py python --mode history --limit 500\n```\nPerfect for quickly building a dataset of post metadata without the overhead of media downloads.\n\n**3. Monitor Mode** - Live watching\n```bash\npython main.py python --mode monitor\n```\nContinuously checks for new posts every 5 minutes. Ideal for tracking breaking news or trending discussions.\n\n---\n\n## The Dashboard Experience\n\nOne of the standout features is the **7-tab Streamlit dashboard** that makes data exploration a joy:\n\n### ๐ Overview Tab\nAt a glance, see:\n- Total posts and comments\n- Cumulative score across all posts\n- Media post breakdown\n- Posts-over-time chart\n- Top 10 posts by score\n\n### ๐ Analytics Tab\nThis is where it gets interesting:\n- **Sentiment Analysis**: Run VADER-based sentiment scoring on your entire dataset\n- **Keyword Cloud**: See the most frequently used terms\n- **Best Posting Times**: Data-driven insights on when posts get the most engagement\n\n### ๐ Search Tab\nFull-text search across all scraped data with filters for:\n- Minimum score\n- Post type (text, image, video, gallery, link)\n- Author\n- Custom sorting\n\n### ๐ฌ Comments Analysis\n- View top-scoring comments\n- See who the most active commenters are\n- Track comment patterns over time\n\n### โ๏ธ Scraper Controls\nStart new scrapes right from the dashboard! Configure:\n- Target subreddit/user\n- Post limits\n- Mode (full/history)\n- Media and comment toggles\n\n### ๐ Job History\nFull observability into every scrape job:\n- Status tracking (running, completed, failed)\n- Duration metrics\n- Post/comment/media counts\n- Error logging\n\n### ๐ Integrations\nPre-configured instructions for connecting:\n- Metabase\n- Grafana\n- DreamFactory\n- DuckDB\n\n---\n\n## The Plugin Architecture\n\nI designed a plugin system to allow extensible post-processing. The architecture is simple but powerful:\n\n```python\nclass Plugin:\n \"\"\"Base class for all plugins.\"\"\"\n name = \"base\"\n description = \"Base plugin\"\n enabled = True\n \n def process_posts(self, posts):\n return posts\n \n def process_comments(self, comments):\n return comments\n```\n\n### Built-in Plugins\n\n**1. Sentiment Tagger**\nAnalyzes the emotional tone of every post and comment using VADER sentiment analysis:\n\n```python\nclass SentimentTagger(Plugin):\n name = \"sentiment_tagger\"\n description = \"Adds sentiment scores and labels to posts\"\n \n def process_posts(self, posts):\n for post in posts:\n text = f\"{post.get('title', '')} {post.get('selftext', '')}\"\n score, label = analyze_sentiment(text)\n post['sentiment_score'] = score\n post['sentiment_label'] = label\n return posts\n```\n\n**2. Deduplicator**\nRemoves duplicate posts that may appear across multiple scraping sessions.\n\n**3. Keyword Extractor**\nPulls out the most significant terms from your scraped content for trend analysis.\n\n### Creating Your Own Plugin\n\nDrop a new Python file in the `plugins/` directory:\n\n```python\nfrom plugins import Plugin\n\nclass MyCustomPlugin(Plugin):\n name = \"my_plugin\"\n description = \"Does something cool\"\n enabled = True\n \n def process_posts(self, posts):\n # Your logic here\n return posts\n```\n\nEnable plugins during scraping:\n```bash\npython main.py python --mode full --plugins\n```\n\n---\n\n## REST API for External Integrations\n\nThe REST API opens up the scraper to a whole ecosystem of tools:\n\n```bash\npython main.py --api\n# API at http://localhost:8000\n# Docs at http://localhost:8000/docs\n```\n\n### Key Endpoints\n\n| Endpoint | Description |\n|----------|-------------|\n| `GET /posts` | List posts with filters (subreddit, limit, offset) |\n| `GET /comments` | List comments |\n| `GET /subreddits` | All scraped subreddits |\n| `GET /jobs` | Job history |\n| `GET /query?sql=...` | Raw SQL queries for power users |\n| `GET /grafana/query` | Grafana-compatible time-series data |\n\n### Real-World Integration: Grafana Dashboard\n\n1. Install the \"JSON API\" or \"Infinity\" plugin in Grafana\n2. Add datasource pointing to `http://localhost:8000`\n3. Use the `/grafana/query` endpoint for time-series panels\n\n```sql\nSELECT date(created_utc) as time, COUNT(*) as posts \nFROM posts GROUP BY date(created_utc)\n```\n\nNow you have a real-time dashboard tracking Reddit activity!\n\n---\n\n## Scheduled Scraping & Notifications\n\n### Automation Made Easy\n\nSet up recurring scrapes with cron-style scheduling:\n\n```bash\n# Scrape every 60 minutes\npython main.py --schedule delhi --every 60\n\n# With custom options\npython main.py --schedule delhi --every 30 --mode full --limit 50\n```\n\n### Get Notified\n\nConfigure Discord or Telegram alerts when scrapes complete:\n\n```bash\n# Environment variables\nexport DISCORD_WEBHOOK_URL=\"https://discord.com/api/webhooks/...\"\nexport TELEGRAM_BOT_TOKEN=\"123456:ABC...\"\nexport TELEGRAM_CHAT_ID=\"987654321\"\n```\n\nNow you get notified with scrape summaries directly in your preferred platform.\n\n---\n\n## Dry Run Mode: Test Before You Commit\n\nOne of my favorite features is **dry run mode**. It simulates the entire scrape without saving any data:\n\n```bash\npython main.py python --mode full --limit 50 --dry-run\n```\n\nOutput:\n```\n๐งช DRY RUN MODE - No data will be saved\n๐งช DRY RUN COMPLETE!\n ๐ Would scrape: 100 posts\n ๐ฌ Would scrape: 245 comments\n```\n\nPerfect for:\n- Testing your scrape configuration\n- Estimating data volume before committing\n- Debugging without cluttering your dataset\n\n---\n\n## Docker Deployment\n\n### Quick Start\n\n```bash\n# Build\ndocker build -t reddit-scraper .\n\n# Run a scrape\ndocker run -v ./data:/app/data reddit-scraper python --limit 100\n\n# Run with plugins\ndocker run -v ./data:/app/data reddit-scraper python --plugins\n```\n\n### Full Stack with Docker Compose\n\n```bash\ndocker-compose up -d\n```\n\nThis spins up:\n- Dashboard at `http://localhost:8501`\n- REST API at `http://localhost:8000`\n\n### Deploy to Any VPS\n\n```bash\nssh user@your-server-ip\ngit clone https://github.com/ksanjeev284/reddit-universal-scraper.git\ncd reddit-universal-scraper\ndocker-compose up -d\n```\n\nOpen the firewall:\n```bash\nsudo ufw allow 8000\nsudo ufw allow 8501\n```\n\nYou now have a production-ready Reddit scraping platform!\n\n---\n\n## Data Export Options\n\n### CSV (Default)\nAll scraped data is saved as CSV files:\n- `data/r_/posts.csv`\n- `data/r_/comments.csv`\n\n### Parquet (Analytics-Optimized)\nExport to columnar format for analytics tools:\n\n```bash\npython main.py --export-parquet python\n```\n\nQuery directly with DuckDB:\n```python\nimport duckdb\nduckdb.query(\"SELECT * FROM 'data/parquet/*.parquet'\").df()\n```\n\n### Database Maintenance\n\n```bash\n# Backup\npython main.py --backup\n\n# Optimize/vacuum\npython main.py --vacuum\n\n# View job history\npython main.py --job-history\n```\n\n---\n\n## Data Schema\n\n### Posts Table\n\n| Column | Description |\n|--------|-------------|\n| `id` | Reddit post ID |\n| `title` | Post title |\n| `author` | Username |\n| `score` | Net upvotes |\n| `num_comments` | Comment count |\n| `post_type` | text/image/video/gallery/link |\n| `selftext` | Post body (for text posts) |\n| `created_utc` | Timestamp |\n| `permalink` | Reddit URL |\n| `is_nsfw` | NSFW flag |\n| `flair` | Post flair |\n| `sentiment_score` | -1.0 to 1.0 (with plugins) |\n\n### Comments Table\n\n| Column | Description |\n|--------|-------------|\n| `comment_id` | Comment ID |\n| `post_permalink` | Parent post URL |\n| `author` | Username |\n| `body` | Comment text |\n| `score` | Upvotes |\n| `depth` | Nesting level |\n| `is_submitter` | Whether author is OP |\n\n---\n\n## Use Cases\n\n### 1. Academic Research\n- Analyze subreddit community dynamics\n- Track sentiment over time during events\n- Study user engagement patterns\n\n### 2. Market Research\n- Monitor brand mentions\n- Track product feedback\n- Identify emerging trends\n\n### 3. Content Creation\n- Find popular topics in your niche\n- Analyze what makes posts go viral\n- Discover optimal posting times\n\n### 4. Data Journalism\n- Archive discussions around breaking news\n- Analyze public sentiment during events\n- Track narrative evolution\n\n### 5. Personal Projects\n- Build a dataset for ML training\n- Create Reddit-based recommendation systems\n- Archive communities you care about\n\n---\n\n## Performance Considerations\n\n### Respect Reddit's Servers\nThe scraper includes built-in delays:\n- **3 second cooldown** between API requests\n- **30 second wait** if all mirrors fail\n- **Automatic mirror rotation** to distribute load\n\n### Optimize Your Scrapes\n- Use `--mode history` for faster metadata-only scrapes\n- Use `--no-media` if you don't need images/videos\n- Use `--no-comments` for post-only data\n\n### Handle Large Datasets\n- Parquet export for analytics queries\n- SQLite database for structured storage\n- Automatic deduplication to avoid bloat\n\n---\n\n## What's Next? Roadmap\n\nI'm actively developing new features:\n\n- [ ] **Async scraping** for even faster data collection\n- [ ] **Multi-subreddit monitoring** in a single command\n- [ ] **Email notifications** in addition to Discord/Telegram\n- [ ] **Cloud deployment templates** (AWS, GCP, Azure)\n- [ ] **Web-based scraper configuration** (no CLI needed)\n\n---\n\n## Getting Started\n\n### Prerequisites\n- Python 3.10+\n- pip\n\n### Installation\n\n```bash\n# Clone the repo\ngit clone https://github.com/ksanjeev284/reddit-universal-scraper.git\ncd reddit-universal-scraper\n\n# Install dependencies\npip install -r requirements.txt\n\n# Your first scrape\npython main.py python --mode full --limit 50\n\n# Launch the dashboard\npython main.py --dashboard\n```\n\nThat's it! You're now scraping Reddit like a pro.\n\n---\n\n## Contributing\n\nThis is an open-source project and contributions are welcome! Whether it's:\n- Bug fixes\n- New plugins\n- Documentation improvements\n- Feature suggestions\n\nOpen an issue or submit a PR on [GitHub](https://github.com/ksanjeev284/reddit-universal-scraper).\n\n---\n\n## Conclusion\n\nThe Universal Reddit Scraper Suite represents months of work solving a problem that many data enthusiasts face. By combining a robust scraping engine with analytics capabilities, a beautiful dashboard, and extensive integration optionsโall without requiring API keysโI hope this tool empowers you to unlock insights from Reddit's vast treasure trove of community discussions.\n\n**Happy scraping!** ๐ค\n\n---\n\n*If you found this useful, consider giving the project a โญ on [GitHub](https://github.com/ksanjeev284/reddit-universal-scraper)!*\n\n---\n\n## Connect\n\n- **GitHub**: [@ksanjeev284](https://github.com/ksanjeev284)\n- **Project**: [reddit-universal-scraper](https://github.com/ksanjeev284/reddit-universal-scraper)\n\n---\n\n*Tags: Reddit, Web Scraping, Python, Data Analysis, Streamlit, REST API, Docker, Open Source*\n"
},
{
"path": "docs/INTEGRATION.md",
"content": "# External Tools Integration Guide\n\nConnect Metabase, Grafana, DreamFactory, or any REST client to your Reddit scraper data.\n\n---\n\n## Quick Start\n\n```powershell\n# Install dependencies\npip install fastapi uvicorn\n\n# Start the API server\npython main.py --api\n```\n\nThe API will be available at `http://localhost:8000`\n\n---\n\n## API Endpoints\n\n| Endpoint | Description |\n|----------|-------------|\n| `GET /posts` | List posts with filters (q, subreddit, author, min_score) |\n| `GET /posts/{id}` | Get single post |\n| `GET /comments` | List comments with filters |\n| `GET /subreddits` | List all scraped subreddits |\n| `GET /subreddits/{name}/stats` | Get subreddit statistics |\n| `GET /jobs` | View job history |\n| `GET /jobs/stats` | Job statistics |\n| `GET /query?sql=...` | Raw SQL SELECT queries |\n| `GET /docs` | Interactive API documentation |\n\n---\n\n## Metabase Setup\n\n1. Start API: `python main.py --api`\n2. In Metabase, add a new \"HTTP\" question\n3. Use `http://localhost:8000/posts?limit=1000` \n4. Or use `/query?sql=SELECT * FROM posts` for custom queries\n\n---\n\n## Grafana Setup\n\n1. Install \"JSON API\" or \"Infinity\" datasource plugin\n2. Add datasource with URL: `http://localhost:8000`\n3. Use `/grafana/query` for time-series data\n4. Or use `/query?sql=...` for custom queries\n\nExample Grafana query:\n```sql\nSELECT date(created_utc) as time, COUNT(*) as posts \nFROM posts \nGROUP BY date(created_utc)\n```\n\n---\n\n## DreamFactory / REST Clients\n\nThe API includes full CORS support. Connect any tool that speaks REST:\n\n```bash\n# Get posts\ncurl http://localhost:8000/posts?subreddit=python&limit=10\n\n# Custom SQL query\ncurl \"http://localhost:8000/query?sql=SELECT title, score FROM posts ORDER BY score DESC LIMIT 5\"\n```\n\n---\n\n## Docker Compose (All-in-One)\n\n```yaml\nversion: '3'\nservices:\n scraper-api:\n build: .\n ports:\n - \"8000:8000\"\n volumes:\n - ./data:/app/data\n command: python main.py --api\n\n metabase:\n image: metabase/metabase\n ports:\n - \"3000:3000\"\n```\n"
},
{
"path": "export/__init__.py",
"content": "# Export module\nfrom .database import *\n"
},
{
"path": "export/cloud.py",
"content": "\"\"\"\nCloud Upload Module - S3 and Google Drive integration\n\"\"\"\nimport os\nimport json\nfrom pathlib import Path\nfrom datetime import datetime\n\n# Try importing boto3 for S3\ntry:\n import boto3\n from botocore.exceptions import ClientError\n HAS_BOTO3 = True\nexcept ImportError:\n HAS_BOTO3 = False\n\n# Try importing Google Drive API\ntry:\n from google.oauth2.credentials import Credentials\n from googleapiclient.discovery import build\n from googleapiclient.http import MediaFileUpload\n HAS_GDRIVE = True\nexcept ImportError:\n HAS_GDRIVE = False\n\n\nclass S3Uploader:\n \"\"\"Upload scraped data to AWS S3.\"\"\"\n \n def __init__(self, bucket_name, aws_access_key=None, aws_secret_key=None, region='us-east-1'):\n \"\"\"\n Initialize S3 uploader.\n \n Args:\n bucket_name: S3 bucket name\n aws_access_key: Optional, uses env/config if not provided\n aws_secret_key: Optional, uses env/config if not provided\n region: AWS region\n \"\"\"\n if not HAS_BOTO3:\n raise ImportError(\"boto3 not installed. Run: pip install boto3\")\n \n self.bucket_name = bucket_name\n self.region = region\n \n # Use provided credentials or fall back to env vars\n self.s3 = boto3.client(\n 's3',\n aws_access_key_id=aws_access_key or os.getenv('AWS_ACCESS_KEY_ID'),\n aws_secret_access_key=aws_secret_key or os.getenv('AWS_SECRET_ACCESS_KEY'),\n region_name=region\n )\n \n def upload_file(self, local_path, s3_key=None):\n \"\"\"\n Upload a single file to S3.\n \n Args:\n local_path: Local file path\n s3_key: S3 object key (default: same as filename)\n \n Returns:\n S3 URL or None on failure\n \"\"\"\n local_path = Path(local_path)\n \n if not local_path.exists():\n print(f\"โ File not found: {local_path}\")\n return None\n \n s3_key = s3_key or local_path.name\n \n try:\n self.s3.upload_file(str(local_path), self.bucket_name, s3_key)\n url = f\"https://{self.bucket_name}.s3.{self.region}.amazonaws.com/{s3_key}\"\n print(f\"โ
Uploaded: {s3_key}\")\n return url\n except ClientError as e:\n print(f\"โ S3 upload failed: {e}\")\n return None\n \n def upload_directory(self, local_dir, s3_prefix=\"\"):\n \"\"\"\n Upload entire directory to S3.\n \n Args:\n local_dir: Local directory path\n s3_prefix: Prefix for S3 keys\n \n Returns:\n Dictionary of uploaded files\n \"\"\"\n local_dir = Path(local_dir)\n \n if not local_dir.exists():\n print(f\"โ Directory not found: {local_dir}\")\n return {}\n \n uploaded = {}\n \n for file_path in local_dir.rglob('*'):\n if file_path.is_file():\n relative_path = file_path.relative_to(local_dir)\n s3_key = f\"{s3_prefix}/{relative_path}\" if s3_prefix else str(relative_path)\n s3_key = s3_key.replace('\\\\', '/') # Windows path fix\n \n url = self.upload_file(file_path, s3_key)\n if url:\n uploaded[str(relative_path)] = url\n \n print(f\"\\n๐ค Uploaded {len(uploaded)} files to S3\")\n return uploaded\n \n def upload_subreddit_data(self, subreddit, prefix=\"u\"):\n \"\"\"\n Upload all data for a subreddit.\n \n Args:\n subreddit: Subreddit name\n prefix: \"r\" for subreddit, \"u\" for user\n \n Returns:\n Upload results\n \"\"\"\n data_dir = Path(f\"data/{prefix}_{subreddit}\")\n \n if not data_dir.exists():\n print(f\"โ Data not found for {prefix}/{subreddit}\")\n return {}\n \n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n s3_prefix = f\"reddit/{prefix}_{subreddit}/{timestamp}\"\n \n return self.upload_directory(data_dir, s3_prefix)\n \n def list_uploads(self, prefix=\"reddit/\"):\n \"\"\"List all uploaded data in S3.\"\"\"\n try:\n response = self.s3.list_objects_v2(\n Bucket=self.bucket_name,\n Prefix=prefix\n )\n \n objects = response.get('Contents', [])\n \n print(f\"\\n๐ S3 Contents ({self.bucket_name}/{prefix}):\")\n for obj in objects[:50]: # Limit to 50\n size_kb = obj['Size'] / 1024\n print(f\" {obj['Key']} ({size_kb:.1f} KB)\")\n \n if len(objects) > 50:\n print(f\" ... and {len(objects) - 50} more\")\n \n return objects\n except ClientError as e:\n print(f\"โ S3 list failed: {e}\")\n return []\n\n\nclass GDriveUploader:\n \"\"\"Upload scraped data to Google Drive.\"\"\"\n \n def __init__(self, credentials_file='credentials.json', token_file='token.json'):\n \"\"\"\n Initialize Google Drive uploader.\n \n Args:\n credentials_file: Path to OAuth credentials JSON\n token_file: Path to token JSON\n \"\"\"\n if not HAS_GDRIVE:\n raise ImportError(\"Google API client not installed. Run: pip install google-api-python-client google-auth-oauthlib\")\n \n self.credentials_file = credentials_file\n self.token_file = token_file\n self.service = None\n self._authenticate()\n \n def _authenticate(self):\n \"\"\"Authenticate with Google Drive API.\"\"\"\n creds = None\n \n if os.path.exists(self.token_file):\n creds = Credentials.from_authorized_user_file(self.token_file)\n \n if not creds or not creds.valid:\n if creds and creds.expired and creds.refresh_token:\n creds.refresh(Request())\n else:\n from google_auth_oauthlib.flow import InstalledAppFlow\n SCOPES = ['https://www.googleapis.com/auth/drive.file']\n flow = InstalledAppFlow.from_client_secrets_file(self.credentials_file, SCOPES)\n creds = flow.run_local_server(port=0)\n \n with open(self.token_file, 'w') as token:\n token.write(creds.to_json())\n \n self.service = build('drive', 'v3', credentials=creds)\n print(\"โ
Google Drive authenticated\")\n \n def create_folder(self, name, parent_id=None):\n \"\"\"Create a folder in Google Drive.\"\"\"\n metadata = {\n 'name': name,\n 'mimeType': 'application/vnd.google-apps.folder'\n }\n \n if parent_id:\n metadata['parents'] = [parent_id]\n \n folder = self.service.files().create(body=metadata, fields='id').execute()\n return folder.get('id')\n \n def upload_file(self, local_path, folder_id=None):\n \"\"\"Upload a file to Google Drive.\"\"\"\n local_path = Path(local_path)\n \n if not local_path.exists():\n print(f\"โ File not found: {local_path}\")\n return None\n \n metadata = {'name': local_path.name}\n if folder_id:\n metadata['parents'] = [folder_id]\n \n media = MediaFileUpload(str(local_path), resumable=True)\n \n try:\n file = self.service.files().create(\n body=metadata,\n media_body=media,\n fields='id,webViewLink'\n ).execute()\n \n print(f\"โ
Uploaded: {local_path.name}\")\n return file.get('webViewLink')\n except Exception as e:\n print(f\"โ Upload failed: {e}\")\n return None\n \n def upload_subreddit_data(self, subreddit, prefix=\"r\"):\n \"\"\"Upload all data for a subreddit.\"\"\"\n data_dir = Path(f\"data/{prefix}_{subreddit}\")\n \n if not data_dir.exists():\n print(f\"โ Data not found for {prefix}/{subreddit}\")\n return {}\n \n # Create folder structure\n root_folder = self.create_folder(f\"reddit_{prefix}_{subreddit}_{datetime.now().strftime('%Y%m%d')}\")\n \n uploaded = {}\n \n for file_path in data_dir.rglob('*'):\n if file_path.is_file():\n url = self.upload_file(file_path, root_folder)\n if url:\n uploaded[str(file_path.name)] = url\n \n print(f\"\\n๐ค Uploaded {len(uploaded)} files to Google Drive\")\n return uploaded\n\n\ndef upload_to_s3(subreddit, bucket_name, prefix=\"r\"):\n \"\"\"\n Convenience function to upload subreddit data to S3.\n \n Args:\n subreddit: Subreddit name\n bucket_name: S3 bucket name\n prefix: \"r\" or \"u\"\n \n Returns:\n Upload results\n \"\"\"\n uploader = S3Uploader(bucket_name)\n return uploader.upload_subreddit_data(subreddit, prefix)\n\n\ndef upload_to_gdrive(subreddit, prefix=\"r\"):\n \"\"\"\n Convenience function to upload subreddit data to Google Drive.\n \n Args:\n subreddit: Subreddit name\n prefix: \"r\" or \"u\"\n \n Returns:\n Upload results\n \"\"\"\n uploader = GDriveUploader()\n return uploader.upload_subreddit_data(subreddit, prefix)\n\n\n# CLI for testing\nif __name__ == \"__main__\":\n import argparse\n \n parser = argparse.ArgumentParser(description=\"Cloud Upload\")\n parser.add_argument(\"subreddit\", help=\"Subreddit to upload\")\n parser.add_argument(\"--s3-bucket\", help=\"S3 bucket name\")\n parser.add_argument(\"--gdrive\", action=\"store_true\", help=\"Upload to Google Drive\")\n parser.add_argument(\"--user\", action=\"store_true\", help=\"Is a user profile\")\n \n args = parser.parse_args()\n prefix = \"u\" if args.user else \"r\"\n \n if args.s3_bucket:\n upload_to_s3(args.subreddit, args.s3_bucket, prefix)\n elif args.gdrive:\n upload_to_gdrive(args.subreddit, prefix)\n else:\n print(\"Please specify --s3-bucket or --gdrive\")\n"
},
{
"path": "export/database.py",
"content": "\"\"\"\nDatabase module - SQLite storage for scraped data\n\"\"\"\nimport sqlite3\nfrom pathlib import Path\nfrom datetime import datetime\nimport json\nimport sys\nsys.path.insert(0, str(Path(__file__).parent.parent))\nfrom config import DB_PATH, DATA_DIR\n\ndef get_connection():\n \"\"\"Get database connection.\"\"\"\n DATA_DIR.mkdir(exist_ok=True)\n conn = sqlite3.connect(DB_PATH)\n conn.row_factory = sqlite3.Row\n return conn\n\ndef init_database():\n \"\"\"Initialize database tables.\"\"\"\n conn = get_connection()\n cursor = conn.cursor()\n \n # Posts table\n cursor.execute(\"\"\"\n CREATE TABLE IF NOT EXISTS posts (\n id TEXT PRIMARY KEY,\n subreddit TEXT,\n title TEXT,\n author TEXT,\n created_utc TEXT,\n permalink TEXT UNIQUE,\n url TEXT,\n score INTEGER DEFAULT 0,\n upvote_ratio REAL DEFAULT 0,\n num_comments INTEGER DEFAULT 0,\n num_crossposts INTEGER DEFAULT 0,\n selftext TEXT,\n post_type TEXT,\n is_nsfw BOOLEAN DEFAULT 0,\n is_spoiler BOOLEAN DEFAULT 0,\n flair TEXT,\n total_awards INTEGER DEFAULT 0,\n has_media BOOLEAN DEFAULT 0,\n media_downloaded BOOLEAN DEFAULT 0,\n source TEXT,\n scraped_at TEXT DEFAULT CURRENT_TIMESTAMP,\n sentiment_score REAL,\n sentiment_label TEXT\n )\n \"\"\")\n \n # Comments table\n cursor.execute(\"\"\"\n CREATE TABLE IF NOT EXISTS comments (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n comment_id TEXT UNIQUE,\n post_id TEXT,\n post_permalink TEXT,\n parent_id TEXT,\n author TEXT,\n body TEXT,\n score INTEGER DEFAULT 0,\n created_utc TEXT,\n depth INTEGER DEFAULT 0,\n is_submitter BOOLEAN DEFAULT 0,\n scraped_at TEXT DEFAULT CURRENT_TIMESTAMP,\n sentiment_score REAL,\n sentiment_label TEXT,\n FOREIGN KEY (post_id) REFERENCES posts(id)\n )\n \"\"\")\n \n # Subreddits table (for tracking)\n cursor.execute(\"\"\"\n CREATE TABLE IF NOT EXISTS subreddits (\n name TEXT PRIMARY KEY,\n last_scraped TEXT,\n total_posts INTEGER DEFAULT 0,\n total_comments INTEGER DEFAULT 0,\n total_media INTEGER DEFAULT 0\n )\n \"\"\")\n \n # Scheduled jobs table\n cursor.execute(\"\"\"\n CREATE TABLE IF NOT EXISTS scheduled_jobs (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n target TEXT,\n is_user BOOLEAN DEFAULT 0,\n mode TEXT DEFAULT 'full',\n limit_posts INTEGER DEFAULT 100,\n cron_expression TEXT,\n last_run TEXT,\n next_run TEXT,\n enabled BOOLEAN DEFAULT 1,\n created_at TEXT DEFAULT CURRENT_TIMESTAMP\n )\n \"\"\")\n \n # Alerts table\n cursor.execute(\"\"\"\n CREATE TABLE IF NOT EXISTS alerts (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n keyword TEXT,\n subreddit TEXT,\n alert_type TEXT DEFAULT 'discord',\n webhook_url TEXT,\n enabled BOOLEAN DEFAULT 1,\n last_triggered TEXT,\n created_at TEXT DEFAULT CURRENT_TIMESTAMP\n )\n \"\"\")\n \n # Job history table for observability\n cursor.execute(\"\"\"\n CREATE TABLE IF NOT EXISTS job_history (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n job_id TEXT UNIQUE,\n target TEXT,\n is_user BOOLEAN DEFAULT 0,\n mode TEXT,\n status TEXT,\n started_at TEXT,\n completed_at TEXT,\n duration_seconds REAL,\n posts_scraped INTEGER DEFAULT 0,\n comments_scraped INTEGER DEFAULT 0,\n media_downloaded INTEGER DEFAULT 0,\n errors TEXT,\n error_count INTEGER DEFAULT 0,\n dry_run BOOLEAN DEFAULT 0\n )\n \"\"\")\n \n # Create indexes\n cursor.execute(\"CREATE INDEX IF NOT EXISTS idx_posts_subreddit ON posts(subreddit)\")\n cursor.execute(\"CREATE INDEX IF NOT EXISTS idx_posts_created ON posts(created_utc)\")\n cursor.execute(\"CREATE INDEX IF NOT EXISTS idx_posts_score ON posts(score)\")\n cursor.execute(\"CREATE INDEX IF NOT EXISTS idx_comments_post ON comments(post_id)\")\n cursor.execute(\"CREATE INDEX IF NOT EXISTS idx_comments_author ON comments(author)\")\n \n conn.commit()\n conn.close()\n print(\"โ
Database initialized\")\n\ndef save_post(post_data, subreddit):\n \"\"\"Save a single post to database.\"\"\"\n conn = get_connection()\n cursor = conn.cursor()\n \n try:\n cursor.execute(\"\"\"\n INSERT OR REPLACE INTO posts \n (id, subreddit, title, author, created_utc, permalink, url, score, \n upvote_ratio, num_comments, num_crossposts, selftext, post_type,\n is_nsfw, is_spoiler, flair, total_awards, has_media, media_downloaded, source)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n \"\"\", (\n post_data.get('id'),\n subreddit,\n post_data.get('title'),\n post_data.get('author'),\n post_data.get('created_utc'),\n post_data.get('permalink'),\n post_data.get('url'),\n post_data.get('score', 0),\n post_data.get('upvote_ratio', 0),\n post_data.get('num_comments', 0),\n post_data.get('num_crossposts', 0),\n post_data.get('selftext', ''),\n post_data.get('post_type'),\n post_data.get('is_nsfw', False),\n post_data.get('is_spoiler', False),\n post_data.get('flair', ''),\n post_data.get('total_awards', 0),\n post_data.get('has_media', False),\n post_data.get('media_downloaded', False),\n post_data.get('source', '')\n ))\n conn.commit()\n return True\n except Exception as e:\n print(f\"DB Error: {e}\")\n return False\n finally:\n conn.close()\n\ndef save_posts_batch(posts, subreddit):\n \"\"\"Save multiple posts efficiently.\"\"\"\n conn = get_connection()\n cursor = conn.cursor()\n saved = 0\n \n for post in posts:\n try:\n cursor.execute(\"\"\"\n INSERT OR IGNORE INTO posts \n (id, subreddit, title, author, created_utc, permalink, url, score, \n upvote_ratio, num_comments, num_crossposts, selftext, post_type,\n is_nsfw, is_spoiler, flair, total_awards, has_media, media_downloaded, source)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n \"\"\", (\n post.get('id'),\n subreddit,\n post.get('title'),\n post.get('author'),\n post.get('created_utc'),\n post.get('permalink'),\n post.get('url'),\n post.get('score', 0),\n post.get('upvote_ratio', 0),\n post.get('num_comments', 0),\n post.get('num_crossposts', 0),\n post.get('selftext', ''),\n post.get('post_type'),\n post.get('is_nsfw', False),\n post.get('is_spoiler', False),\n post.get('flair', ''),\n post.get('total_awards', 0),\n post.get('has_media', False),\n post.get('media_downloaded', False),\n post.get('source', '')\n ))\n if cursor.rowcount > 0:\n saved += 1\n except:\n continue\n \n conn.commit()\n conn.close()\n return saved\n\ndef save_comments_batch(comments, post_id):\n \"\"\"Save multiple comments efficiently.\"\"\"\n conn = get_connection()\n cursor = conn.cursor()\n saved = 0\n \n for comment in comments:\n try:\n cursor.execute(\"\"\"\n INSERT OR IGNORE INTO comments \n (comment_id, post_id, post_permalink, parent_id, author, body, \n score, created_utc, depth, is_submitter)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n \"\"\", (\n comment.get('comment_id'),\n post_id,\n comment.get('post_permalink'),\n comment.get('parent_id'),\n comment.get('author'),\n comment.get('body'),\n comment.get('score', 0),\n comment.get('created_utc'),\n comment.get('depth', 0),\n comment.get('is_submitter', False)\n ))\n if cursor.rowcount > 0:\n saved += 1\n except:\n continue\n \n conn.commit()\n conn.close()\n return saved\n\ndef search_posts(query=None, subreddit=None, author=None, min_score=None, \n start_date=None, end_date=None, post_type=None, limit=100):\n \"\"\"Search posts with filters.\"\"\"\n conn = get_connection()\n cursor = conn.cursor()\n \n sql = \"SELECT * FROM posts WHERE 1=1\"\n params = []\n \n if query:\n sql += \" AND (title LIKE ? OR selftext LIKE ?)\"\n params.extend([f\"%{query}%\", f\"%{query}%\"])\n \n if subreddit:\n sql += \" AND subreddit = ?\"\n params.append(subreddit)\n \n if author:\n sql += \" AND author = ?\"\n params.append(author)\n \n if min_score:\n sql += \" AND score >= ?\"\n params.append(min_score)\n \n if start_date:\n sql += \" AND created_utc >= ?\"\n params.append(start_date)\n \n if end_date:\n sql += \" AND created_utc <= ?\"\n params.append(end_date)\n \n if post_type:\n sql += \" AND post_type = ?\"\n params.append(post_type)\n \n sql += \" ORDER BY created_utc DESC LIMIT ?\"\n params.append(limit)\n \n cursor.execute(sql, params)\n results = [dict(row) for row in cursor.fetchall()]\n conn.close()\n return results\n\ndef search_comments(query=None, post_id=None, author=None, min_score=None, limit=100):\n \"\"\"Search comments with filters.\"\"\"\n conn = get_connection()\n cursor = conn.cursor()\n \n sql = \"SELECT * FROM comments WHERE 1=1\"\n params = []\n \n if query:\n sql += \" AND body LIKE ?\"\n params.append(f\"%{query}%\")\n \n if post_id:\n sql += \" AND post_id = ?\"\n params.append(post_id)\n \n if author:\n sql += \" AND author = ?\"\n params.append(author)\n \n if min_score:\n sql += \" AND score >= ?\"\n params.append(min_score)\n \n sql += \" ORDER BY score DESC LIMIT ?\"\n params.append(limit)\n \n cursor.execute(sql, params)\n results = [dict(row) for row in cursor.fetchall()]\n conn.close()\n return results\n\ndef get_subreddit_stats(subreddit):\n \"\"\"Get statistics for a subreddit.\"\"\"\n conn = get_connection()\n cursor = conn.cursor()\n \n stats = {}\n \n # Post stats\n cursor.execute(\"\"\"\n SELECT \n COUNT(*) as total_posts,\n AVG(score) as avg_score,\n MAX(score) as max_score,\n SUM(num_comments) as total_comments,\n AVG(upvote_ratio) as avg_upvote_ratio\n FROM posts WHERE subreddit = ?\n \"\"\", (subreddit,))\n row = cursor.fetchone()\n if row:\n stats.update(dict(row))\n \n # Post type distribution\n cursor.execute(\"\"\"\n SELECT post_type, COUNT(*) as count \n FROM posts WHERE subreddit = ? \n GROUP BY post_type\n \"\"\", (subreddit,))\n stats['post_types'] = {row['post_type']: row['count'] for row in cursor.fetchall()}\n \n # Top authors\n cursor.execute(\"\"\"\n SELECT author, COUNT(*) as post_count, SUM(score) as total_score\n FROM posts WHERE subreddit = ? AND author != '[deleted]'\n GROUP BY author ORDER BY post_count DESC LIMIT 10\n \"\"\", (subreddit,))\n stats['top_authors'] = [dict(row) for row in cursor.fetchall()]\n \n # Activity by hour\n cursor.execute(\"\"\"\n SELECT strftime('%H', created_utc) as hour, COUNT(*) as count\n FROM posts WHERE subreddit = ?\n GROUP BY hour ORDER BY hour\n \"\"\", (subreddit,))\n stats['hourly_activity'] = {row['hour']: row['count'] for row in cursor.fetchall()}\n \n conn.close()\n return stats\n\ndef get_all_subreddits():\n \"\"\"Get list of all scraped subreddits.\"\"\"\n conn = get_connection()\n cursor = conn.cursor()\n \n cursor.execute(\"\"\"\n SELECT subreddit, COUNT(*) as post_count, \n MAX(created_utc) as latest_post,\n MIN(created_utc) as oldest_post\n FROM posts GROUP BY subreddit ORDER BY post_count DESC\n \"\"\")\n \n results = [dict(row) for row in cursor.fetchall()]\n conn.close()\n return results\n\n# --- JOB HISTORY FUNCTIONS ---\n\ndef start_job_record(target, mode, is_user=False, dry_run=False):\n \"\"\"\n Start tracking a new scrape job.\n \n Returns:\n job_id: Unique identifier for the job\n \"\"\"\n import uuid\n \n conn = get_connection()\n cursor = conn.cursor()\n \n job_id = str(uuid.uuid4())[:8]\n started_at = datetime.now().isoformat()\n \n cursor.execute(\"\"\"\n INSERT INTO job_history (job_id, target, is_user, mode, status, started_at, dry_run)\n VALUES (?, ?, ?, ?, 'running', ?, ?)\n \"\"\", (job_id, target, is_user, mode, started_at, dry_run))\n \n conn.commit()\n conn.close()\n \n print(f\"๐ Job started: {job_id}\")\n return job_id\n\ndef complete_job_record(job_id, status, posts=0, comments=0, media=0, errors=None):\n \"\"\"\n Complete a job record with results.\n \n Args:\n job_id: Job ID from start_job_record\n status: 'completed' or 'failed'\n posts: Number of posts scraped\n comments: Number of comments scraped\n media: Number of media files downloaded\n errors: Error message if failed\n \"\"\"\n conn = get_connection()\n cursor = conn.cursor()\n \n completed_at = datetime.now().isoformat()\n \n # Calculate duration\n cursor.execute(\"SELECT started_at FROM job_history WHERE job_id = ?\", (job_id,))\n row = cursor.fetchone()\n \n duration = 0\n error_count = 0\n if row:\n started = datetime.fromisoformat(row['started_at'])\n duration = (datetime.now() - started).total_seconds()\n \n if errors:\n error_count = 1\n \n cursor.execute(\"\"\"\n UPDATE job_history \n SET status = ?, completed_at = ?, duration_seconds = ?,\n posts_scraped = ?, comments_scraped = ?, media_downloaded = ?,\n errors = ?, error_count = ?\n WHERE job_id = ?\n \"\"\", (status, completed_at, duration, posts, comments, media, errors, error_count, job_id))\n \n conn.commit()\n conn.close()\n \n if status == 'completed':\n print(f\"โ
Job {job_id} completed: {posts} posts, {comments} comments in {duration:.1f}s\")\n else:\n print(f\"โ Job {job_id} failed: {errors}\")\n\ndef get_job_history(limit=50, target=None, status=None):\n \"\"\"Get recent job history.\"\"\"\n conn = get_connection()\n cursor = conn.cursor()\n \n sql = \"SELECT * FROM job_history WHERE 1=1\"\n params = []\n \n if target:\n sql += \" AND target = ?\"\n params.append(target)\n \n if status:\n sql += \" AND status = ?\"\n params.append(status)\n \n sql += \" ORDER BY started_at DESC LIMIT ?\"\n params.append(limit)\n \n cursor.execute(sql, params)\n results = [dict(row) for row in cursor.fetchall()]\n conn.close()\n return results\n\ndef get_job_stats():\n \"\"\"Get aggregated job statistics.\"\"\"\n conn = get_connection()\n cursor = conn.cursor()\n \n stats = {}\n \n # Overall counts\n cursor.execute(\"\"\"\n SELECT \n COUNT(*) as total_jobs,\n SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) as completed,\n SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed,\n SUM(CASE WHEN status = 'running' THEN 1 ELSE 0 END) as running,\n AVG(duration_seconds) as avg_duration,\n SUM(posts_scraped) as total_posts,\n SUM(comments_scraped) as total_comments\n FROM job_history\n \"\"\")\n row = cursor.fetchone()\n if row:\n stats.update(dict(row))\n \n # Recent jobs\n cursor.execute(\"\"\"\n SELECT target, status, duration_seconds, posts_scraped, started_at\n FROM job_history ORDER BY started_at DESC LIMIT 10\n \"\"\")\n stats['recent_jobs'] = [dict(row) for row in cursor.fetchall()]\n \n conn.close()\n return stats\n\ndef print_job_history(limit=20):\n \"\"\"Pretty print job history.\"\"\"\n jobs = get_job_history(limit)\n \n print(\"\\n๐ Job History\")\n print(\"-\" * 80)\n print(f\"{'ID':<10} {'Target':<15} {'Status':<10} {'Posts':<8} {'Duration':<10} {'Started':<20}\")\n print(\"-\" * 80)\n \n for job in jobs:\n status_icon = \"โ
\" if job['status'] == 'completed' else \"โ\" if job['status'] == 'failed' else \"๐\"\n duration = f\"{job['duration_seconds']:.1f}s\" if job['duration_seconds'] else \"-\"\n started = job['started_at'][:19] if job['started_at'] else \"-\"\n dry = \" (dry)\" if job['dry_run'] else \"\"\n \n print(f\"{status_icon} {job['job_id']:<8} {job['target']:<15} {job['status']:<10} \"\n f\"{job['posts_scraped']:<8} {duration:<10} {started}{dry}\")\n \n print(\"-\" * 80)\n \n stats = get_job_stats()\n success_rate = (stats['completed'] / stats['total_jobs'] * 100) if stats['total_jobs'] else 0\n print(f\"\\n๐ Stats: {stats['total_jobs']} jobs | {success_rate:.0f}% success | \"\n f\"{stats['total_posts'] or 0} posts total\")\n\n# --- SQLITE MAINTENANCE FUNCTIONS ---\n\ndef enable_auto_vacuum():\n \"\"\"Enable incremental auto-vacuum on SQLite database.\"\"\"\n conn = get_connection()\n try:\n conn.execute(\"PRAGMA auto_vacuum = INCREMENTAL\")\n conn.execute(\"PRAGMA incremental_vacuum\")\n conn.commit()\n print(\"โ
Auto-vacuum enabled\")\n finally:\n conn.close()\n\ndef vacuum_database():\n \"\"\"Run VACUUM to optimize and compact the database.\"\"\"\n conn = get_connection()\n try:\n print(\"๐ง Running VACUUM...\")\n conn.execute(\"VACUUM\")\n print(\"โ
Database optimized\")\n finally:\n conn.close()\n\ndef backup_database(backup_path=None):\n \"\"\"\n Create a backup of the SQLite database.\n \n Args:\n backup_path: Optional custom backup path\n \n Returns:\n Path to the backup file\n \"\"\"\n import shutil\n \n backup_dir = DATA_DIR / \"backups\"\n backup_dir.mkdir(exist_ok=True)\n \n if backup_path is None:\n timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n backup_path = backup_dir / f\"reddit_scraper_{timestamp}.db\"\n \n shutil.copy2(DB_PATH, backup_path)\n \n # Get file size\n size_mb = Path(backup_path).stat().st_size / (1024 * 1024)\n print(f\"โ
Backup created: {backup_path} ({size_mb:.2f} MB)\")\n \n return str(backup_path)\n\ndef get_database_info():\n \"\"\"Get database size and table info.\"\"\"\n info = {}\n \n # File size\n if DB_PATH.exists():\n info['size_mb'] = DB_PATH.stat().st_size / (1024 * 1024)\n \n conn = get_connection()\n cursor = conn.cursor()\n \n # Table counts\n tables = ['posts', 'comments', 'job_history', 'alerts', 'subreddits']\n info['tables'] = {}\n \n for table in tables:\n try:\n cursor.execute(f\"SELECT COUNT(*) FROM {table}\")\n info['tables'][table] = cursor.fetchone()[0]\n except:\n info['tables'][table] = 0\n \n conn.close()\n return info\n\n# Initialize on import\ninit_database()\n\n"
},
{
"path": "export/parquet.py",
"content": "\"\"\"\nParquet Export Module - For DuckDB/Warehouse integration\nExport scraped data to Parquet format for analytics tools.\n\"\"\"\nimport pandas as pd\nfrom pathlib import Path\nfrom datetime import datetime\n\ndef export_to_parquet(subreddit, output_dir=None, prefix=\"r\"):\n \"\"\"\n Export subreddit data to Parquet format.\n \n Args:\n subreddit: Subreddit name\n output_dir: Output directory (default: data/parquet)\n prefix: \"r\" for subreddit, \"u\" for user\n \n Returns:\n Dictionary with paths to exported files\n \"\"\"\n try:\n import pyarrow\n except ImportError:\n raise ImportError(\"pyarrow required for Parquet export. Run: pip install pyarrow\")\n \n # Setup paths\n data_dir = Path(f\"data/{prefix}_{subreddit}\")\n output_path = Path(output_dir) if output_dir else Path(\"data/parquet\")\n output_path.mkdir(parents=True, exist_ok=True)\n \n if not data_dir.exists():\n print(f\"โ No data found for {prefix}/{subreddit}\")\n return {}\n \n exported = {}\n timestamp = datetime.now().strftime(\"%Y%m%d\")\n \n # Export posts\n posts_csv = data_dir / \"posts.csv\"\n if posts_csv.exists():\n print(f\"๐ฆ Converting posts to Parquet...\")\n df = pd.read_csv(posts_csv)\n \n # Convert datetime columns\n if 'created_utc' in df.columns:\n df['created_utc'] = pd.to_datetime(df['created_utc'], errors='coerce')\n \n # Optimize dtypes\n for col in ['score', 'num_comments', 'num_crossposts', 'total_awards']:\n if col in df.columns:\n df[col] = pd.to_numeric(df[col], errors='coerce').fillna(0).astype('int32')\n \n for col in ['is_nsfw', 'is_spoiler', 'has_media', 'media_downloaded']:\n if col in df.columns:\n df[col] = df[col].astype(bool)\n \n output_file = output_path / f\"{subreddit}_posts_{timestamp}.parquet\"\n df.to_parquet(output_file, engine=\"pyarrow\", compression=\"snappy\")\n \n size_mb = output_file.stat().st_size / (1024 * 1024)\n print(f\" โ
{output_file.name} ({len(df)} rows, {size_mb:.2f} MB)\")\n exported['posts'] = str(output_file)\n \n # Export comments\n comments_csv = data_dir / \"comments.csv\"\n if comments_csv.exists():\n print(f\"๐ฆ Converting comments to Parquet...\")\n df = pd.read_csv(comments_csv)\n \n if 'created_utc' in df.columns:\n df['created_utc'] = pd.to_datetime(df['created_utc'], errors='coerce')\n \n if 'score' in df.columns:\n df['score'] = pd.to_numeric(df['score'], errors='coerce').fillna(0).astype('int32')\n \n output_file = output_path / f\"{subreddit}_comments_{timestamp}.parquet\"\n df.to_parquet(output_file, engine=\"pyarrow\", compression=\"snappy\")\n \n size_mb = output_file.stat().st_size / (1024 * 1024)\n print(f\" โ
{output_file.name} ({len(df)} rows, {size_mb:.2f} MB)\")\n exported['comments'] = str(output_file)\n \n print(f\"\\nโ
Export complete! Files saved to: {output_path}\")\n print(f\" ๐ก Query with DuckDB: duckdb.query(\\\"SELECT * FROM '{exported.get('posts', '')}' LIMIT 10\\\")\")\n \n return exported\n\n\ndef export_database_to_parquet(output_dir=None):\n \"\"\"\n Export entire SQLite database to Parquet files.\n \n Args:\n output_dir: Output directory\n \n Returns:\n Dictionary with paths to exported files\n \"\"\"\n try:\n import pyarrow\n except ImportError:\n raise ImportError(\"pyarrow required. Run: pip install pyarrow\")\n \n from export.database import get_connection\n \n output_path = Path(output_dir) if output_dir else Path(\"data/parquet\")\n output_path.mkdir(parents=True, exist_ok=True)\n \n conn = get_connection()\n exported = {}\n timestamp = datetime.now().strftime(\"%Y%m%d\")\n \n tables = ['posts', 'comments', 'job_history']\n \n for table in tables:\n try:\n print(f\"๐ฆ Exporting {table}...\")\n df = pd.read_sql(f\"SELECT * FROM {table}\", conn)\n \n if len(df) > 0:\n output_file = output_path / f\"db_{table}_{timestamp}.parquet\"\n df.to_parquet(output_file, engine=\"pyarrow\", compression=\"snappy\")\n \n size_mb = output_file.stat().st_size / (1024 * 1024)\n print(f\" โ
{output_file.name} ({len(df)} rows, {size_mb:.2f} MB)\")\n exported[table] = str(output_file)\n else:\n print(f\" โญ๏ธ {table} is empty, skipping\")\n except Exception as e:\n print(f\" โ Failed to export {table}: {e}\")\n \n conn.close()\n return exported\n\n\ndef list_parquet_files(directory=\"data/parquet\"):\n \"\"\"List all Parquet files in directory.\"\"\"\n parquet_dir = Path(directory)\n \n if not parquet_dir.exists():\n print(f\"๐ No Parquet directory found at {directory}\")\n return []\n \n files = list(parquet_dir.glob(\"*.parquet\"))\n \n print(f\"\\n๐ Parquet Files in {directory}:\")\n print(\"-\" * 60)\n \n for f in files:\n size_mb = f.stat().st_size / (1024 * 1024)\n mtime = datetime.fromtimestamp(f.stat().st_mtime).strftime(\"%Y-%m-%d %H:%M\")\n print(f\" {f.name:<40} {size_mb:>6.2f} MB {mtime}\")\n \n print(\"-\" * 60)\n print(f\"Total: {len(files)} files\")\n \n return [str(f) for f in files]\n\n\n# CLI for testing\nif __name__ == \"__main__\":\n import argparse\n \n parser = argparse.ArgumentParser(description=\"Parquet Export\")\n parser.add_argument(\"subreddit\", nargs='?', help=\"Subreddit to export\")\n parser.add_argument(\"--user\", action=\"store_true\", help=\"Is a user profile\")\n parser.add_argument(\"--output\", type=str, help=\"Output directory\")\n parser.add_argument(\"--database\", action=\"store_true\", help=\"Export entire database\")\n parser.add_argument(\"--list\", action=\"store_true\", help=\"List Parquet files\")\n \n args = parser.parse_args()\n \n if args.list:\n list_parquet_files()\n elif args.database:\n export_database_to_parquet(args.output)\n elif args.subreddit:\n prefix = \"u\" if args.user else \"r\"\n export_to_parquet(args.subreddit, args.output, prefix)\n else:\n parser.print_help()\n"
},
{
"path": "main.py",
"content": "\"\"\"\n๐ค Universal Reddit Scraper Suite\nFull-featured scraper with analytics, dashboard, notifications, and scheduling.\n\"\"\"\nimport requests\nimport pandas as pd\nimport datetime\nimport time\nimport os\nimport xml.etree.ElementTree as ET\nimport argparse\nimport random\nimport sys\nimport json\nimport subprocess\nimport tempfile\nfrom urllib.parse import urlparse\nfrom pathlib import Path\n\n# --- CONFIGURATION ---\nUSER_AGENT = \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36\"\n\nMIRRORS = [\n \"https://old.reddit.com\",\n \"https://redlib.catsarch.com\",\n \"https://redlib.vsls.cz\",\n \"https://r.nf\",\n \"https://libreddit.northboot.xyz\",\n \"https://redlib.tux.pizza\"\n]\n\nSEEN_URLS = set()\nSESSION = requests.Session()\nSESSION.headers.update({\"User-Agent\": USER_AGENT})\n\n# --- DIRECTORY SETUP ---\ndef setup_directories(target, prefix):\n \"\"\"Creates organized folder structure for scraped data.\"\"\"\n base_dir = f\"data/{prefix}_{target}\"\n dirs = {\n \"base\": base_dir,\n \"posts\": f\"{base_dir}/posts.csv\",\n \"comments\": f\"{base_dir}/comments.csv\",\n \"media\": f\"{base_dir}/media\",\n \"images\": f\"{base_dir}/media/images\",\n \"videos\": f\"{base_dir}/media/videos\",\n }\n \n for key in [\"base\", \"media\", \"images\", \"videos\"]:\n if not os.path.exists(dirs[key]):\n os.makedirs(dirs[key])\n \n return dirs\n\ndef get_file_path(target, type_prefix):\n \"\"\"Legacy function for backward compatibility.\"\"\"\n if not os.path.exists(\"data\"):\n os.makedirs(\"data\")\n sanitized_target = target.replace(\"/\", \"_\")\n return f\"data/{type_prefix}_{sanitized_target}.csv\"\n\ndef load_history(filepath):\n \"\"\"Loads existing CSV history to prevent duplicates.\"\"\"\n SEEN_URLS.clear()\n if os.path.exists(filepath):\n try:\n df = pd.read_csv(filepath)\n for url in df['permalink']:\n SEEN_URLS.add(str(url))\n print(f\"๐ Loaded {len(SEEN_URLS)} existing items from {filepath}\")\n except:\n pass\n\ndef save_posts_csv(posts, filepath):\n \"\"\"Saves posts to CSV with all metadata.\"\"\"\n if not posts:\n return 0\n \n new_posts = [p for p in posts if p['permalink'] not in SEEN_URLS]\n \n if new_posts:\n df = pd.DataFrame(new_posts)\n if os.path.exists(filepath):\n df.to_csv(filepath, mode='a', header=False, index=False)\n else:\n df.to_csv(filepath, index=False)\n \n for p in new_posts:\n SEEN_URLS.add(p['permalink'])\n \n print(f\"โ
Saved {len(new_posts)} new posts\")\n return len(new_posts)\n else:\n print(\"๐ค No new unique posts found.\")\n return 0\n\ndef save_comments_csv(comments, filepath):\n \"\"\"Saves comments to CSV.\"\"\"\n if not comments:\n return\n \n df = pd.DataFrame(comments)\n if os.path.exists(filepath):\n df.to_csv(filepath, mode='a', header=False, index=False)\n else:\n df.to_csv(filepath, index=False)\n \n print(f\"๐ฌ Saved {len(comments)} comments\")\n\n# --- MEDIA DOWNLOAD ---\ndef get_media_urls(post_data):\n \"\"\"Extracts all media URLs from a post.\"\"\"\n media = {\"images\": [], \"videos\": [], \"galleries\": []}\n \n url = post_data.get('url', '')\n if any(ext in url.lower() for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp']):\n media[\"images\"].append(url)\n \n if 'i.redd.it' in url:\n media[\"images\"].append(url)\n \n if post_data.get('is_video'):\n reddit_video = post_data.get('media', {})\n if reddit_video and 'reddit_video' in reddit_video:\n video_url = reddit_video['reddit_video'].get('fallback_url', '')\n if video_url:\n media[\"videos\"].append(video_url.split('?')[0])\n \n preview = post_data.get('preview', {})\n if preview and 'images' in preview:\n for img in preview['images']:\n source = img.get('source', {})\n if source.get('url'):\n clean_url = source['url'].replace('&', '&')\n media[\"images\"].append(clean_url)\n \n if post_data.get('is_gallery'):\n gallery_data = post_data.get('gallery_data', {})\n media_metadata = post_data.get('media_metadata', {})\n \n if gallery_data and media_metadata:\n for item in gallery_data.get('items', []):\n media_id = item.get('media_id')\n if media_id and media_id in media_metadata:\n meta = media_metadata[media_id]\n if meta.get('s', {}).get('u'):\n clean_url = meta['s']['u'].replace('&', '&')\n media[\"galleries\"].append(clean_url)\n \n if 'youtube.com' in url or 'youtu.be' in url:\n media[\"videos\"].append(url)\n \n return media\n\ndef download_media(url, save_path, media_type=\"image\"):\n \"\"\"Downloads a single media file.\"\"\"\n try:\n if os.path.exists(save_path):\n return True\n \n response = SESSION.get(url, timeout=30, stream=True)\n if response.status_code == 200:\n with open(save_path, 'wb') as f:\n for chunk in response.iter_content(chunk_size=8192):\n f.write(chunk)\n return True\n except Exception as e:\n pass\n return False\n\ndef download_reddit_video_with_audio(video_url, save_path):\n \"\"\"\n Downloads Reddit video with audio by fetching both streams and merging.\n Reddit stores video and audio separately - this combines them.\n \"\"\"\n try:\n if os.path.exists(save_path):\n return True\n \n # Try to find the audio URL by replacing video quality with audio\n # Reddit videos have audio at URLs like .../DASH_audio.mp4 or .../DASH_AUDIO_128.mp4\n base_url = video_url.rsplit('/', 1)[0]\n \n # Common audio URL patterns\n audio_urls = [\n f\"{base_url}/DASH_audio.mp4\",\n f\"{base_url}/DASH_AUDIO_128.mp4\",\n f\"{base_url}/DASH_AUDIO_64.mp4\",\n f\"{base_url}/audio.mp4\",\n f\"{base_url}/audio\"\n ]\n \n # Download video to temp file first\n with tempfile.NamedTemporaryFile(suffix='_video.mp4', delete=False) as video_temp:\n video_temp_path = video_temp.name\n response = SESSION.get(video_url, timeout=60, stream=True)\n if response.status_code != 200:\n return False\n for chunk in response.iter_content(chunk_size=8192):\n video_temp.write(chunk)\n \n # Try to download audio\n audio_temp_path = None\n for audio_url in audio_urls:\n try:\n response = SESSION.get(audio_url, timeout=30, stream=True)\n if response.status_code == 200:\n with tempfile.NamedTemporaryFile(suffix='_audio.mp4', delete=False) as audio_temp:\n audio_temp_path = audio_temp.name\n for chunk in response.iter_content(chunk_size=8192):\n audio_temp.write(chunk)\n break\n except:\n continue\n \n if audio_temp_path:\n # Merge video and audio using ffmpeg\n try:\n cmd = [\n 'ffmpeg', '-y', '-hide_banner', '-loglevel', 'error',\n '-i', video_temp_path,\n '-i', audio_temp_path,\n '-c:v', 'copy', '-c:a', 'aac',\n '-shortest', save_path\n ]\n result = subprocess.run(cmd, capture_output=True, timeout=120)\n \n if result.returncode == 0:\n # Cleanup temp files\n os.unlink(video_temp_path)\n os.unlink(audio_temp_path)\n return True\n else:\n # ffmpeg failed, fall back to video only\n print(f\" โ ๏ธ ffmpeg merge failed, saving video without audio\")\n os.rename(video_temp_path, save_path)\n os.unlink(audio_temp_path)\n return True\n except FileNotFoundError:\n # ffmpeg not installed, save video only\n print(f\" โ ๏ธ ffmpeg not found, saving video without audio\")\n os.rename(video_temp_path, save_path)\n if audio_temp_path:\n os.unlink(audio_temp_path)\n return True\n except Exception as e:\n # Other error, save video only\n os.rename(video_temp_path, save_path)\n if audio_temp_path and os.path.exists(audio_temp_path):\n os.unlink(audio_temp_path)\n return True\n else:\n # No audio found, just use video\n os.rename(video_temp_path, save_path)\n return True\n \n except Exception as e:\n # Cleanup any temp files on error\n pass\n return False\n\ndef download_post_media(post_data, dirs, post_id):\n \"\"\"Downloads all media from a post.\"\"\"\n media = get_media_urls(post_data)\n downloaded = {\"images\": 0, \"videos\": 0}\n \n for i, img_url in enumerate(media[\"images\"][:5]):\n ext = os.path.splitext(urlparse(img_url).path)[1] or '.jpg'\n save_path = os.path.join(dirs[\"images\"], f\"{post_id}_{i}{ext}\")\n if download_media(img_url, save_path, \"image\"):\n downloaded[\"images\"] += 1\n \n for i, img_url in enumerate(media[\"galleries\"][:10]):\n ext = '.jpg'\n save_path = os.path.join(dirs[\"images\"], f\"{post_id}_gallery_{i}{ext}\")\n if download_media(img_url, save_path, \"gallery\"):\n downloaded[\"images\"] += 1\n \n for i, vid_url in enumerate(media[\"videos\"][:2]):\n if 'youtube' not in vid_url:\n ext = '.mp4'\n save_path = os.path.join(dirs[\"videos\"], f\"{post_id}_{i}{ext}\")\n # Use enhanced download for Reddit videos (includes audio)\n if 'v.redd.it' in vid_url or 'reddit.com' in vid_url:\n if download_reddit_video_with_audio(vid_url, save_path):\n downloaded[\"videos\"] += 1\n elif download_media(vid_url, save_path, \"video\"):\n downloaded[\"videos\"] += 1\n \n return downloaded\n\n# --- COMMENT SCRAPING ---\ndef scrape_comments(permalink, max_depth=3):\n \"\"\"Scrapes comments from a post.\"\"\"\n comments = []\n \n try:\n if not permalink.startswith('http'):\n url = f\"https://old.reddit.com{permalink}.json?limit=100\"\n else:\n url = f\"{permalink}.json?limit=100\"\n \n response = SESSION.get(url, timeout=15)\n if response.status_code != 200:\n return comments\n \n data = response.json()\n \n if len(data) > 1:\n comment_data = data[1]['data']['children']\n comments = parse_comments(comment_data, permalink, depth=0, max_depth=max_depth)\n \n except Exception as e:\n pass\n \n if len(comments) > 0:\n print(f\" + Scraped {len(comments)} comments\")\n \n return comments\n\ndef parse_comments(comment_list, post_permalink, depth=0, max_depth=3):\n \"\"\"Recursively parses comments.\"\"\"\n comments = []\n \n if depth > max_depth:\n return comments\n \n for item in comment_list:\n if item['kind'] != 't1':\n continue\n \n c = item['data']\n \n comment = {\n \"post_permalink\": post_permalink,\n \"comment_id\": c.get('id'),\n \"parent_id\": c.get('parent_id'),\n \"author\": c.get('author'),\n \"body\": c.get('body', ''),\n \"score\": c.get('score', 0),\n \"created_utc\": datetime.datetime.fromtimestamp(c.get('created_utc', 0)).isoformat(),\n \"depth\": depth,\n \"is_submitter\": c.get('is_submitter', False),\n }\n comments.append(comment)\n \n replies = c.get('replies')\n if replies and isinstance(replies, dict):\n reply_children = replies.get('data', {}).get('children', [])\n comments.extend(parse_comments(reply_children, post_permalink, depth + 1, max_depth))\n \n return comments\n\n# --- POST EXTRACTION ---\ndef extract_post_data(post_json):\n \"\"\"Extracts comprehensive post data.\"\"\"\n p = post_json\n \n post_type = \"text\"\n if p.get('is_video'):\n post_type = \"video\"\n elif p.get('is_gallery'):\n post_type = \"gallery\"\n elif any(ext in p.get('url', '').lower() for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp']) or 'i.redd.it' in p.get('url', ''):\n post_type = \"image\"\n elif p.get('is_self'):\n post_type = \"text\"\n else:\n post_type = \"link\"\n \n return {\n \"id\": p.get('id'),\n \"title\": p.get('title'),\n \"author\": p.get('author'),\n \"created_utc\": datetime.datetime.fromtimestamp(p.get('created_utc', 0)).isoformat(),\n \"permalink\": p.get('permalink'),\n \"url\": p.get('url_overridden_by_dest', p.get('url')),\n \"score\": p.get('score', 0),\n \"upvote_ratio\": p.get('upvote_ratio', 0),\n \"num_comments\": p.get('num_comments', 0),\n \"num_crossposts\": p.get('num_crossposts', 0),\n \"selftext\": p.get('selftext', ''),\n \"post_type\": post_type,\n \"is_nsfw\": p.get('over_18', False),\n \"is_spoiler\": p.get('spoiler', False),\n \"flair\": p.get('link_flair_text', ''),\n \"total_awards\": p.get('total_awards_received', 0),\n \"has_media\": p.get('is_video', False) or p.get('is_gallery', False) or 'i.redd.it' in p.get('url', ''),\n \"media_downloaded\": False,\n \"source\": \"History-Full\"\n }\n\n# --- FULL HISTORY SCRAPE ---\ndef run_full_history(target, limit, is_user=False, download_media_flag=True, \n scrape_comments_flag=True, dry_run=False, use_plugins=False):\n \"\"\"\n Full scrape with images, videos, and comments.\n \n Args:\n target: Subreddit or username\n limit: Maximum posts to scrape\n is_user: True if target is a user\n download_media_flag: Download images/videos\n scrape_comments_flag: Scrape comments\n dry_run: Simulate without saving data\n use_plugins: Run post-processing plugins\n \"\"\"\n prefix = \"u\" if is_user else \"r\"\n mode = \"full\" if download_media_flag and scrape_comments_flag else \"history\"\n \n # Display mode banner\n if dry_run:\n print(\"=\" * 50)\n print(\"๐งช DRY RUN MODE - No data will be saved\")\n print(\"=\" * 50)\n \n print(f\"๐ Starting {'DRY RUN' if dry_run else 'FULL HISTORY'} scrape for {prefix}/{target}\")\n print(f\" ๐ Target posts: {limit}\")\n print(f\" ๐ผ๏ธ Download media: {download_media_flag and not dry_run}\")\n print(f\" ๐ฌ Scrape comments: {scrape_comments_flag}\")\n print(f\" ๐ Plugins enabled: {use_plugins}\")\n print(\"-\" * 50)\n \n # Start job tracking\n job_id = None\n try:\n from export.database import start_job_record, complete_job_record\n job_id = start_job_record(target, mode, is_user, dry_run)\n except Exception as e:\n print(f\"โ ๏ธ Job tracking unavailable: {e}\")\n \n # Setup directories (even for dry run, to check existing data)\n dirs = setup_directories(target, prefix)\n load_history(dirs[\"posts\"])\n \n after = None\n total_posts = 0\n total_media = {\"images\": 0, \"videos\": 0}\n total_comments = 0\n all_scraped_posts = [] # For plugin processing\n all_scraped_comments = []\n start_time = time.time()\n error_msg = None\n \n try:\n while total_posts < limit:\n random.shuffle(MIRRORS)\n success = False\n \n for base_url in MIRRORS:\n try:\n if is_user:\n path = f\"/user/{target}/submitted.json\"\n else:\n path = f\"/r/{target}/new.json\"\n \n # Use proper batch size - min of remaining posts needed or 100 (Reddit's max per request)\n batch_size = min(100, limit - total_posts)\n target_url = f\"{base_url}{path}?limit={batch_size}&raw_json=1\"\n if after:\n target_url += f\"&after={after}\"\n \n print(f\"\\n๐ก Fetching from: {base_url}\")\n response = SESSION.get(target_url, timeout=15)\n \n if response.status_code == 200:\n data = response.json()\n posts = []\n batch_comments = []\n \n children = data['data']['children']\n print(f\" Found {len(children)} posts in this batch\")\n \n for child in children:\n p = child['data']\n post = extract_post_data(p)\n \n if post['permalink'] in SEEN_URLS:\n continue\n \n # Download media (skip in dry run)\n if download_media_flag and not dry_run:\n downloaded = download_post_media(p, dirs, post['id'])\n post['media_downloaded'] = downloaded['images'] > 0 or downloaded['videos'] > 0\n total_media['images'] += downloaded['images']\n total_media['videos'] += downloaded['videos']\n \n if downloaded['images'] > 0 or downloaded['videos'] > 0:\n print(f\" + Downloaded: {downloaded['images']} images, {downloaded['videos']} videos\")\n \n posts.append(post)\n \n # Scrape comments\n if scrape_comments_flag and post['num_comments'] > 0:\n print(f\" ๐ฌ Fetching comments for: {post['title'][:40]}...\")\n comments = scrape_comments(post['permalink'])\n batch_comments.extend(comments)\n total_comments += len(comments)\n time.sleep(1)\n \n # Collect for plugins\n all_scraped_posts.extend(posts)\n all_scraped_comments.extend(batch_comments)\n \n # Save data (skip in dry run)\n if not dry_run:\n saved = save_posts_csv(posts, dirs[\"posts\"])\n total_posts += saved\n \n if batch_comments:\n save_comments_csv(batch_comments, dirs[\"comments\"])\n else:\n # In dry run, just count\n total_posts += len(posts)\n print(f\" ๐งช [DRY RUN] Would save {len(posts)} posts\")\n \n print(f\"\\n๐ Progress: {total_posts}/{limit} posts\")\n print(f\" ๐ผ๏ธ Images: {total_media['images']} | ๐ฌ Videos: {total_media['videos']}\")\n print(f\" ๐ฌ Comments: {total_comments}\")\n \n after = data['data'].get('after')\n if not after:\n print(\"\\n๐ Reached end of available history.\")\n break\n \n success = True\n break\n \n except Exception as e:\n print(f\" โ ๏ธ Error with {base_url}: {e}\")\n continue\n \n if not after:\n break\n \n if not success:\n print(\"\\nโ All sources failed. Waiting 30s...\")\n time.sleep(30)\n else:\n print(f\"\\nโธ๏ธ Cooling down (3s)...\")\n time.sleep(3)\n \n # Run plugins on collected data\n if use_plugins and (all_scraped_posts or all_scraped_comments):\n print(\"\\n๐ Running post-processing plugins...\")\n try:\n from plugins import load_plugins, run_plugins\n plugins = load_plugins()\n if plugins:\n all_scraped_posts, all_scraped_comments = run_plugins(\n all_scraped_posts, all_scraped_comments, plugins\n )\n print(f\" โ
Processed {len(all_scraped_posts)} posts with {len(plugins)} plugins\")\n else:\n print(\" โ ๏ธ No plugins found\")\n except Exception as e:\n print(f\" โ ๏ธ Plugin error: {e}\")\n \n except Exception as e:\n error_msg = str(e)\n print(f\"\\nโ Scrape error: {e}\")\n \n duration = time.time() - start_time\n \n # Complete job tracking\n if job_id:\n try:\n status = 'failed' if error_msg else 'completed'\n complete_job_record(\n job_id, status, \n total_posts, total_comments, \n total_media['images'] + total_media['videos'],\n error_msg\n )\n except Exception as e:\n print(f\"โ ๏ธ Failed to complete job record: {e}\")\n \n # Summary\n print(\"\\n\" + \"=\" * 50)\n if dry_run:\n print(\"๐งช DRY RUN COMPLETE!\")\n print(f\" ๐ Would scrape: {total_posts} posts\")\n print(f\" ๐ฌ Would scrape: {total_comments} comments\")\n else:\n print(\"โ
SCRAPE COMPLETE!\")\n print(f\" ๐ Data saved to: {dirs['base']}\")\n print(f\" ๐ Total posts: {total_posts}\")\n print(f\" ๐ผ๏ธ Total images: {total_media['images']}\")\n print(f\" ๐ฌ Total videos: {total_media['videos']}\")\n print(f\" ๐ฌ Total comments: {total_comments}\")\n print(f\" โฑ๏ธ Duration: {duration:.1f}s\")\n \n return {\n 'posts': total_posts,\n 'images': total_media['images'],\n 'videos': total_media['videos'],\n 'comments': total_comments,\n 'duration': f\"{duration:.1f}s\",\n 'dry_run': dry_run,\n 'job_id': job_id\n }\n\n# --- MONITOR MODE ---\ndef run_monitor(target, is_user=False):\n prefix = \"u\" if is_user else \"r\"\n if is_user:\n rss_url = f\"https://www.reddit.com/user/{target}/submitted.rss?limit=100\"\n else:\n rss_url = f\"https://www.reddit.com/r/{target}/new.rss?limit=100\"\n\n print(f\"[{datetime.datetime.now()}] ๐ก Checking RSS for {prefix}/{target}...\")\n \n try:\n response = SESSION.get(rss_url, timeout=15)\n \n if response.status_code != 200:\n print(f\"โ RSS blocked (Status {response.status_code}), trying JSON...\")\n run_full_history(target, 25, is_user, download_media_flag=False, scrape_comments_flag=False)\n return\n\n root = ET.fromstring(response.content)\n namespace = {'atom': 'http://www.w3.org/2005/Atom'}\n posts = []\n \n for entry in root.findall('atom:entry', namespace):\n posts.append({\n \"id\": \"\",\n \"title\": entry.find('atom:title', namespace).text,\n \"author\": \"\",\n \"created_utc\": entry.find('atom:published', namespace).text,\n \"permalink\": entry.find('atom:link', namespace).attrib['href'],\n \"url\": entry.find('atom:link', namespace).attrib['href'],\n \"score\": 0,\n \"upvote_ratio\": 0,\n \"num_comments\": 0,\n \"num_crossposts\": 0,\n \"selftext\": \"\",\n \"post_type\": \"unknown\",\n \"is_nsfw\": False,\n \"is_spoiler\": False,\n \"flair\": \"\",\n \"total_awards\": 0,\n \"has_media\": False,\n \"media_downloaded\": False,\n \"source\": \"Monitor-RSS\"\n })\n \n dirs = setup_directories(target, prefix)\n save_posts_csv(posts, dirs[\"posts\"])\n\n except Exception as e:\n print(f\"โ Monitor Error: {e}\")\n\n# --- CLI ---\ndef main():\n parser = argparse.ArgumentParser(\n description=\"๐ค Universal Reddit Scraper Suite\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=\"\"\"\nCommands:\n SCRAPING:\n python main.py --mode full --limit 100\n python main.py --mode history --limit 500\n python main.py --mode monitor\n python main.py --dry-run # Test without saving\n python main.py --plugins # Enable post-processing\n \n SEARCH:\n python main.py --search \"keyword\" --subreddit delhi\n python main.py --search \"keyword\" --min-score 100\n \n DASHBOARD:\n python main.py --dashboard\n \n SCHEDULE:\n python main.py --schedule delhi --every 60\n \n ANALYTICS:\n python main.py --analyze delhi --sentiment\n python main.py --analyze delhi --keywords\n \n MAINTENANCE:\n python main.py --job-history # View job history\n python main.py --backup # Backup database\n python main.py --vacuum # Optimize database\n python main.py --export-parquet python # Export to Parquet\n python main.py --list-plugins # List available plugins\n \n REST API:\n python main.py --api # Start REST API server\n \"\"\"\n )\n \n # Scraping args\n parser.add_argument(\"target\", nargs='?', help=\"Subreddit or username to scrape\")\n parser.add_argument(\"--mode\", choices=[\"monitor\", \"history\", \"full\"], default=\"full\")\n parser.add_argument(\"--user\", action=\"store_true\", help=\"Target is a user\")\n parser.add_argument(\"--limit\", type=int, default=100, help=\"Max posts to scrape\")\n parser.add_argument(\"--no-media\", action=\"store_true\", help=\"Skip media download\")\n parser.add_argument(\"--no-comments\", action=\"store_true\", help=\"Skip comments\")\n \n # Dashboard\n parser.add_argument(\"--dashboard\", action=\"store_true\", help=\"Launch web dashboard\")\n \n # Search\n parser.add_argument(\"--search\", type=str, help=\"Search scraped data\")\n parser.add_argument(\"--subreddit\", type=str, help=\"Filter by subreddit\")\n parser.add_argument(\"--min-score\", type=int, help=\"Filter by minimum score\")\n parser.add_argument(\"--author\", type=str, help=\"Filter by author\")\n \n # Analytics\n parser.add_argument(\"--analyze\", type=str, help=\"Run analytics on subreddit\")\n parser.add_argument(\"--sentiment\", action=\"store_true\", help=\"Run sentiment analysis\")\n parser.add_argument(\"--keywords\", action=\"store_true\", help=\"Extract keywords\")\n \n # Schedule\n parser.add_argument(\"--schedule\", type=str, help=\"Schedule scraping for target\")\n parser.add_argument(\"--every\", type=int, help=\"Interval in minutes\")\n \n # Alerts\n parser.add_argument(\"--alert\", type=str, help=\"Set keyword alert\")\n parser.add_argument(\"--discord-webhook\", type=str, help=\"Discord webhook URL\")\n parser.add_argument(\"--telegram-token\", type=str, help=\"Telegram bot token\")\n parser.add_argument(\"--telegram-chat\", type=str, help=\"Telegram chat ID\")\n \n # New: Observability & Maintenance\n parser.add_argument(\"--dry-run\", action=\"store_true\", help=\"Simulate scrape without saving data\")\n parser.add_argument(\"--plugins\", action=\"store_true\", help=\"Enable post-processing plugins\")\n parser.add_argument(\"--list-plugins\", action=\"store_true\", help=\"List available plugins\")\n parser.add_argument(\"--job-history\", action=\"store_true\", help=\"View job history\")\n parser.add_argument(\"--backup\", action=\"store_true\", help=\"Backup SQLite database\")\n parser.add_argument(\"--vacuum\", action=\"store_true\", help=\"Optimize SQLite database\")\n parser.add_argument(\"--export-parquet\", type=str, help=\"Export subreddit to Parquet format\")\n parser.add_argument(\"--api\", action=\"store_true\", help=\"Start REST API server (port 8000)\")\n \n args = parser.parse_args()\n \n print(\"=\" * 50)\n print(\"๐ค UNIVERSAL REDDIT SCRAPER SUITE\")\n print(\"=\" * 50)\n \n # Dashboard mode\n if args.dashboard:\n print(\"\\n๐ Launching Dashboard...\")\n print(\" Open: http://localhost:8501\")\n os.system(\"streamlit run dashboard/app.py\")\n return\n \n # REST API mode\n if args.api:\n print(\"\\n๐ Starting REST API server...\")\n print(\" ๐ Docs: http://localhost:8000/docs\")\n print(\" ๐ Connect Metabase/Grafana to http://localhost:8000\")\n try:\n import uvicorn\n from api.server import app\n uvicorn.run(app, host=\"0.0.0.0\", port=8000)\n except ImportError:\n print(\"โ Install dependencies: pip install fastapi uvicorn\")\n return\n \n # --- NEW: Maintenance & Observability Commands ---\n \n # Job history\n if args.job_history:\n from export.database import print_job_history\n print_job_history()\n return\n \n # Backup database\n if args.backup:\n from export.database import backup_database\n backup_database()\n return\n \n # Vacuum/optimize database\n if args.vacuum:\n from export.database import vacuum_database\n vacuum_database()\n return\n \n # Export to Parquet\n if args.export_parquet:\n from export.parquet import export_to_parquet\n prefix = \"u\" if args.user else \"r\"\n export_to_parquet(args.export_parquet, prefix=prefix)\n return\n \n # List plugins\n if args.list_plugins:\n from plugins import list_plugins\n list_plugins()\n return\n \n # Search mode\n if args.search:\n print(f\"\\n๐ Searching for: {args.search}\")\n from search.query import search_all_data, print_search_results\n \n results = search_all_data(\n query=args.search,\n min_score=args.min_score,\n author=args.author\n )\n print_search_results(results)\n return\n \n # Analytics mode\n if args.analyze:\n print(f\"\\n๐ Analyzing: {args.analyze}\")\n \n # Load data\n data_dir = Path(f\"data/r_{args.analyze}\")\n if not data_dir.exists():\n print(f\"โ No data found for r/{args.analyze}\")\n return\n \n posts_file = data_dir / \"posts.csv\"\n if not posts_file.exists():\n print(f\"โ No posts data found\")\n return\n \n import pandas as pd\n df = pd.read_csv(posts_file)\n posts = df.to_dict('records')\n \n if args.sentiment:\n from analytics.sentiment import analyze_posts_sentiment\n analyzed, counts = analyze_posts_sentiment(posts)\n print(f\"\\n๐ Sentiment Analysis:\")\n print(f\" Positive: {counts['positive']}\")\n print(f\" Neutral: {counts['neutral']}\")\n print(f\" Negative: {counts['negative']}\")\n \n if args.keywords:\n from analytics.sentiment import extract_keywords\n texts = [str(p.get('title', '') or '') + ' ' + str(p.get('selftext', '') or '') for p in posts]\n keywords = extract_keywords(texts, top_n=20)\n print(f\"\\nโ๏ธ Top Keywords:\")\n for word, count in keywords:\n print(f\" {word}: {count}\")\n \n return\n \n # Schedule mode\n if args.schedule:\n if not args.every:\n print(\"โ Please specify --every \")\n return\n \n from scheduler.cron import run_scheduled\n run_scheduled(args.schedule, args.every, args.mode, args.limit, args.user)\n return\n \n # Regular scraping mode\n if not args.target:\n parser.print_help()\n return\n \n if args.mode == \"monitor\":\n prefix = \"u\" if args.user else \"r\"\n dirs = setup_directories(args.target, prefix)\n load_history(dirs[\"posts\"])\n print(f\"๐ Monitoring {prefix}/{args.target} every 5 mins...\")\n while True:\n run_monitor(args.target, args.user)\n time.sleep(300)\n elif args.mode == \"history\":\n run_full_history(args.target, args.limit, args.user, \n download_media_flag=False, scrape_comments_flag=False,\n dry_run=args.dry_run, use_plugins=args.plugins)\n else:\n run_full_history(args.target, args.limit, args.user,\n download_media_flag=not args.no_media,\n scrape_comments_flag=not args.no_comments,\n dry_run=args.dry_run, use_plugins=args.plugins)\n\nif __name__ == \"__main__\":\n main()\n"
},
{
"path": "plugins/__init__.py",
"content": "\"\"\"\nLightweight Plugin System for Post-Processing\nPlugins can process posts and comments after scraping.\n\"\"\"\nfrom abc import ABC, abstractmethod\nfrom pathlib import Path\nimport importlib.util\nimport sys\n\n\nclass Plugin(ABC):\n \"\"\"\n Base class for scraper plugins.\n \n To create a plugin:\n 1. Create a new .py file in the plugins/ directory\n 2. Create a class that inherits from Plugin\n 3. Implement the process_posts() method\n 4. Optionally implement process_comments()\n \n Example:\n class MyPlugin(Plugin):\n name = \"my_plugin\"\n description = \"Does something cool\"\n \n def process_posts(self, posts):\n for post in posts:\n post['processed'] = True\n return posts\n \"\"\"\n name = \"base\"\n description = \"Base plugin\"\n enabled = True\n \n @abstractmethod\n def process_posts(self, posts: list) -> list:\n \"\"\"\n Process posts after scraping.\n \n Args:\n posts: List of post dictionaries\n \n Returns:\n Modified list of posts\n \"\"\"\n pass\n \n def process_comments(self, comments: list) -> list:\n \"\"\"\n Process comments after scraping (optional).\n \n Args:\n comments: List of comment dictionaries\n \n Returns:\n Modified list of comments\n \"\"\"\n return comments\n \n def __repr__(self):\n return f\"\"\n\n\ndef load_plugins(plugin_dir=None):\n \"\"\"\n Load all plugins from the plugins directory.\n \n Args:\n plugin_dir: Path to plugins directory\n \n Returns:\n List of plugin instances\n \"\"\"\n if plugin_dir is None:\n plugin_dir = Path(__file__).parent\n else:\n plugin_dir = Path(plugin_dir)\n \n plugins = []\n \n for file in plugin_dir.glob(\"*.py\"):\n # Skip __init__.py and base files\n if file.name.startswith(\"_\"):\n continue\n \n try:\n # Load the module\n spec = importlib.util.spec_from_file_location(file.stem, file)\n module = importlib.util.module_from_spec(spec)\n sys.modules[file.stem] = module\n spec.loader.exec_module(module)\n \n # Find Plugin subclasses\n for attr_name in dir(module):\n attr = getattr(module, attr_name)\n if (isinstance(attr, type) and \n issubclass(attr, Plugin) and \n attr != Plugin and\n hasattr(attr, 'name')):\n \n plugin_instance = attr()\n if plugin_instance.enabled:\n plugins.append(plugin_instance)\n \n except Exception as e:\n print(f\"โ ๏ธ Failed to load plugin {file.name}: {e}\")\n \n return plugins\n\n\ndef run_plugins(posts, comments, plugins):\n \"\"\"\n Run all plugins on scraped data.\n \n Args:\n posts: List of posts\n comments: List of comments\n plugins: List of plugin instances\n \n Returns:\n Tuple of (processed_posts, processed_comments)\n \"\"\"\n for plugin in plugins:\n try:\n print(f\"๐ Running plugin: {plugin.name}\")\n posts = plugin.process_posts(posts)\n comments = plugin.process_comments(comments)\n except Exception as e:\n print(f\"โ ๏ธ Plugin {plugin.name} failed: {e}\")\n \n return posts, comments\n\n\ndef list_plugins(plugin_dir=None):\n \"\"\"List all available plugins.\"\"\"\n plugins = load_plugins(plugin_dir)\n \n print(\"\\n๐ Available Plugins:\")\n print(\"-\" * 50)\n \n if not plugins:\n print(\" No plugins found\")\n else:\n for plugin in plugins:\n status = \"โ
\" if plugin.enabled else \"โ\"\n print(f\" {status} {plugin.name:<20} {plugin.description}\")\n \n print(\"-\" * 50)\n return plugins\n"
},
{
"path": "plugins/deduplicator.py",
"content": "\"\"\"\nDeduplicator Plugin\nRemoves duplicate posts based on permalink.\n\"\"\"\nfrom plugins import Plugin\n\n\nclass Deduplicator(Plugin):\n \"\"\"Remove duplicate posts by permalink.\"\"\"\n \n name = \"deduplicator\"\n description = \"Removes duplicate posts by permalink\"\n enabled = True\n \n def process_posts(self, posts):\n \"\"\"Remove duplicate posts.\"\"\"\n seen = set()\n unique = []\n duplicates = 0\n \n for post in posts:\n key = post.get('permalink')\n if key and key not in seen:\n seen.add(key)\n unique.append(post)\n else:\n duplicates += 1\n \n if duplicates > 0:\n print(f\" ๐ Removed {duplicates} duplicate posts\")\n \n return unique\n \n def process_comments(self, comments):\n \"\"\"Remove duplicate comments.\"\"\"\n seen = set()\n unique = []\n \n for comment in comments:\n key = comment.get('comment_id')\n if key and key not in seen:\n seen.add(key)\n unique.append(comment)\n \n return unique\n"
},
{
"path": "plugins/keyword_extractor.py",
"content": "\"\"\"\nKeyword Extractor Plugin\nExtracts and tags posts with top keywords.\n\"\"\"\nimport sys\nfrom pathlib import Path\n\nsys.path.insert(0, str(Path(__file__).parent.parent))\n\nfrom plugins import Plugin\nfrom analytics.sentiment import extract_keywords\n\n\nclass KeywordExtractor(Plugin):\n \"\"\"Extract and add keywords to posts.\"\"\"\n \n name = \"keyword_extractor\"\n description = \"Adds top keywords to each post\"\n enabled = True\n top_n = 5 # Number of keywords per post\n \n def process_posts(self, posts):\n \"\"\"Add keywords to each post.\"\"\"\n for post in posts:\n text = f\"{post.get('title', '')} {post.get('selftext', '')}\"\n keywords = extract_keywords([text], top_n=self.top_n)\n post['keywords'] = ','.join([kw for kw, count in keywords])\n \n # Also extract global keywords\n all_texts = [f\"{p.get('title', '')} {p.get('selftext', '')}\" for p in posts]\n global_keywords = extract_keywords(all_texts, top_n=10)\n \n print(f\" ๐ท๏ธ Top keywords: {', '.join([kw for kw, _ in global_keywords[:5]])}\")\n \n return posts\n"
},
{
"path": "plugins/sentiment_tagger.py",
"content": "\"\"\"\nSentiment Tagger Plugin\nAdds sentiment scores and labels to posts and comments.\n\"\"\"\nimport sys\nfrom pathlib import Path\n\n# Add parent to path for imports\nsys.path.insert(0, str(Path(__file__).parent.parent))\n\nfrom plugins import Plugin\nfrom analytics.sentiment import analyze_sentiment\n\n\nclass SentimentTagger(Plugin):\n \"\"\"Add sentiment analysis to scraped content.\"\"\"\n \n name = \"sentiment_tagger\"\n description = \"Adds sentiment scores and labels to posts\"\n enabled = True\n \n def process_posts(self, posts):\n \"\"\"Add sentiment to posts.\"\"\"\n for post in posts:\n text = f\"{post.get('title', '')} {post.get('selftext', '')}\"\n score, label = analyze_sentiment(text)\n post['sentiment_score'] = score\n post['sentiment_label'] = label\n \n # Count sentiments\n pos = sum(1 for p in posts if p.get('sentiment_label') == 'positive')\n neg = sum(1 for p in posts if p.get('sentiment_label') == 'negative')\n neu = len(posts) - pos - neg\n \n print(f\" ๐ Sentiment: {pos} positive, {neu} neutral, {neg} negative\")\n return posts\n \n def process_comments(self, comments):\n \"\"\"Add sentiment to comments.\"\"\"\n for comment in comments:\n score, label = analyze_sentiment(comment.get('body', ''))\n comment['sentiment_score'] = score\n comment['sentiment_label'] = label\n \n return comments\n"
},
{
"path": "requirements.txt",
"content": "# Core\npandas\nrequests\n\n# Async\naiohttp\naiofiles\n\n# Dashboard\nstreamlit\n\n# Export\nopenpyxl\npyarrow\n\n# REST API\nfastapi\nuvicorn\n\n# System & Analytics\npsutil\nduckdb\n"
},
{
"path": "scheduler/__init__.py",
"content": "# Scheduler module\nfrom .cron import *\n"
},
{
"path": "scheduler/cron.py",
"content": "\"\"\"\nScheduler module - Cron-style scheduling for scrape jobs\n\"\"\"\nimport time\nimport threading\nfrom datetime import datetime, timedelta\nimport json\nfrom pathlib import Path\nimport sys\n\nclass CronScheduler:\n \"\"\"Simple cron-style scheduler for Reddit scraping jobs.\"\"\"\n \n def __init__(self):\n self.jobs = []\n self.running = False\n self.thread = None\n \n def add_job(self, target, mode='full', limit=100, is_user=False, \n interval_minutes=60, run_at_start=True):\n \"\"\"\n Add a scheduled scraping job.\n \n Args:\n target: Subreddit or username\n mode: 'full', 'history', or 'monitor'\n limit: Post limit per run\n is_user: True if target is a user\n interval_minutes: Minutes between runs\n run_at_start: Run immediately when scheduler starts\n \"\"\"\n job = {\n 'id': len(self.jobs) + 1,\n 'target': target,\n 'mode': mode,\n 'limit': limit,\n 'is_user': is_user,\n 'interval_minutes': interval_minutes,\n 'run_at_start': run_at_start,\n 'last_run': None,\n 'next_run': datetime.now() if run_at_start else datetime.now() + timedelta(minutes=interval_minutes),\n 'enabled': True,\n 'run_count': 0\n }\n self.jobs.append(job)\n print(f\"๐
Added job #{job['id']}: {'u/' if is_user else 'r/'}{target} every {interval_minutes}min\")\n return job['id']\n \n def remove_job(self, job_id):\n \"\"\"Remove a scheduled job.\"\"\"\n self.jobs = [j for j in self.jobs if j['id'] != job_id]\n print(f\"๐๏ธ Removed job #{job_id}\")\n \n def disable_job(self, job_id):\n \"\"\"Temporarily disable a job.\"\"\"\n for job in self.jobs:\n if job['id'] == job_id:\n job['enabled'] = False\n print(f\"โธ๏ธ Disabled job #{job_id}\")\n \n def enable_job(self, job_id):\n \"\"\"Enable a disabled job.\"\"\"\n for job in self.jobs:\n if job['id'] == job_id:\n job['enabled'] = True\n print(f\"โถ๏ธ Enabled job #{job_id}\")\n \n def list_jobs(self):\n \"\"\"List all scheduled jobs.\"\"\"\n print(\"\\n๐ Scheduled Jobs:\")\n print(\"-\" * 60)\n for job in self.jobs:\n status = \"โ
\" if job['enabled'] else \"โธ๏ธ\"\n prefix = \"u/\" if job['is_user'] else \"r/\"\n next_run = job['next_run'].strftime(\"%H:%M:%S\") if job['next_run'] else \"Never\"\n print(f\"{status} #{job['id']} | {prefix}{job['target']} | \"\n f\"Every {job['interval_minutes']}min | Next: {next_run} | \"\n f\"Runs: {job['run_count']}\")\n print()\n return self.jobs\n \n def _run_job(self, job):\n \"\"\"Execute a single job.\"\"\"\n # Import here to avoid circular imports\n try:\n from main import run_full_history\n \n prefix = \"u/\" if job['is_user'] else \"r/\"\n print(f\"\\n๐ Running scheduled job: {prefix}{job['target']}\")\n \n run_full_history(\n job['target'],\n job['limit'],\n job['is_user'],\n download_media_flag=(job['mode'] == 'full'),\n scrape_comments_flag=(job['mode'] == 'full')\n )\n \n job['last_run'] = datetime.now()\n job['run_count'] += 1\n print(f\"โ
Job completed: {prefix}{job['target']}\")\n \n except Exception as e:\n print(f\"โ Job failed: {e}\")\n \n def _scheduler_loop(self):\n \"\"\"Main scheduler loop.\"\"\"\n print(\"๐ Scheduler started\")\n \n while self.running:\n now = datetime.now()\n \n for job in self.jobs:\n if not job['enabled']:\n continue\n \n if job['next_run'] and now >= job['next_run']:\n self._run_job(job)\n job['next_run'] = now + timedelta(minutes=job['interval_minutes'])\n \n # Check every 30 seconds\n time.sleep(30)\n \n print(\"๐ Scheduler stopped\")\n \n def start(self):\n \"\"\"Start the scheduler in background.\"\"\"\n if self.running:\n print(\"โ ๏ธ Scheduler already running\")\n return\n \n self.running = True\n self.thread = threading.Thread(target=self._scheduler_loop, daemon=True)\n self.thread.start()\n print(\"โ
Scheduler started in background\")\n \n def stop(self):\n \"\"\"Stop the scheduler.\"\"\"\n self.running = False\n if self.thread:\n self.thread.join(timeout=5)\n print(\"๐ Scheduler stopped\")\n \n def save_jobs(self, filepath='scheduler_jobs.json'):\n \"\"\"Save jobs to file.\"\"\"\n jobs_data = []\n for job in self.jobs:\n job_copy = job.copy()\n job_copy['last_run'] = job_copy['last_run'].isoformat() if job_copy['last_run'] else None\n job_copy['next_run'] = job_copy['next_run'].isoformat() if job_copy['next_run'] else None\n jobs_data.append(job_copy)\n \n with open(filepath, 'w') as f:\n json.dump(jobs_data, f, indent=2)\n print(f\"๐พ Saved {len(self.jobs)} jobs to {filepath}\")\n \n def load_jobs(self, filepath='scheduler_jobs.json'):\n \"\"\"Load jobs from file.\"\"\"\n if not Path(filepath).exists():\n print(\"โ ๏ธ No saved jobs found\")\n return\n \n with open(filepath, 'r') as f:\n jobs_data = json.load(f)\n \n for job_data in jobs_data:\n if job_data['last_run']:\n job_data['last_run'] = datetime.fromisoformat(job_data['last_run'])\n if job_data['next_run']:\n job_data['next_run'] = datetime.fromisoformat(job_data['next_run'])\n self.jobs.append(job_data)\n \n print(f\"๐ Loaded {len(jobs_data)} jobs from {filepath}\")\n\n\n# Simple interval-based scheduler for CLI\ndef run_scheduled(target, interval_minutes, mode='full', limit=100, is_user=False):\n \"\"\"\n Run a scrape job on a schedule.\n \n Args:\n target: Subreddit or username\n interval_minutes: Minutes between runs\n mode: 'full', 'history', or 'monitor'\n limit: Post limit per run\n is_user: True if target is a user\n \"\"\"\n from main import run_full_history\n \n prefix = \"u/\" if is_user else \"r/\"\n print(f\"๐
Scheduled: {prefix}{target} every {interval_minutes} minutes\")\n print(\"Press Ctrl+C to stop\\n\")\n \n run_count = 0\n \n try:\n while True:\n run_count += 1\n print(f\"\\n{'='*50}\")\n print(f\"๐ Run #{run_count} - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")\n print(f\"{'='*50}\")\n \n run_full_history(\n target,\n limit,\n is_user,\n download_media_flag=(mode == 'full'),\n scrape_comments_flag=(mode == 'full')\n )\n \n print(f\"\\nโฐ Next run in {interval_minutes} minutes...\")\n time.sleep(interval_minutes * 60)\n \n except KeyboardInterrupt:\n print(f\"\\n\\n๐ Scheduler stopped after {run_count} runs\")\n"
},
{
"path": "scraper/__init__.py",
"content": "# Scraper module\nfrom .async_scraper import run_async_scraper, scrape_async\n"
},
{
"path": "scraper/async_scraper.py",
"content": "\"\"\"\nAsync Reddit Scraper - 10x Speed Boost with aiohttp\n\"\"\"\nimport asyncio\nimport aiohttp\nimport aiofiles\nimport pandas as pd\nimport datetime\nimport time\nimport os\nimport random\nfrom pathlib import Path\nfrom urllib.parse import urlparse\nimport sys\n\nsys.path.insert(0, str(Path(__file__).parent.parent))\nfrom config import USER_AGENT, MIRRORS, ASYNC_MAX_CONCURRENT, ASYNC_BATCH_SIZE\nimport subprocess\nimport tempfile\n\n# Semaphore to limit concurrent requests\nsemaphore = None\n\nasync def fetch_json(session, url, retries=3):\n \"\"\"Fetch JSON with retry logic.\"\"\"\n for attempt in range(retries):\n try:\n async with session.get(url, timeout=aiohttp.ClientTimeout(total=15)) as response:\n if response.status == 200:\n return await response.json()\n elif response.status == 429: # Rate limited\n await asyncio.sleep(5 * (attempt + 1))\n except Exception as e:\n if attempt < retries - 1:\n await asyncio.sleep(2)\n return None\n\nasync def fetch_posts_page(session, base_url, target, after=None, is_user=False, batch_size=100):\n \"\"\"Fetch a single page of posts.\"\"\"\n if is_user:\n path = f\"/user/{target}/submitted.json\"\n else:\n path = f\"/r/{target}/new.json\"\n \n url = f\"{base_url}{path}?limit={batch_size}&raw_json=1\"\n if after:\n url += f\"&after={after}\"\n \n return await fetch_json(session, url)\n\nasync def download_media_async(session, url, save_path):\n \"\"\"Download media file asynchronously.\"\"\"\n global semaphore\n \n if os.path.exists(save_path):\n return True\n \n async with semaphore:\n try:\n async with session.get(url, timeout=aiohttp.ClientTimeout(total=60)) as response:\n if response.status == 200:\n async with aiofiles.open(save_path, 'wb') as f:\n async for chunk in response.content.iter_chunked(8192):\n await f.write(chunk)\n return True\n except:\n pass\n return False\n\nasync def download_reddit_video_with_audio_async(session, video_url, save_path):\n \"\"\"\n Downloads Reddit video with audio asynchronously.\n Reddit stores video and audio separately - this combines them using ffmpeg.\n \"\"\"\n global semaphore\n \n if os.path.exists(save_path):\n return True\n \n async with semaphore:\n try:\n # Find audio URL by replacing video quality with audio\n base_url = video_url.rsplit('/', 1)[0]\n audio_urls = [\n f\"{base_url}/DASH_audio.mp4\",\n f\"{base_url}/DASH_AUDIO_128.mp4\",\n f\"{base_url}/DASH_AUDIO_64.mp4\",\n f\"{base_url}/audio.mp4\",\n f\"{base_url}/audio\"\n ]\n \n # Download video to temp file\n video_temp = tempfile.NamedTemporaryFile(suffix='_video.mp4', delete=False)\n video_temp_path = video_temp.name\n video_temp.close()\n \n try:\n async with session.get(video_url, timeout=aiohttp.ClientTimeout(total=60)) as response:\n if response.status != 200:\n return False\n async with aiofiles.open(video_temp_path, 'wb') as f:\n async for chunk in response.content.iter_chunked(8192):\n await f.write(chunk)\n except:\n if os.path.exists(video_temp_path):\n os.unlink(video_temp_path)\n return False\n \n # Try to download audio\n audio_temp_path = None\n for audio_url in audio_urls:\n try:\n async with session.get(audio_url, timeout=aiohttp.ClientTimeout(total=30)) as response:\n if response.status == 200:\n audio_temp = tempfile.NamedTemporaryFile(suffix='_audio.mp4', delete=False)\n audio_temp_path = audio_temp.name\n audio_temp.close()\n async with aiofiles.open(audio_temp_path, 'wb') as f:\n async for chunk in response.content.iter_chunked(8192):\n await f.write(chunk)\n break\n except:\n continue\n \n if audio_temp_path:\n # Merge video and audio using ffmpeg\n try:\n cmd = [\n 'ffmpeg', '-y', '-hide_banner', '-loglevel', 'error',\n '-i', video_temp_path,\n '-i', audio_temp_path,\n '-c:v', 'copy', '-c:a', 'aac',\n '-shortest', save_path\n ]\n proc = await asyncio.create_subprocess_exec(\n *cmd,\n stdout=asyncio.subprocess.PIPE,\n stderr=asyncio.subprocess.PIPE\n )\n await asyncio.wait_for(proc.wait(), timeout=120)\n \n if proc.returncode == 0:\n os.unlink(video_temp_path)\n os.unlink(audio_temp_path)\n return True\n else:\n # ffmpeg failed, use video only\n os.rename(video_temp_path, save_path)\n os.unlink(audio_temp_path)\n return True\n except FileNotFoundError:\n # ffmpeg not installed\n os.rename(video_temp_path, save_path)\n if audio_temp_path and os.path.exists(audio_temp_path):\n os.unlink(audio_temp_path)\n return True\n except Exception:\n os.rename(video_temp_path, save_path)\n if audio_temp_path and os.path.exists(audio_temp_path):\n os.unlink(audio_temp_path)\n return True\n else:\n # No audio found, just use video\n os.rename(video_temp_path, save_path)\n return True\n \n except Exception:\n pass\n return False\n\nasync def fetch_comments_async(session, permalink):\n \"\"\"Fetch comments asynchronously.\"\"\"\n global semaphore\n \n async with semaphore:\n url = f\"https://old.reddit.com{permalink}.json?limit=100\"\n data = await fetch_json(session, url)\n \n if data and len(data) > 1:\n return parse_comments_sync(data[1]['data']['children'], permalink)\n return []\n\ndef parse_comments_sync(comment_list, post_permalink, depth=0, max_depth=3):\n \"\"\"Parse comments (sync helper).\"\"\"\n comments = []\n \n if depth > max_depth:\n return comments\n \n for item in comment_list:\n if item['kind'] != 't1':\n continue\n \n c = item['data']\n comments.append({\n \"post_permalink\": post_permalink,\n \"comment_id\": c.get('id'),\n \"parent_id\": c.get('parent_id'),\n \"author\": c.get('author'),\n \"body\": c.get('body', ''),\n \"score\": c.get('score', 0),\n \"created_utc\": datetime.datetime.fromtimestamp(c.get('created_utc', 0)).isoformat(),\n \"depth\": depth,\n \"is_submitter\": c.get('is_submitter', False),\n })\n \n replies = c.get('replies')\n if replies and isinstance(replies, dict):\n comments.extend(parse_comments_sync(\n replies.get('data', {}).get('children', []),\n post_permalink, depth + 1, max_depth\n ))\n \n return comments\n\ndef extract_media_urls(post_data):\n \"\"\"Extract all media URLs from a post.\"\"\"\n media = {\"images\": [], \"videos\": [], \"galleries\": []}\n \n url = post_data.get('url', '')\n \n if any(ext in url.lower() for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp']):\n media[\"images\"].append(url)\n \n if 'i.redd.it' in url:\n media[\"images\"].append(url)\n \n if post_data.get('is_video'):\n reddit_video = post_data.get('media', {})\n if reddit_video and 'reddit_video' in reddit_video:\n video_url = reddit_video['reddit_video'].get('fallback_url', '')\n if video_url:\n media[\"videos\"].append(video_url.split('?')[0])\n \n preview = post_data.get('preview', {})\n if preview and 'images' in preview:\n for img in preview['images']:\n source = img.get('source', {})\n if source.get('url'):\n media[\"images\"].append(source['url'].replace('&', '&'))\n \n if post_data.get('is_gallery'):\n gallery_data = post_data.get('gallery_data', {})\n media_metadata = post_data.get('media_metadata', {})\n \n if gallery_data and media_metadata:\n for item in gallery_data.get('items', []):\n media_id = item.get('media_id')\n if media_id and media_id in media_metadata:\n meta = media_metadata[media_id]\n if meta.get('s', {}).get('u'):\n media[\"galleries\"].append(meta['s']['u'].replace('&', '&'))\n \n return media\n\ndef extract_post_data(p):\n \"\"\"Extract post data from JSON.\"\"\"\n post_type = \"text\"\n if p.get('is_video'):\n post_type = \"video\"\n elif p.get('is_gallery'):\n post_type = \"gallery\"\n elif any(ext in p.get('url', '').lower() for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp']) or 'i.redd.it' in p.get('url', ''):\n post_type = \"image\"\n elif p.get('is_self'):\n post_type = \"text\"\n else:\n post_type = \"link\"\n \n return {\n \"id\": p.get('id'),\n \"title\": p.get('title'),\n \"author\": p.get('author'),\n \"created_utc\": datetime.datetime.fromtimestamp(p.get('created_utc', 0)).isoformat(),\n \"permalink\": p.get('permalink'),\n \"url\": p.get('url_overridden_by_dest', p.get('url')),\n \"score\": p.get('score', 0),\n \"upvote_ratio\": p.get('upvote_ratio', 0),\n \"num_comments\": p.get('num_comments', 0),\n \"num_crossposts\": p.get('num_crossposts', 0),\n \"selftext\": p.get('selftext', ''),\n \"post_type\": post_type,\n \"is_nsfw\": p.get('over_18', False),\n \"is_spoiler\": p.get('spoiler', False),\n \"flair\": p.get('link_flair_text', ''),\n \"total_awards\": p.get('total_awards_received', 0),\n \"has_media\": p.get('is_video', False) or p.get('is_gallery', False) or 'i.redd.it' in p.get('url', ''),\n \"media_downloaded\": False,\n \"source\": \"Async-Scraper\"\n }\n\nasync def scrape_async(target, limit=100, is_user=False, download_media=True, scrape_comments=True):\n \"\"\"\n Main async scraping function.\n \n Args:\n target: Subreddit or username\n limit: Max posts to scrape\n is_user: True if scraping a user\n download_media: Download images/videos\n scrape_comments: Scrape comments\n \"\"\"\n global semaphore\n semaphore = asyncio.Semaphore(ASYNC_MAX_CONCURRENT)\n \n prefix = \"u\" if is_user else \"r\"\n print(f\"๐ ASYNC Scraper starting for {prefix}/{target}\")\n print(f\" Target: {limit} posts | Media: {download_media} | Comments: {scrape_comments}\")\n print(f\" Concurrency: {ASYNC_MAX_CONCURRENT} simultaneous requests\")\n print(\"-\" * 50)\n \n # Setup directories\n base_dir = f\"data/{prefix}_{target}\"\n media_dir = f\"{base_dir}/media\"\n images_dir = f\"{media_dir}/images\"\n videos_dir = f\"{media_dir}/videos\"\n \n for d in [base_dir, media_dir, images_dir, videos_dir]:\n os.makedirs(d, exist_ok=True)\n \n start_time = time.time()\n all_posts = []\n all_comments = []\n media_tasks = []\n seen_permalinks = set()\n \n # Load existing data\n posts_file = f\"{base_dir}/posts.csv\"\n if os.path.exists(posts_file):\n try:\n df = pd.read_csv(posts_file)\n seen_permalinks = set(df['permalink'].astype(str).tolist())\n print(f\"๐ Loaded {len(seen_permalinks)} existing posts\")\n except:\n pass\n \n async with aiohttp.ClientSession(headers={\"User-Agent\": USER_AGENT}) as session:\n after = None\n total_fetched = 0\n \n while total_fetched < limit:\n # Try mirrors\n mirrors = MIRRORS.copy()\n random.shuffle(mirrors)\n \n data = None\n for mirror in mirrors:\n # Use proper batch size\n batch_size = min(100, limit - total_fetched)\n data = await fetch_posts_page(session, mirror, target, after, is_user, batch_size)\n if data:\n print(f\"โ
Fetched from {mirror}\")\n break\n \n if not data:\n print(\"โ All mirrors failed\")\n break\n \n children = data.get('data', {}).get('children', [])\n if not children:\n print(\"๐ No more posts\")\n break\n \n print(f\" Processing {len(children)} posts...\")\n \n # Process posts\n batch_posts = []\n comment_tasks = []\n \n for child in children:\n p = child['data']\n post = extract_post_data(p)\n \n if post['permalink'] in seen_permalinks:\n continue\n \n seen_permalinks.add(post['permalink'])\n batch_posts.append(post)\n \n # Queue media downloads\n if download_media:\n media = extract_media_urls(p)\n \n for i, img_url in enumerate(media['images'][:5]):\n ext = os.path.splitext(urlparse(img_url).path)[1] or '.jpg'\n save_path = f\"{images_dir}/{post['id']}_{i}{ext}\"\n media_tasks.append(download_media_async(session, img_url, save_path))\n \n for i, img_url in enumerate(media['galleries'][:10]):\n save_path = f\"{images_dir}/{post['id']}_gallery_{i}.jpg\"\n media_tasks.append(download_media_async(session, img_url, save_path))\n \n for i, vid_url in enumerate(media['videos'][:2]):\n if 'youtube' not in vid_url:\n save_path = f\"{videos_dir}/{post['id']}_{i}.mp4\"\n # Use enhanced download for Reddit videos (includes audio)\n if 'v.redd.it' in vid_url or 'reddit.com' in vid_url:\n media_tasks.append(download_reddit_video_with_audio_async(session, vid_url, save_path))\n else:\n media_tasks.append(download_media_async(session, vid_url, save_path))\n \n # Queue comment fetching\n if scrape_comments and post['num_comments'] > 0:\n comment_tasks.append(fetch_comments_async(session, post['permalink']))\n \n all_posts.extend(batch_posts)\n total_fetched += len(batch_posts)\n \n # Fetch comments in parallel\n if comment_tasks:\n print(f\" ๐ฌ Fetching comments for {len(comment_tasks)} posts...\")\n comment_results = await asyncio.gather(*comment_tasks, return_exceptions=True)\n for result in comment_results:\n if isinstance(result, list):\n all_comments.extend(result)\n \n print(f\" ๐ Progress: {total_fetched}/{limit} posts | {len(all_comments)} comments\")\n \n after = data.get('data', {}).get('after')\n if not after:\n print(\"๐ Reached end of available posts\")\n break\n \n await asyncio.sleep(1) # Small delay between pages\n \n # Download all media in parallel\n if media_tasks:\n print(f\"\\n๐ผ๏ธ Downloading {len(media_tasks)} media files in parallel...\")\n media_results = await asyncio.gather(*media_tasks, return_exceptions=True)\n downloaded = sum(1 for r in media_results if r is True)\n print(f\" โ
Downloaded {downloaded}/{len(media_tasks)} files\")\n \n # Save data\n if all_posts:\n df = pd.DataFrame(all_posts)\n if os.path.exists(posts_file):\n df.to_csv(posts_file, mode='a', header=False, index=False)\n else:\n df.to_csv(posts_file, index=False)\n print(f\"\\n๐พ Saved {len(all_posts)} posts to {posts_file}\")\n \n if all_comments:\n comments_file = f\"{base_dir}/comments.csv\"\n df = pd.DataFrame(all_comments)\n if os.path.exists(comments_file):\n df.to_csv(comments_file, mode='a', header=False, index=False)\n else:\n df.to_csv(comments_file, index=False)\n print(f\"๐พ Saved {len(all_comments)} comments\")\n \n duration = time.time() - start_time\n \n print(\"\\n\" + \"=\" * 50)\n print(\"โ
ASYNC SCRAPE COMPLETE!\")\n print(f\" ๐ Posts: {len(all_posts)}\")\n print(f\" ๐ฌ Comments: {len(all_comments)}\")\n print(f\" ๐ผ๏ธ Media: {len(media_tasks)} queued\")\n print(f\" โฑ๏ธ Duration: {duration:.1f}s\")\n print(f\" โก Speed: {len(all_posts) / duration:.1f} posts/sec\")\n \n return {\n 'posts': len(all_posts),\n 'comments': len(all_comments),\n 'duration': duration\n }\n\ndef run_async_scraper(target, limit=100, is_user=False, download_media=True, scrape_comments=True):\n \"\"\"Wrapper to run async scraper from sync code.\"\"\"\n return asyncio.run(scrape_async(target, limit, is_user, download_media, scrape_comments))\n\n# CLI for testing\nif __name__ == \"__main__\":\n import argparse\n \n parser = argparse.ArgumentParser(description=\"Async Reddit Scraper\")\n parser.add_argument(\"target\", help=\"Subreddit or username\")\n parser.add_argument(\"--limit\", type=int, default=100)\n parser.add_argument(\"--user\", action=\"store_true\")\n parser.add_argument(\"--no-media\", action=\"store_true\")\n parser.add_argument(\"--no-comments\", action=\"store_true\")\n \n args = parser.parse_args()\n \n run_async_scraper(\n args.target,\n args.limit,\n args.user,\n not args.no_media,\n not args.no_comments\n )\n"
},
{
"path": "search/__init__.py",
"content": "# Search module\nfrom .query import *\n"
},
{
"path": "search/query.py",
"content": "\"\"\"\nSearch & Query module - Search and filter scraped data\n\"\"\"\nimport pandas as pd\nfrom pathlib import Path\nfrom datetime import datetime\nimport re\n\ndef search_csv(filepath, query=None, column=None, min_score=None, max_score=None,\n start_date=None, end_date=None, post_type=None, author=None, limit=50):\n \"\"\"\n Search within a CSV file with various filters.\n \n Args:\n filepath: Path to CSV file\n query: Text to search for\n column: Specific column to search in (default: all text columns)\n min_score: Minimum score filter\n max_score: Maximum score filter\n start_date: Start date (YYYY-MM-DD)\n end_date: End date (YYYY-MM-DD)\n post_type: Filter by post type (image, video, text, etc.)\n author: Filter by author\n limit: Maximum results to return\n \n Returns:\n DataFrame with matching results\n \"\"\"\n if not Path(filepath).exists():\n print(f\"โ File not found: {filepath}\")\n return pd.DataFrame()\n \n df = pd.read_csv(filepath)\n \n # Text search\n if query:\n if column and column in df.columns:\n mask = df[column].astype(str).str.contains(query, case=False, na=False)\n else:\n # Search in all text columns\n text_cols = ['title', 'selftext', 'body']\n mask = pd.Series([False] * len(df))\n for col in text_cols:\n if col in df.columns:\n mask |= df[col].astype(str).str.contains(query, case=False, na=False)\n df = df[mask]\n \n # Score filter\n if min_score is not None and 'score' in df.columns:\n df = df[df['score'] >= min_score]\n if max_score is not None and 'score' in df.columns:\n df = df[df['score'] <= max_score]\n \n # Date filter\n if 'created_utc' in df.columns:\n if start_date:\n df = df[df['created_utc'] >= start_date]\n if end_date:\n df = df[df['created_utc'] <= end_date]\n \n # Post type filter\n if post_type and 'post_type' in df.columns:\n df = df[df['post_type'] == post_type]\n \n # Author filter\n if author and 'author' in df.columns:\n df = df[df['author'] == author]\n \n return df.head(limit)\n\ndef search_all_data(data_dir='data', query=None, **kwargs):\n \"\"\"\n Search across all scraped data.\n \n Args:\n data_dir: Data directory path\n query: Text to search for\n **kwargs: Additional filters passed to search_csv\n \n Returns:\n Dictionary with results from each subreddit\n \"\"\"\n results = {}\n data_path = Path(data_dir)\n \n if not data_path.exists():\n print(f\"โ Data directory not found: {data_dir}\")\n return results\n \n # Find all posts.csv files\n for sub_dir in data_path.iterdir():\n if sub_dir.is_dir():\n posts_file = sub_dir / 'posts.csv'\n if posts_file.exists():\n df = search_csv(str(posts_file), query=query, **kwargs)\n if len(df) > 0:\n results[sub_dir.name] = df\n \n # Also check legacy format\n for csv_file in data_path.glob('*.csv'):\n if csv_file.stem not in [r.replace('r_', '').replace('u_', '') for r in results.keys()]:\n df = search_csv(str(csv_file), query=query, **kwargs)\n if len(df) > 0:\n results[csv_file.stem] = df\n \n return results\n\ndef print_search_results(results, show_preview=True):\n \"\"\"Pretty print search results.\"\"\"\n total = sum(len(df) for df in results.values())\n \n print(f\"\\n๐ Found {total} results across {len(results)} sources\\n\")\n print(\"=\" * 70)\n \n for source, df in results.items():\n print(f\"\\n๐ {source} ({len(df)} matches)\")\n print(\"-\" * 50)\n \n for _, row in df.iterrows():\n title = str(row.get('title', row.get('body', 'N/A')))[:60]\n score = row.get('score', 0)\n date = str(row.get('created_utc', ''))[:10]\n \n print(f\" [{score:>4}โฌ] {title}...\")\n if show_preview and 'selftext' in row and row['selftext']:\n preview = str(row['selftext'])[:100].replace('\\n', ' ')\n print(f\" โโ {preview}...\")\n print()\n\ndef advanced_search(data_dir='data', query=None, regex=False, sort_by='score', \n ascending=False, **kwargs):\n \"\"\"\n Advanced search with regex support and sorting.\n \n Args:\n data_dir: Data directory path\n query: Search query (text or regex pattern)\n regex: Treat query as regex pattern\n sort_by: Column to sort results by\n ascending: Sort ascending (default: descending)\n **kwargs: Additional filters\n \n Returns:\n Combined DataFrame of all results\n \"\"\"\n all_results = []\n data_path = Path(data_dir)\n \n for sub_dir in data_path.iterdir():\n if sub_dir.is_dir():\n posts_file = sub_dir / 'posts.csv'\n if posts_file.exists():\n df = pd.read_csv(posts_file)\n df['source'] = sub_dir.name\n all_results.append(df)\n \n if not all_results:\n return pd.DataFrame()\n \n combined = pd.concat(all_results, ignore_index=True)\n \n # Apply query\n if query:\n if regex:\n pattern = query\n else:\n pattern = re.escape(query)\n \n mask = pd.Series([False] * len(combined))\n for col in ['title', 'selftext']:\n if col in combined.columns:\n mask |= combined[col].astype(str).str.contains(pattern, case=False, na=False, regex=True)\n combined = combined[mask]\n \n # Apply other filters\n if kwargs.get('min_score') and 'score' in combined.columns:\n combined = combined[combined['score'] >= kwargs['min_score']]\n \n if kwargs.get('author') and 'author' in combined.columns:\n combined = combined[combined['author'] == kwargs['author']]\n \n if kwargs.get('post_type') and 'post_type' in combined.columns:\n combined = combined[combined['post_type'] == kwargs['post_type']]\n \n # Sort\n if sort_by in combined.columns:\n combined = combined.sort_values(sort_by, ascending=ascending)\n \n limit = kwargs.get('limit', 100)\n return combined.head(limit)\n\ndef get_top_posts(data_dir='data', n=10, by='score'):\n \"\"\"Get top N posts across all scraped data.\"\"\"\n df = advanced_search(data_dir, sort_by=by, ascending=False, limit=n)\n return df\n\ndef get_recent_posts(data_dir='data', n=10):\n \"\"\"Get most recent posts across all scraped data.\"\"\"\n df = advanced_search(data_dir, sort_by='created_utc', ascending=False, limit=n)\n return df\n\ndef find_author_posts(data_dir='data', author=None):\n \"\"\"Find all posts by a specific author.\"\"\"\n return advanced_search(data_dir, author=author, limit=1000)\n\ndef export_search_results(results, output_path, format='csv'):\n \"\"\"Export search results to file.\"\"\"\n if isinstance(results, dict):\n combined = pd.concat(results.values(), ignore_index=True)\n else:\n combined = results\n \n if format == 'csv':\n combined.to_csv(output_path, index=False)\n elif format == 'json':\n combined.to_json(output_path, orient='records', indent=2)\n elif format == 'excel':\n combined.to_excel(output_path, index=False)\n \n print(f\"๐พ Exported {len(combined)} results to {output_path}\")\n"
}
]