', methods=['GET', 'POST', 'PUT', 'DELETE', 'HEAD', 'OPTIONS'])
def proxy(path):
try:
method = request.method
headers = dict(request.headers)
original_host = headers.get('Host', 'claude.ai')
# 保留原始Cookie
cookies = request.cookies
# 删除可能导致问题的头部
for h in ['Host', 'host', 'X-Real-Ip', 'X-Forwarded-For']:
headers.pop(h, None)
if original_host in PROXY_DOMAINS:
base_url = PROXY_DOMAINS[original_host]
else:
base_url = 'https://claude.ai'
target_url = f"{base_url}/{path}" if path else base_url
if request.query_string:
target_url += f"?{request.query_string.decode('utf-8')}"
target_host = urlparse(base_url).netloc
headers['Host'] = target_host
# 添加Claude特定的头部
if 'claude.ai' in target_host:
headers['Origin'] = 'https://claude.ai'
headers['Referer'] = 'https://claude.ai/'
headers['anthropic-client-platform'] = 'web_claude_ai'
print(f"Proxying {method} request from {original_host} to: {target_url}")
# 更精确的SSE判断条件
is_sse = (
'/completion' in path and # 必须是completion端点
method == 'POST' # 必须是POST请求
) or (
'perplexity_ask' in path # 保持原有的perplexity支持
)
print(f"Is SSE request: {is_sse}")
if is_sse:
return handle_sse_request(method, target_url, headers, cookies)
else:
return handle_normal_request(method, target_url, headers, cookies)
except Exception as e:
print(f"Exception: {str(e)}")
import traceback
traceback.print_exc()
return Response(f"Error: {str(e)}", status=500)
def handle_normal_request(method, target_url, headers, cookies):
"""处理普通请求"""
session = session_manager.get_session()
try:
# 打印请求详情以便调试
print(f"Request headers: {json.dumps(headers, indent=2)}")
if method in ["POST", "PUT", "PATCH", "DELETE"]:
raw_data = request.get_data()
if raw_data:
# 打印请求体以便调试
try:
print(f"Request body: {json.dumps(json.loads(raw_data), indent=2)}")
except:
print(f"Request body (raw): {raw_data[:200]}...")
resp = session.request(
method,
target_url,
data=raw_data,
headers=headers,
cookies=cookies,
timeout=30
)
else:
resp = session.request(
method,
target_url,
headers=headers,
cookies=cookies,
timeout=30
)
else:
resp = session.request(
method,
target_url,
headers=headers,
cookies=cookies,
timeout=30
)
print(f"Response status: {resp.status_code}")
# 打印响应头和内容以便调试
print(f"Response headers: {json.dumps(dict(resp.headers), indent=2)}")
try:
print(f"Response body: {json.dumps(json.loads(resp.text), indent=2)}")
except:
print(f"Response body (raw): {resp.text[:200]}...")
response_headers = {}
for key, value in resp.headers.items():
if key.lower() not in ['content-encoding', 'transfer-encoding', 'connection']:
response_headers[key] = value
return Response(
resp.content,
status=resp.status_code,
headers=response_headers
)
except Exception as e:
print(f"Error in normal request: {str(e)}")
return Response(f"Request failed: {str(e)}", status=500)
def handle_sse_request(method, target_url, headers, cookies):
"""处理 SSE (Server-Sent Events) 请求"""
def generate():
response_queue = Queue()
stop_event = Event()
heartbeat_thread = Thread(target=heartbeat_generator, args=(response_queue, stop_event))
heartbeat_thread.daemon = True
heartbeat_thread.start()
try:
session = session_manager.get_session()
# 打印请求详情以便调试
print(f"SSE Request headers: {json.dumps(headers, indent=2)}")
if method in ["POST", "PUT", "PATCH"]:
raw_data = request.get_data()
# 打印请求体以便调试
try:
print(f"SSE Request body: {json.dumps(json.loads(raw_data), indent=2)}")
except:
print(f"SSE Request body (raw): {raw_data[:200]}...")
resp = session.request(
method,
target_url,
data=raw_data,
headers=headers,
cookies=cookies,
stream=True,
timeout=60
)
else:
resp = session.request(
method,
target_url,
headers=headers,
cookies=cookies,
stream=True,
timeout=60
)
print(f"SSE Response status: {resp.status_code}")
print(f"SSE Response headers: {json.dumps(dict(resp.headers), indent=2)}")
# 如果不是200状态码，返回错误信息
if resp.status_code != 200:
stop_event.set()
error_content = resp.content
try:
error_json = json.loads(error_content)
error_message = error_json.get('error', {}).get('message', str(error_content))
yield f"data: {{\"error\": \"{error_message}\"}}\n\n".encode()
except:
yield f"data: {{\"error\": \"Request failed with status {resp.status_code}\"}}\n\n".encode()
return
# 正常处理流式响应
for chunk in resp.iter_content(chunk_size=1024):
if chunk:
yield chunk
while not response_queue.empty():
response_queue.get()
except Exception as e:
print(f"Error in SSE request: {str(e)}")
yield f"data: {{\"error\": \"SSE request failed: {str(e)}\"}}\n\n".encode()
finally:
stop_event.set()
heartbeat_thread.join(timeout=1)
response_headers = {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'X-Accel-Buffering': 'no'
}
return Response(
stream_with_context(generate()),
status=200,
headers=response_headers
)
if __name__ == '__main__':
import ssl
import os
if not os.path.exists('/opt/cert.pem'):
print("Generating self-signed certificate...")
domains = list(PROXY_DOMAINS.keys())
san_list = ','.join([f'DNS:{domain}' for domain in domains])
os.system(f'openssl req -x509 -newkey rsa:4096 -nodes -out /opt/cert.pem -keyout /opt/key.pem -days 365 -subj "/CN=proxy.local" -addext "subjectAltName={san_list}"')
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
context.load_cert_chain('/opt/cert.pem', '/opt/key.pem')
app.run(host='0.0.0.0', port=443, ssl_context=context, threaded=True)
================================================
FILE: postgresql/root.crt
================================================
-----BEGIN CERTIFICATE-----
MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIICGzCCAaGgAwIBAgIQQdKd0XLq7qeAwSxs6S+HUjAKBggqhkjOPQQDAzBPMQsw
CQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2gg
R3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMjAeFw0yMDA5MDQwMDAwMDBaFw00
MDA5MTcxNjAwMDBaME8xCzAJBgNVBAYTAlVTMSkwJwYDVQQKEyBJbnRlcm5ldCBT
ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEVMBMGA1UEAxMMSVNSRyBSb290IFgyMHYw
EAYHKoZIzj0CAQYFK4EEACIDYgAEzZvVn4CDCuwJSvMWSj5cz3es3mcFDR0HttwW
+1qLFNvicWDEukWVEYmO6gbf9yoWHKS5xcUy4APgHoIYOIvXRdgKam7mAHf7AlF9
ItgKbppbd9/w+kHsOdx1ymgHDB/qo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0T
AQH/BAUwAwEB/zAdBgNVHQ4EFgQUfEKWrt5LSDv6kviejM9ti6lyN5UwCgYIKoZI
zj0EAwMDaAAwZQIwe3lORlCEwkSHRhtFcP9Ymd70/aTSVaYgLXTWNLxBo1BfASdW
tL4ndQavEi51mI38AjEAi/V3bNTIZargCyzuFJ0nN6T5U6VR5CmD1/iQMVtCnwr1
/q4AaOeMSQ+2b1tbFfLn
-----END CERTIFICATE-----
================================================
FILE: pteer.js
================================================
import fs from "fs";
import path from "path";
import puppeteer from "puppeteer-extra";
import StealthPlugin from "puppeteer-extra-plugin-stealth";
import dotenv from "dotenv";
import { fileURLToPath } from "url";
import { dirname, join } from "path";
puppeteer.use(StealthPlugin());
// Load the default .env file
dotenv.config();
if (fs.existsSync(".env.local")) {
console.log("Using .env.local file to supply config environment variables");
const envConfig = dotenv.parse(fs.readFileSync(".env.local"));
for (const k in envConfig) {
process.env[k] = envConfig[k];
}
}
// 从环境变量解析用户名和密码
const usernames = process.env.USERNAMES.split(",");
const passwords = process.env.PASSWORDS.split(",");
const loginUrl = process.env.WEBSITE;
// 每个浏览器实例之间的延迟时间(毫秒)
const delayBetweenInstances = 10000;
//随机等待时间
function delayClick(time) {
return new Promise(function (resolve) {
setTimeout(resolve, time);
});
}
(async () => {
try {
if (usernames.length !== passwords.length) {
console.log(usernames.length, usernames, passwords.length, passwords);
console.log("用户名和密码的数量不匹配！");
return;
}
// 并发启动浏览器实例进行登录
const loginPromises = usernames.map((username, index) => {
const password = passwords[index];
const delay = index * delayBetweenInstances;
return new Promise((resolve, reject) => {
//其实直接使用await就可以了
setTimeout(() => {
launchBrowserForUser(username, password).then(resolve).catch(reject);
}, delay);
});
});
// 等待所有登录操作完成
await Promise.all(loginPromises);
} catch (error) {
// 错误处理逻辑
console.error("发生错误：", error);
}
})();
async function launchBrowserForUser(username, password) {
try {
const browser = await puppeteer.launch({
headless: process.env.ENVIRONMENT !== "dev", // 当ENVIRONMENT不是'dev'时启用无头模式
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-infobars",
"--window-position=0,0",
"--ignore-certifcate-errors",
"--ignore-certifcate-errors-spki-list",
"--disable-web-security",
"--disable-features=IsolateOrigins,site-per-process",
"--allow-running-insecure-content",
"--disable-blink-features=AutomationControlled",
"--no-sandbox",
"--mute-audio",
"--no-zygote",
"--no-xshm",
"--window-size=1920,1080",
"--no-first-run",
"--no-default-browser-check",
"--disable-dev-shm-usage",
"--disable-gpu",
"--enable-webgl",
"--ignore-certificate-errors",
"--lang=en-US,en;q=0.9",
"--password-store=basic",
"--disable-gpu-sandbox",
"--disable-software-rasterizer",
"--disable-background-timer-throttling",
"--disable-backgrounding-occluded-windows",
"--disable-renderer-backgrounding",
"--disable-infobars",
"--disable-breakpad",
"--disable-canvas-aa",
"--disable-2d-canvas-clip-aa",
"--disable-gl-drawing-for-tests",
"--enable-low-end-device-mode",
], //linux需要
defaultViewport: {
width: 1280,
height: 800,
userAgent:
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36",
},
});
const page = await browser.newPage();
// 设置额外的 headers
await page.setExtraHTTPHeaders({
"accept-language": "en-US,en;q=0.9",
});
// 调用封装的反检测函数
await bypassDetection(page);
// 验证 `navigator.webdriver` 属性是否为 undefined
const isWebDriverUndefined = await page.evaluate(() => {
return `${navigator.webdriver}`;
});
console.log("navigator.webdriver is :", isWebDriverUndefined); // 输出应为 true
page.on("pageerror", (error) => {
console.error(`Page error: ${error.message}`);
});
page.on("error", async (error) => {
console.error(`Error: ${error.message}`);
// 检查是否是 localStorage 的访问权限错误
if (
error.message.includes(
"Failed to read the 'localStorage' property from 'Window'"
)
) {
console.log("Trying to refresh the page to resolve the issue...");
await page.reload(); // 刷新页面
// 重新尝试你的操作...
}
});
page.on("console", async (msg) => {
console.log("PAGE LOG:", msg.text());
// 使用一个标志变量来检测是否已经刷新过页面
if (
!page._isReloaded &&
msg.text().includes("the server responded with a status of 429")
) {
// 设置标志变量为 true，表示即将刷新页面
page._isReloaded = true;
//由于油候脚本它这个时候可能会导航到新的网页,会导致直接执行代码报错,所以使用这个来在每个新网页加载之前来执行
await page.evaluateOnNewDocument(() => {
localStorage.setItem("autoLikeEnabled", "false");
});
// 等待一段时间，比如 3 秒
await new Promise((resolve) => setTimeout(resolve, 3000));
console.log("Retrying now...");
// 尝试刷新页面
await page.reload();
}
});
//登录操作
await page.goto(loginUrl, { waitUntil: "networkidle0" });
console.log("登录操作");
// 使用XPath查询找到包含"登录"或"login"文本的按钮
await page.evaluate(() => {
let loginButton = Array.from(document.querySelectorAll("button")).find(
(button) =>
button.textContent.includes("登录") ||
button.textContent.includes("login")
);
// 如果没有找到，尝试根据类名查找
if (!loginButton) {
loginButton = document.querySelector(
".widget-button.btn.btn-primary.btn-small.login-button.btn-icon-text"
);
}
console.log(loginButton);
if (loginButton) {
loginButton.click();
console.log("Login button clicked.");
} else {
console.log("Login button not found.");
}
});
await login(page, username, password);
// 查找具有类名 "avatar" 的 img 元素验证登录是否成功
const avatarImg = await page.$("img.avatar");
if (avatarImg) {
console.log("找到avatarImg，登录成功");
// 可以继续对 avatarImg 进行操作，比如获取其属性等
} else {
console.log("未找到avatarImg，登录失败");
}
//真正执行阅读脚本
// 读取外部脚本文件的内容
const externalScriptPath = path.join(
dirname(fileURLToPath(import.meta.url)),
"external.js"
);
const externalScript = fs.readFileSync(externalScriptPath, "utf8");
// 在每个新的文档加载时执行外部脚本
await page.evaluateOnNewDocument((...args) => {
const [scriptToEval] = args;
eval(scriptToEval);
}, externalScript);
// 添加一个监听器来监听每次页面加载完成的事件
page.on("load", async () => {
// await page.evaluate(externalScript); //因为这个是在页面加载好之后执行的,而脚本是在页面加载好时刻来判断是否要执行，由于已经加载好了，脚本就不会起作用
});
await page.goto("https://linux.do/t/topic/13716/190");
} catch (err) {
console.log(err);
}
}
async function login(page, username, password) {
// 等待用户名输入框加载
await page.waitForSelector("#login-account-name");
// 模拟人类在找到输入框后的短暂停顿
await delayClick(500); // 延迟500毫秒
// 清空输入框并输入用户名
await page.click("#login-account-name", { clickCount: 3 });
await page.type("#login-account-name", username, {
delay: 100,
}); // 输入时在每个按键之间添加额外的延迟
// 等待密码输入框加载
await page.waitForSelector("#login-account-password");
// 模拟人类在输入用户名后的短暂停顿
await delayClick(500);
// 清空输入框并输入密码
await page.click("#login-account-password", { clickCount: 3 });
await page.type("#login-account-password", password, {
delay: 100,
});
// 模拟人类在输入完成后思考的短暂停顿
await delayClick(1000);
// 假设登录按钮的ID是'login-button'，点击登录按钮
await page.waitForSelector("#login-button");
await delayClick(500); // 模拟在点击登录按钮前的短暂停顿
try {
await Promise.all([
page.waitForNavigation({ waitUntil: "domcontentloaded" }), // 等待 页面跳转 DOMContentLoaded 事件
page.click("#login-button"), // 点击登录按钮触发跳转
]); //注意如果登录失败，这里会一直等待跳转，导致脚本执行失败
} catch (error) {
console.error("Navigation timed out in login.:", error);
throw new Error("Navigation timed out in login.");
}
await delayClick(1000);
}
async function bypassDetection(page) {
// 在页面上下文中执行脚本，修改 `navigator.webdriver` 以及其他反检测属性
await page.evaluateOnNewDocument(() => {
// Overwrite the `navigator.webdriver` property to return `undefined`.
Object.defineProperty(navigator, "webdriver", {
get: () => undefined,
});
// Pass the Chrome Test.
window.chrome = {
runtime: {},
// Add more if needed
};
// Pass the Permissions Test.
const originalQuery = window.navigator.permissions.query;
window.navigator.permissions.query = (parameters) =>
parameters.name === "notifications"
? Promise.resolve({ state: Notification.permission })
: originalQuery(parameters);
// Pass the Plugins Length Test.
Object.defineProperty(navigator, "plugins", {
get: () => [1, 2, 3, 4, 5], // Make plugins array non-empty
});
// Pass the Languages Test.
Object.defineProperty(navigator, "languages", {
get: () => ["en-US", "en"],
});
});
}
================================================
FILE: src/browser_retry.js
================================================
================================================
FILE: src/db.js
================================================
// 多数据库工具 (PostgreSQL + MongoDB + MySQL)
import fs from "fs";
import pkg from "pg";
const { Pool } = pkg;
import { MongoClient } from "mongodb";
import mysql from "mysql2/promise";
import dotenv from "dotenv";
dotenv.config();
// Utility function to format error information for better logging
function formatErrorInfo(error) {
if (!error) return { errorMsg: '未知错误', errorCode: '无错误代码' };
let errorMsg = error?.message || error?.toString() || '未知错误';
const errorCode = error?.code || '无错误代码';
// Handle AggregateError specially
if (error instanceof AggregateError && error.errors?.length > 0) {
const innerError = error.errors[0];
const innerMsg = innerError?.message || innerError?.toString() || '内部错误';
errorMsg = `${errorMsg} (${innerMsg})`;
}
return { errorMsg, errorCode };
}
if (fs.existsSync(".env.local")) {
console.log("Using .env.local file to supply config environment variables");
const envConfig = dotenv.parse(fs.readFileSync(".env.local"));
for (const k in envConfig) {
process.env[k] = envConfig[k];
}
} else {
console.log(
"Using .env file to supply config environment variables, you can create a .env.local file to overwrite defaults, it doesn't upload to git"
);
}
// 主数据库连接池 (Aiven PostgreSQL)
const pool = new Pool({
connectionString: process.env.POSTGRES_URI,
max: 5,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 15000, // 增加到15秒，适应跨区域网络延迟
ssl: { rejectUnauthorized: false },
});
// 备用数据库连接池 (CockroachDB)
const cockroachPool = new Pool({
connectionString: process.env.COCKROACH_URI,
max: 3,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 15000, // 增加到15秒，适应跨区域网络延迟
ssl: { rejectUnauthorized: false },
});
// 备用数据库连接池 (Neon)
// 备用数据库连接池 (Neon)
const neonPool = new Pool({
connectionString: process.env.NEON_URI,
max: 3,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 15000, // 增加到15秒，适应跨区域网络延迟
ssl: { rejectUnauthorized: false },
});
// MySQL 连接池 (Aiven MySQL)
let mysqlPool;
// 初始化 MySQL 连接池
async function initMySQL() {
if (process.env.AIVEN_MYSQL_URI && !mysqlPool) {
try {
mysqlPool = mysql.createPool({
uri: process.env.AIVEN_MYSQL_URI,
connectionLimit: 5,
acquireTimeout: 60000,
timeout: 60000,
ssl: { rejectUnauthorized: false },
});
console.log("✅ MySQL 连接池创建成功");
} catch (error) {
console.error("❌ MySQL 连接池创建失败:", error.message);
mysqlPool = null;
}
}
return mysqlPool;
}
// MongoDB 连接
let mongoClient;
let mongoDb;
// 初始化 MongoDB 连接
async function initMongoDB() {
if (process.env.MONGO_URI && !mongoClient) {
try {
mongoClient = new MongoClient(process.env.MONGO_URI, {
maxPoolSize: 5,
serverSelectionTimeoutMS: 5000,
socketTimeoutMS: 10000,
});
// 连接到数据库
await mongoClient.connect();
mongoDb = mongoClient.db("auto_read_posts"); // 使用专门的数据库名
console.log("✅ MongoDB 连接成功");
} catch (error) {
console.error("❌ MongoDB 连接失败:", error.message);
mongoClient = null;
mongoDb = null;
}
}
return mongoDb;
}
// 所有数据库连接池数组 (PostgreSQL)
const allPools = [
//注释以下开启数据库
// { name: "Aiven PostgreSQL", pool: pool },
{ name: "CockroachDB", pool: cockroachPool },
// { name: "Neon", pool: neonPool },
];
// 获取所有数据库连接数组 (包括 MongoDB 和 MySQL)
async function getAllDatabases() {
//注释以下开启数据库
// const mongoDb = await initMongoDB();
// const mysqlPool = await initMySQL();
return [
...allPools,
...(mongoDb ? [{ name: "MongoDB", db: mongoDb, type: "mongo" }] : []),
...(mysqlPool
? [{ name: "Aiven MySQL", pool: mysqlPool, type: "mysql" }]
: []),
];
}
export async function savePosts(posts) {
if (!Array.isArray(posts) || posts.length === 0) {
console.warn("无效的帖子数据或空数组，跳过保存");
return;
}
// 验证帖子数据
const validPosts = posts.filter(post => {
if (!post || !post.guid || typeof post.guid !== 'string' || post.guid.trim() === '') {
console.warn(`跳过无效帖子数据: ${JSON.stringify({title: post?.title, guid: post?.guid})}`);
return false;
}
return true;
});
if (validPosts.length === 0) {
console.warn("没有有效的帖子数据，跳过保存");
return;
}
const allDatabases = await getAllDatabases();
if (allDatabases.length === 0) {
console.warn("没有可用的数据库连接，跳过保存");
return;
}
const createTableQuery = `
CREATE TABLE IF NOT EXISTS posts (
id SERIAL PRIMARY KEY,
title TEXT,
creator TEXT,
description TEXT,
link TEXT,
pubDate TEXT,
guid TEXT UNIQUE,
guidIsPermaLink TEXT,
source TEXT,
sourceUrl TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
`;
const insertQuery = `
INSERT INTO posts (title, creator, description, link, pubDate, guid, guidIsPermaLink, source, sourceUrl)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
ON CONFLICT (guid) DO NOTHING
`;
// 并行操作所有数据库
const savePromises = allDatabases.map(async ({ name, pool, db, type }) => {
try {
console.log(`正在保存到 ${name}...`);
if (type === "mongo" && db) {
// MongoDB 操作
const collection = db.collection("posts");
// 准备 MongoDB 文档
const mongoDocuments = validPosts.map((post) => ({
title: post.title,
creator: post.creator,
description: post.description,
link: post.link,
pubDate: post.pubDate,
guid: post.guid,
guidIsPermaLink: post.guidIsPermaLink,
source: post.source,
sourceUrl: post.sourceUrl,
created_at: new Date(),
}));
// 使用 upsert 操作避免重复
const bulkOps = mongoDocuments.map((doc) => ({
updateOne: {
filter: { guid: doc.guid },
update: { $set: doc },
upsert: true,
},
}));
if (bulkOps.length > 0) {
await collection.bulkWrite(bulkOps);
}
} else if (type === "mysql" && pool) {
// MySQL 操作
const mysqlCreateTableQuery = `
CREATE TABLE IF NOT EXISTS posts (
id INT AUTO_INCREMENT PRIMARY KEY,
title TEXT,
creator TEXT,
description TEXT,
link TEXT,
pubDate TEXT,
guid VARCHAR(500) UNIQUE,
guidIsPermaLink TEXT,
source TEXT,
sourceUrl TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
`;
const mysqlInsertQuery = `
INSERT INTO posts (title, creator, description, link, pubDate, guid, guidIsPermaLink, source, sourceUrl)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
title = VALUES(title),
creator = VALUES(creator),
description = VALUES(description),
link = VALUES(link),
pubDate = VALUES(pubDate),
guidIsPermaLink = VALUES(guidIsPermaLink),
source = VALUES(source),
sourceUrl = VALUES(sourceUrl)
`;
// 建表
await pool.execute(mysqlCreateTableQuery);
// 插入数据
for (const post of validPosts) {
await pool.execute(mysqlInsertQuery, [
post.title,
post.creator,
post.description,
post.link,
post.pubDate,
post.guid,
post.guidIsPermaLink,
post.source,
post.sourceUrl,
]);
}
} else if (pool) {
// PostgreSQL 操作
// 建表
await pool.query(createTableQuery);
// 插入数据
for (const post of validPosts) {
await pool.query(insertQuery, [
post.title,
post.creator,
post.description,
post.link,
post.pubDate,
post.guid,
post.guidIsPermaLink,
post.source,
post.sourceUrl,
]);
}
}
console.log(`✅ ${name} 保存成功 (${validPosts.length} 条记录)`);
return { name, success: true };
} catch (error) {
const { errorMsg, errorCode } = formatErrorInfo(error);
console.error(`❌ ${name} 保存失败 [${errorCode}]:`, errorMsg);
return { name, success: false, error: errorMsg };
}
});
// 等待所有数据库操作完成
const results = await Promise.allSettled(savePromises);
// 统计结果
const successCount = results.filter(
(result) => result.status === "fulfilled" && result.value.success
).length;
console.log(
`数据库保存结果: ${successCount}/${allDatabases.length} 个数据库保存成功`
);
// 如果至少有一个数据库保存成功，就认为操作成功
if (successCount === 0) {
throw new Error("所有数据库保存都失败了");
}
}
export async function isGuidExists(guid) {
// 验证输入参数
if (!guid || typeof guid !== 'string' || guid.trim() === '') {
console.warn(`无效的GUID参数: ${JSON.stringify(guid)}`);
return false;
}
// 优先查询主数据库 (CockroachDB)
try {
const res = await cockroachPool.query(
"SELECT 1 FROM posts WHERE guid = $1 LIMIT 1",
[guid]
);
// console.log("isGuidExists查询结果:", res.rows); 存在的返回[ { '?column?': 1 } ]
if (res.rowCount > 0) {
return true;
}
} catch (error) {
const { errorMsg, errorCode } = formatErrorInfo(error);
console.warn(`主数据库查询GUID失败 [${errorCode}]: ${errorMsg}`);
}
// 如果主数据库查询失败或未找到，尝试查询备用数据库
const allDatabases = await getAllDatabases();
for (const { name, pool, db, type } of allDatabases.slice(1)) {
// 跳过主数据库
try {
if (type === "mongo" && db) {
// MongoDB 查询
const collection = db.collection("posts");
const count = await collection.countDocuments(
{ guid: guid },
{ limit: 1 }
);
if (count > 0) {
console.log(`在备用数据库 ${name} 中找到GUID: ${guid}`);
return true;
}
} else if (type === "mysql" && pool) {
// MySQL 查询
const [rows] = await pool.execute(
"SELECT 1 FROM posts WHERE guid = ? LIMIT 1",
[guid]
);
if (rows.length > 0) {
console.log(`在备用数据库 ${name} 中找到GUID: ${guid}`);
return true;
}
} else if (pool) {
// PostgreSQL 查询
const res = await pool.query(
"SELECT 1 FROM posts WHERE guid = $1 LIMIT 1",
[guid]
);
if (res.rowCount > 0) {
console.log(`在备用数据库 ${name} 中找到GUID: ${guid}`);
return true;
}
}
} catch (error) {
const { errorMsg, errorCode } = formatErrorInfo(error);
console.warn(`备用数据库 ${name} 查询GUID失败 [${errorCode}]: ${errorMsg}`);
}
}
return false;
}
// 测试所有数据库连接
export async function testAllConnections() {
console.log("正在测试所有数据库连接...");
const allDatabases = await getAllDatabases();
const testPromises = allDatabases.map(async ({ name, pool, db, type }) => {
try {
if (type === "mongo" && db) {
// 测试 MongoDB 连接
await db.admin().ping();
} else if (type === "mysql" && pool) {
// 测试 MySQL 连接
await pool.execute("SELECT 1");
} else if (pool) {
// 测试 PostgreSQL 连接
await pool.query("SELECT 1");
}
console.log(`✅ ${name} 连接正常`);
return { name, connected: true };
} catch (error) {
const { errorMsg, errorCode } = formatErrorInfo(error);
console.error(`❌ ${name} 连接失败 [${errorCode}]:`, errorMsg);
return { name, connected: false, error: errorMsg };
}
});
const results = await Promise.allSettled(testPromises);
const connectedCount = results.filter(
(result) => result.status === "fulfilled" && result.value.connected
).length;
console.log(
`数据库连接测试结果: ${connectedCount}/${allDatabases.length} 个数据库连接正常`
);
return results;
}
// 获取所有数据库的统计信息
export async function getAllDatabaseStats() {
console.log("正在获取所有数据库统计信息...");
const allDatabases = await getAllDatabases();
const statsPromises = allDatabases.map(async ({ name, pool, db, type }) => {
try {
let stats;
if (type === "mongo" && db) {
// MongoDB 统计
const collection = db.collection("posts");
const totalPosts = await collection.countDocuments();
const latestPost = await collection.findOne(
{},
{ sort: { created_at: -1 } }
);
stats = {
name,
totalPosts,
latestPost: latestPost?.created_at || null,
status: "healthy",
};
} else if (type === "mysql" && pool) {
// MySQL 统计
const [countResult] = await pool.execute(
"SELECT COUNT(*) as count FROM posts"
);
const [latestResult] = await pool.execute(
"SELECT created_at FROM posts ORDER BY created_at DESC LIMIT 1"
);
stats = {
name,
totalPosts: parseInt(countResult[0].count),
latestPost: latestResult[0]?.created_at || null,
status: "healthy",
};
} else if (pool) {
// PostgreSQL 统计
const countResult = await pool.query(
"SELECT COUNT(*) as count FROM posts"
);
const latestResult = await pool.query(
"SELECT created_at FROM posts ORDER BY created_at DESC LIMIT 1"
);
stats = {
name,
totalPosts: parseInt(countResult.rows[0].count),
latestPost: latestResult.rows[0]?.created_at || null,
status: "healthy",
};
}
console.log(`📊 ${name}: ${stats.totalPosts} 条记录`);
return stats;
} catch (error) {
const { errorMsg, errorCode } = formatErrorInfo(error);
console.error(`❌ ${name} 统计信息获取失败 [${errorCode}]:`, errorMsg);
return {
name,
totalPosts: -1,
latestPost: null,
status: "error",
error: errorMsg,
};
}
});
const results = await Promise.allSettled(statsPromises);
return results.map((result) =>
result.status === "fulfilled" ? result.value : result.reason
);
}
// 关闭所有数据库连接
export async function closeAllConnections() {
console.log("正在关闭所有数据库连接...");
const allDatabases = await getAllDatabases();
const closePromises = allDatabases.map(async ({ name, pool, type }) => {
try {
if (type === "mongo") {
// 关闭 MongoDB 连接
if (mongoClient) {
await mongoClient.close();
mongoClient = null;
mongoDb = null;
}
} else if (type === "mysql" && pool) {
// 关闭 MySQL 连接
await pool.end();
mysqlPool = null;
} else if (pool) {
// 关闭 PostgreSQL 连接
await pool.end();
}
console.log(`✅ ${name} 连接已关闭`);
} catch (error) {
const { errorMsg, errorCode } = formatErrorInfo(error);
console.error(`❌ ${name} 连接关闭失败 [${errorCode}]:`, errorMsg);
}
});
await Promise.allSettled(closePromises);
console.log("所有数据库连接关闭完成");
}
// 保存话题 JSON 数据的函数
export async function saveTopicData(topicData) {
if (!topicData || !topicData.id) {
console.warn("无效的话题数据，跳过保存");
return;
}
const allDatabases = await getAllDatabases();
const createTopicsTableQuery = `
CREATE TABLE IF NOT EXISTS topics (
id SERIAL PRIMARY KEY,
topic_id INTEGER UNIQUE,
title TEXT,
slug TEXT,
posts_count INTEGER,
created_at TIMESTAMP,
last_posted_at TIMESTAMP,
views INTEGER,
like_count INTEGER,
category_id INTEGER,
tags TEXT[],
raw_data JSONB,
saved_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
`;
const insertTopicQuery = `
INSERT INTO topics (topic_id, title, slug, posts_count, created_at, last_posted_at, views, like_count, category_id, tags, raw_data)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
ON CONFLICT (topic_id) DO UPDATE SET
title = EXCLUDED.title,
slug = EXCLUDED.slug,
posts_count = EXCLUDED.posts_count,
last_posted_at = EXCLUDED.last_posted_at,
views = EXCLUDED.views,
like_count = EXCLUDED.like_count,
tags = EXCLUDED.tags,
raw_data = EXCLUDED.raw_data,
saved_at = CURRENT_TIMESTAMP
`;
const savePromises = allDatabases.map(async ({ name, pool, db, type }) => {
try {
console.log(`正在保存话题数据到 ${name}...`);
if (type === "mongo" && db) {
// MongoDB 操作
const collection = db.collection("topics");
const mongoDocument = {
topic_id: topicData.id,
title: topicData.title,
slug: topicData.slug,
posts_count: topicData.posts_count,
created_at: new Date(topicData.created_at),
last_posted_at: topicData.last_posted_at ? new Date(topicData.last_posted_at) : null,
views: topicData.views,
like_count: topicData.like_count,
category_id: topicData.category_id,
tags: topicData.tags || [],
raw_data: topicData,
saved_at: new Date(),
};
await collection.updateOne(
{ topic_id: topicData.id },
{ $set: mongoDocument },
{ upsert: true }
);
} else if (type === "mysql" && pool) { // MySQL 操作
const mysqlCreateTableQuery = `
CREATE TABLE IF NOT EXISTS topics (
id INT AUTO_INCREMENT PRIMARY KEY,
topic_id INT UNIQUE,
title TEXT,
slug TEXT,
posts_count INT,
created_at DATETIME,
last_posted_at DATETIME NULL,
views INT,
like_count INT,
category_id INT,
tags JSON,
raw_data JSON,
saved_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
`;
const mysqlInsertQuery = `
INSERT INTO topics (topic_id, title, slug, posts_count, created_at, last_posted_at, views, like_count, category_id, tags, raw_data)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
title = VALUES(title),
slug = VALUES(slug),
posts_count = VALUES(posts_count),
last_posted_at = VALUES(last_posted_at),
views = VALUES(views),
like_count = VALUES(like_count),
tags = VALUES(tags),
raw_data = VALUES(raw_data),
saved_at = CURRENT_TIMESTAMP
`; await pool.execute(mysqlCreateTableQuery);
// 转换日期格式为 MySQL 兼容格式 (YYYY-MM-DD HH:MM:SS)
const formatDateForMySQL = (dateString) => {
if (!dateString) return null;
const date = new Date(dateString);
return date.toISOString().slice(0, 19).replace('T', ' ');
};
await pool.execute(mysqlInsertQuery, [
topicData.id,
topicData.title,
topicData.slug,
topicData.posts_count,
formatDateForMySQL(topicData.created_at),
formatDateForMySQL(topicData.last_posted_at),
topicData.views,
topicData.like_count,
topicData.category_id,
JSON.stringify(topicData.tags || []),
JSON.stringify(topicData),
]);
} else if (pool) {
// PostgreSQL 操作
await pool.query(createTopicsTableQuery);
await pool.query(insertTopicQuery, [
topicData.id,
topicData.title,
topicData.slug,
topicData.posts_count,
topicData.created_at,
topicData.last_posted_at,
topicData.views,
topicData.like_count,
topicData.category_id,
topicData.tags || [],
topicData,
]);
}
console.log(`✅ ${name} 话题数据保存成功 (话题ID: ${topicData.id})`);
return { name, success: true };
} catch (error) {
const { errorMsg, errorCode } = formatErrorInfo(error);
console.error(`❌ ${name} 话题数据保存失败 [${errorCode}]:`, errorMsg);
return { name, success: false, error: errorMsg };
}
});
const results = await Promise.allSettled(savePromises);
const successCount = results.filter(
(result) => result.status === "fulfilled" && result.value.success
).length;
console.log(
`话题数据保存结果: ${successCount}/${allDatabases.length} 个数据库保存成功`
);
if (successCount === 0) {
throw new Error("所有数据库话题数据保存都失败了");
}
}
================================================
FILE: src/db.test.js
================================================
import { savePosts, isGuidExists } from "./db.js";
async function runDbTest() {
const testGuid = "test-guid-" + Date.now();
const testPost = {
title: "GitHub Action 测试标题" + Date.now(),
creator: "测试作者",
description: "测试内容",
link: "https://example.com",
pubDate: new Date().toISOString(),
guid: testGuid,
guidIsPermaLink: "false",
source: "test-source",
sourceUrl: "https://source.com"
};
// 1. 保存帖子
await savePosts([testPost]);
console.log("savePosts 测试通过");
// 2. 检查 guid 是否存在
const exists = await isGuidExists(testGuid);
if (exists) {
console.log("isGuidExists 测试通过");
process.exit(0);
} else {
console.error("isGuidExists 测试失败");
process.exit(1);
}
}
runDbTest().catch((e) => { console.error(e); process.exit(1); });
================================================
FILE: src/format_for_telegram.js
================================================
import fs from 'fs';
const inputFilePath = 'parsed_rss_data.json';
const outputFilePath = 'telegram_message.txt';
// Function to strip HTML tags
function stripHtml(html) {
if (!html) return '';
// First, try to extract content from the first tag
const pMatch = html.match(/
(.*?)<\/p>/i);
if (pMatch && pMatch[1]) {
// Remove any nested tags within the first
return pMatch[1].replace(/<[^>]+>/g, '').trim();
}
// Fallback: remove all HTML tags if no
or if it's empty
return html.replace(/<[^>]+>/g, '').trim();
}
fs.readFile(inputFilePath, 'utf8', (err, data) => {
if (err) {
console.error(`Failed to read ${inputFilePath}:`, err);
return;
}
try {
const jsonData = JSON.parse(data);
let telegramMessage = `*New Posts from "${jsonData.channelTitle}"*\n`;
telegramMessage += `${jsonData.channelLink}\n\n`;
telegramMessage += `Channel Description:\n${jsonData.channelDescription.replace(/\*\*(.*?)\*\*/g, '*$1*')}\n\n`; // Keep markdown bold
telegramMessage += "---\n\n";
if (jsonData.items && Array.isArray(jsonData.items)) {
jsonData.items.forEach(item => {
const descriptionSnippet = stripHtml(item.description);
telegramMessage += `*${item.title}*\n`;
telegramMessage += `By: ${item.creator}\n`;
if (descriptionSnippet) {
telegramMessage += `Content: ${descriptionSnippet}\n`;
}
telegramMessage += `Read more: ${item.link}\n`;
telegramMessage += "---\n\n";
});
}
fs.writeFile(outputFilePath, telegramMessage, (writeErr) => {
if (writeErr) {
console.error(`Failed to write ${outputFilePath}:`, writeErr);
return;
}
console.log(`Successfully formatted message and wrote to ${outputFilePath}`);
});
} catch (parseErr) {
console.error(`Failed to parse JSON from ${inputFilePath}:`, parseErr);
}
});
================================================
FILE: src/parse_rss.js
================================================
import xml2js from "xml2js";
import { savePosts, isGuidExists } from "./db.js";
// 生成适合Telegram的文本内容，去掉“阅读完整话题”和“Read full topic”内容
function cleanDescription(desc) {
if (!desc) return "无内容";
// 去掉HTML标签
let text = desc.replace(/<[^>]+>/g, "");
// 去掉特定内容
text = text.replace(/阅读完整话题/g, "");
text = text.replace(/Read full topic/gi, "");
// 去掉多余的空白字符
text = text.replace(/\s+/g, " ");
return text.trim() || "无内容";
}
/**
* 解析RSS XML并返回适合Telegram的文本内容
* @param {string} xmlData - RSS XML字符串
* @returns {Promise} 适合Telegram的文本内容
*/
export async function parseRss(xmlData) {
const parser = new xml2js.Parser({ explicitArray: false, trim: true });
const result = await parser.parseStringPromise(xmlData);
const channel = result.rss.channel;
const items = channel.item;
const extractedItems = [];
if (items && Array.isArray(items)) {
for (const item of items) {
extractedItems.push({
title: item.title,
creator: item["dc:creator"],
description: item.description,
link: item.link,
pubDate: item.pubDate,
guid: item.guid?._,
guidIsPermaLink: item.guid?.$?.isPermaLink,
source: item.source?._,
sourceUrl: item.source?.$?.url,
});
}
} else if (items) {
extractedItems.push({
title: items.title,
creator: items["dc:creator"],
description: items.description,
link: items.link,
pubDate: items.pubDate,
guid: items.guid?._,
guidIsPermaLink: items.guid?.$?.isPermaLink,
source: items.source?._,
sourceUrl: items.source?.$?.url,
});
} // items反转，最新的内容排在最前面
const reversedItems = extractedItems.reverse();
// 先过滤出新的帖子（不存在的）
const newItems = [];
for (const item of reversedItems) {
if (item.guid) {
const exists = await isGuidExists(item.guid);
if (exists) {
console.warn(`GUID ${item.guid} 已存在，跳过。`);
continue;
}
}
newItems.push(item);
}
// 如果没有新帖子，直接返回空内容
if (newItems.length === 0) {
console.log("没有新帖子需要处理");
return "";
}
// 先保存新帖子到数据库
try {
await savePosts(newItems);
console.log(`成功保存 ${newItems.length} 个新帖子到数据库`);
} catch (e) {
console.error("保存帖子到数据库失败:", e);
// 如果保存失败，不推送任何内容
return "";
}
// 保存成功后，构建推送内容
const textContentArr = [];
for (let idx = 0; idx < newItems.length; idx++) {
const item = newItems[idx];
const isFirst = idx === 0;
const isLast = idx === newItems.length - 1;
const msg = [
isFirst ? `标题: ${item.title}` : "",
`作者: ${item.creator}`,
isFirst
? `内容: ${cleanDescription(item.description)}`
: `回复: ${cleanDescription(item.description)}`,
`时间: ${item.pubDate}`,
isLast ? `链接: ${item.link}` : "",
"",
]
.filter(Boolean)
.join("\n")
.trim();
if (msg) {
textContentArr.push(msg);
}
}
const textContent = textContentArr.filter(Boolean).join("\n");
return textContent;
}
================================================
FILE: src/proxy_config.js
================================================
/**
* 代理配置模块
* 支持HTTP、HTTPS、SOCKS代理
*/
// 代理配置类型枚举
export const ProxyTypes = {
HTTP: 'http',
HTTPS: 'https',
SOCKS4: 'socks4',
SOCKS5: 'socks5'
};
/**
* 解析代理URL
* @param {string} proxyUrl - 代理URL (http://user:pass@host:port 或 socks5://user:pass@host:port)
* @returns {Object} 解析后的代理配置
*/
export function parseProxyUrl(proxyUrl) {
if (!proxyUrl) return null;
try {
const url = new URL(proxyUrl);
const config = {
type: url.protocol.replace(':', ''),
host: url.hostname,
port: parseInt(url.port),
};
if (url.username) {
config.username = url.username;
}
if (url.password) {
config.password = url.password;
}
return config;
} catch (error) {
console.error('代理URL解析错误:', error.message);
return null;
}
}
/**
* 获取代理配置
* @returns {Object|null} 代理配置对象
*/
export function getProxyConfig() {
const proxyUrl = process.env.PROXY_URL;
const proxyHost = process.env.PROXY_HOST;
const proxyPort = process.env.PROXY_PORT;
const proxyType = process.env.PROXY_TYPE || 'http';
const proxyUsername = process.env.PROXY_USERNAME;
const proxyPassword = process.env.PROXY_PASSWORD;
// 优先使用PROXY_URL
if (proxyUrl) {
return parseProxyUrl(proxyUrl);
}
// 使用分离的配置参数
if (proxyHost && proxyPort) {
const config = {
type: proxyType,
host: proxyHost,
port: parseInt(proxyPort),
};
if (proxyUsername) {
config.username = proxyUsername;
}
if (proxyPassword) {
config.password = proxyPassword;
}
return config;
}
return null;
}
/**
* 为Puppeteer生成代理配置
* @param {Object} proxyConfig - 代理配置
* @returns {Object} Puppeteer代理配置
*/
export function getPuppeteerProxyArgs(proxyConfig) {
if (!proxyConfig) return [];
const args = [];
if (proxyConfig.type === 'http' || proxyConfig.type === 'https') {
args.push(`--proxy-server=${proxyConfig.type}://${proxyConfig.host}:${proxyConfig.port}`);
} else if (proxyConfig.type === 'socks4' || proxyConfig.type === 'socks5') {
args.push(`--proxy-server=${proxyConfig.type}://${proxyConfig.host}:${proxyConfig.port}`);
}
return args;
}
/**
* 为Playwright生成代理配置
* @param {Object} proxyConfig - 代理配置
* @returns {Object} Playwright代理配置
*/
export function getPlaywrightProxyConfig(proxyConfig) {
if (!proxyConfig) return null;
const config = {
server: `${proxyConfig.type}://${proxyConfig.host}:${proxyConfig.port}`,
};
if (proxyConfig.username && proxyConfig.password) {
config.username = proxyConfig.username;
config.password = proxyConfig.password;
}
return config;
}
/**
* 测试代理连接
* @param {Object} proxyConfig - 代理配置
* @returns {Promise} 是否连接成功
*/
export async function testProxyConnection(proxyConfig) {
if (!proxyConfig) return false;
try {
const { default: fetch } = await import('node-fetch');
const { HttpsProxyAgent } = await import('https-proxy-agent');
const { SocksProxyAgent } = await import('socks-proxy-agent');
let agent;
if (proxyConfig.type === 'http' || proxyConfig.type === 'https') {
const proxyUrl = proxyConfig.username && proxyConfig.password
? `${proxyConfig.type}://${proxyConfig.username}:${proxyConfig.password}@${proxyConfig.host}:${proxyConfig.port}`
: `${proxyConfig.type}://${proxyConfig.host}:${proxyConfig.port}`;
agent = new HttpsProxyAgent(proxyUrl);
} else if (proxyConfig.type === 'socks4' || proxyConfig.type === 'socks5') {
const proxyUrl = proxyConfig.username && proxyConfig.password
? `${proxyConfig.type}://${proxyConfig.username}:${proxyConfig.password}@${proxyConfig.host}:${proxyConfig.port}`
: `${proxyConfig.type}://${proxyConfig.host}:${proxyConfig.port}`;
agent = new SocksProxyAgent(proxyUrl);
}
const response = await fetch('https://httpbin.org/ip', {
agent,
timeout: 10000
});
if (response.ok) {
const data = await response.json();
console.log('代理测试成功，IP地址:', data.origin);
return true;
}
return false;
} catch (error) {
console.error('代理测试失败:', error.message);
return false;
}
}
/**
* 获取当前IP地址（不使用代理）
*/
export async function getCurrentIP() {
try {
const { default: fetch } = await import('node-fetch');
const response = await fetch('https://httpbin.org/ip', { timeout: 10000 });
if (response.ok) {
const data = await response.json();
return data.origin;
}
} catch (error) {
console.error('获取IP地址失败:', error.message);
}
return null;
}
export default {
ProxyTypes,
parseProxyUrl,
getProxyConfig,
getPuppeteerProxyArgs,
getPlaywrightProxyConfig,
testProxyConnection,
getCurrentIP
};
================================================
FILE: src/topic_data.js
================================================
// 话题数据获取和保存模块
import { saveTopicData } from './db.js';
// 从话题页面提取话题ID
export function extractTopicIdFromUrl(url) {
const match = url.match(/\/t\/topic\/(\d+)/);
return match ? parseInt(match[1]) : null;
}
// 获取话题JSON数据
export async function fetchTopicJson(page, topicId) {
try {
const jsonUrl = `https://linux.do/t/${topicId}.json`;
console.log(`正在获取话题JSON数据: ${jsonUrl}`);
// 在新页面中获取JSON数据
const jsonPage = await page.browser().newPage();
await jsonPage.goto(jsonUrl, {
waitUntil: "domcontentloaded",
timeout: 15000,
});
// 等待页面加载完成
await new Promise((r) => setTimeout(r, 1000));
// 获取JSON内容
const jsonText = await jsonPage.evaluate(() => {
// 尝试从pre标签获取内容（某些浏览器显示JSON的方式）
const pre = document.querySelector('pre');
if (pre) {
return pre.textContent;
}
// 否则获取body的文本内容
return document.body.textContent;
});
await jsonPage.close();
if (!jsonText) {
console.warn(`无法获取话题 ${topicId} 的JSON数据`);
return null;
}
// 解析JSON
const topicData = JSON.parse(jsonText);
console.log(`✅ 成功获取话题 ${topicId} 的JSON数据: ${topicData.title}`);
return topicData;
} catch (error) {
console.error(`获取话题 ${topicId} JSON数据失败:`, error.message);
return null;
}
}
// 处理话题数据并保存到数据库
export async function processAndSaveTopicData(page, url) {
try {
const topicId = extractTopicIdFromUrl(url);
if (!topicId) {
console.warn(`无法从URL提取话题ID: ${url}`);
return;
}
// 获取JSON数据
const topicData = await fetchTopicJson(page, topicId);
if (!topicData) {
return;
}
// 保存到数据库
await saveTopicData(topicData);
console.log(`✅ 话题 ${topicId} 数据处理完成`);
} catch (error) {
console.error(`处理话题数据失败:`, error.message);
}
}
================================================
FILE: telegram_message.txt
================================================
*New Posts from "长期在线回收iPhone"*
https://linux.do/t/topic/525305
Channel Description:
各位佬有二手iPhone想出手的，可以来我这里问问价。 专业二手iPhone贩子，长期在线回收二手iPhone，只要能开机都可以回收。 但是 id机，监管机不收。
所有交易都走闲鱼，安全靠谱。
闲鱼店铺名称： *东莞乐回收数码*
---
*长期在线回收iPhone*
By: KotoriMinami
Content: 好的ok
Read more: https://linux.do/t/topic/525305/293
---
*长期在线回收iPhone*
By: 鲨鱼辣椒
Content: 不到 2个月的 ip16 256G，无磕碰多少钱
Read more: https://linux.do/t/topic/525305/292
---
*长期在线回收iPhone*
By: Kubrick
Content: 挂小红书上 高价卖给小仙女 现在低像素手机 小仙女最爱
Read more: https://linux.do/t/topic/525305/291
---
*长期在线回收iPhone*
By: shenpvip
Content: iphone 6s要吗
Read more: https://linux.do/t/topic/525305/290
---
*长期在线回收iPhone*
By: 二手iPhone贩子
Content: 收， 收的就是 8p， 只要是全原装， 能开机就收。
Read more: https://linux.do/t/topic/525305/289
---
*长期在线回收iPhone*
By: 二手iPhone贩子
Content: 全原装 600 。
Read more: https://linux.do/t/topic/525305/288
---
*长期在线回收iPhone*
By: 沐风
Content: 不知道iphone8还收不，俺的iphone8除了电池不行其他的都还好，不过估计也值不了几个钱了
Read more: https://linux.do/t/topic/525305/287
---
*长期在线回收iPhone*
By: sunny22
Content: 国行xr 128的多少钱收
Read more: https://linux.do/t/topic/525305/286
---
*长期在线回收iPhone*
By: 二手iPhone贩子
Content: 我这边 1450 直接秒了。
Read more: https://linux.do/t/topic/525305/285
---
*长期在线回收iPhone*
By: 二手iPhone贩子
Content: 无磕碰， 4800 回收。
Read more: https://linux.do/t/topic/525305/284
---
*长期在线回收iPhone*
By: 二手iPhone贩子
Content: 6月初 13p 128后爆，也能收个 2k 。 还行吧。
Read more: https://linux.do/t/topic/525305/283
---
*长期在线回收iPhone*
By: 二手iPhone贩子
Content: Read full topic
Read more: https://linux.do/t/topic/525305/282
---
*长期在线回收iPhone*
By: 二手iPhone贩子
Content: 美版有锁就自己留着吧，没啥回收价值了。 美版无锁还能值个 400 左右。 自己挂闲鱼看看垃圾佬能不能出到 650
Read more: https://linux.do/t/topic/525305/281
---
*长期在线回收iPhone*
By: Those
Content: 白色国行12pro128 无拆修，电池83，外观和屏幕完美，估价多少？
Read more: https://linux.do/t/topic/525305/280
---
*长期在线回收iPhone*
By: snow7y
Content: 佬友，15pro 原色 256g 电池90，边框屏幕很好，无修，值多少
Read more: https://linux.do/t/topic/525305/279
---
*长期在线回收iPhone*
By: didiBird
Content: 13pro128 背壳碎，我就卖了 1988还包了邮费，黄鱼还扣了1%手续费。 不过比转转上门多了600，转转只给1300，我没卖，黄鱼自己卖了，一天就出掉了
Read more: https://linux.do/t/topic/525305/278
---
*长期在线回收iPhone*
By: KotoriMinami
Content: 佬，xs64g，电池80以下，人脸识别的深感摄像投进过一次水坏了，多少钱，还有没有出的必要
Read more: https://linux.do/t/topic/525305/277
---
*长期在线回收iPhone*
By: 潇7
Content: 是的，换国内屏
Read more: https://linux.do/t/topic/525305/276
---
*长期在线回收iPhone*
By: 二手iPhone贩子
Content: 3200 回收。
Read more: https://linux.do/t/topic/525305/275
---
*长期在线回收iPhone*
By: 二手iPhone贩子
Content: 应该有个 150 左右。 想卖的话 ，给你打包一起收了。
Read more: https://linux.do/t/topic/525305/274
---
*长期在线回收iPhone*
By: 熊猫竹子
Content: 国行 14 Pro 128G，电池健康 78% ，无拆无修，成色 9成 - 95 成新吧， 自用的，还有购买时候的原盒 值多少
Read more: https://linux.do/t/topic/525305/273
---
*长期在线回收iPhone*
By: 害你的猪
Content: 我的iPhone5s 16G和iPhone7 32G还能开机。不知道能卖多少
Read more: https://linux.do/t/topic/525305/272
---
*长期在线回收iPhone*
By: 二手iPhone贩子
Content: 国行 保修 280+ ， 7000 走闲鱼。
Read more: https://linux.do/t/topic/525305/271
---
*长期在线回收iPhone*
By: Jiushuself
Content: 充电27次
Read more: https://linux.do/t/topic/525305/270
---
*长期在线回收iPhone*
By: 二手iPhone贩子
Content: hone16promax 256G多少米，使用一个月
Read more: https://linux.do/t/topic/525305/269
---
================================================
FILE: test_linuxdo.js
================================================
import { connect } from "puppeteer-real-browser";
while (true) {
console.log("Start of test.js");
const { page, browser } = await connect({
headless: 'auto',
args: [],
customConfig: {},
skipTarget: [],
fingerprint: false,
turnstile: true,
connectOption: {},
});
var cl = setInterval(() => {
page
.screenshot({ path: "example.png", fullPage: true })
.catch((e) => console.log(e.message));
}, 1000);
console.log("Connected to browser");
// https://nopecha.com/demo/cloudflare https://linux.do
await page.goto("https://linux.do", {
waitUntil: "domcontentloaded",
});
console.log("Navigated to page");
await page.waitForSelector(".link_row", {
timeout: 60000,
});
// await page.screenshot({ path: 'example.png' });
clearInterval(cl);
await browser.close();
console.log("End of test.js");
}
================================================
FILE: test_multi_db.js
================================================
// 多数据库功能测试脚本 (PostgreSQL + MongoDB + MySQL)
import {
testAllConnections,
getAllDatabaseStats,
savePosts,
isGuidExists,
closeAllConnections,
} from "./src/db.js";
async function testMultiDatabase() {
console.log("🚀 开始多数据库功能测试 (PostgreSQL + MongoDB + MySQL)...\n");
try {
// 1. 测试所有数据库连接
console.log("=== 1. 测试数据库连接 ===");
await testAllConnections();
console.log("");
// 2. 获取当前统计信息
console.log("=== 2. 获取当前统计信息 ===");
const statsBefore = await getAllDatabaseStats();
console.log("");
// 3. 测试保存数据
console.log("=== 3. 测试保存数据 ===");
const testPosts = [
{
title: "测试帖子标题 - 多数据库测试 (含MongoDB+MySQL)",
creator: "test_user",
description:
"这是一个多数据库功能测试帖子，包括 PostgreSQL、MongoDB 和 MySQL",
link: "https://linux.do/t/topic/test-mysql-123",
pubDate: new Date().toISOString(),
guid: `test-multi-db-mysql-${Date.now()}`,
guidIsPermaLink: "false",
source: "Linux.do",
sourceUrl: "https://linux.do",
},
];
await savePosts(testPosts);
console.log("");
// 4. 测试GUID存在性检查
console.log("=== 4. 测试GUID存在性检查 ===");
const testGuid = testPosts[0].guid;
const exists = await isGuidExists(testGuid);
console.log(`GUID ${testGuid} 存在性: ${exists ? "✅ 存在" : "❌ 不存在"}`);
console.log("");
// 5. 获取更新后的统计信息
console.log("=== 5. 获取更新后的统计信息 ===");
const statsAfter = await getAllDatabaseStats();
console.log("");
// 6. 比较统计信息
console.log("=== 6. 统计信息对比 ===");
statsBefore.forEach((beforeStat, index) => {
const afterStat = statsAfter[index];
if (beforeStat.status === "healthy" && afterStat.status === "healthy") {
const increase = afterStat.totalPosts - beforeStat.totalPosts;
console.log(
`${beforeStat.name}: ${beforeStat.totalPosts} → ${afterStat.totalPosts} (+${increase})`
);
}
});
console.log("\n✅ 多数据库功能测试完成 (PostgreSQL + MongoDB + MySQL)");
} catch (error) {
console.error("❌ 测试过程中发生错误:", error);
} finally {
// 关闭所有连接
console.log("\n=== 关闭数据库连接 ===");
await closeAllConnections();
}
}
// 运行测试
testMultiDatabase().catch(console.error);
================================================
FILE: test_proxy.js
================================================
#!/usr/bin/env node
/**
* 代理测试工具
* 用于测试代理服务器的连通性和配置正确性
*/
import dotenv from "dotenv";
import {
getProxyConfig,
testProxyConnection,
getCurrentIP,
parseProxyUrl,
} from "./src/proxy_config.js";
// 加载环境变量
dotenv.config();
async function main() {
console.log("🔍 代理配置测试工具");
console.log("==================");
// 获取当前IP
console.log("\n📍 当前网络状态:");
const currentIP = await getCurrentIP();
if (currentIP) {
console.log(`✅ 当前IP地址: ${currentIP}`);
} else {
console.log("❌ 无法获取当前IP地址");
}
// 检查代理配置
console.log("\n🔧 代理配置检查:");
const proxyConfig = getProxyConfig();
if (!proxyConfig) {
console.log("❌ 未配置代理服务器");
console.log("\n💡 配置方法:");
console.log("1. 设置环境变量 PROXY_URL，例如:");
console.log(" PROXY_URL=http://username:password@proxy.example.com:8080");
console.log(
" PROXY_URL=socks5://username:password@proxy.example.com:1080"
);
console.log("\n2. 或者分别设置:");
console.log(" PROXY_TYPE=http");
console.log(" PROXY_HOST=proxy.example.com");
console.log(" PROXY_PORT=8080");
console.log(" PROXY_USERNAME=your_username");
console.log(" PROXY_PASSWORD=your_password");
return;
}
console.log(`✅ 代理类型: ${proxyConfig.type}`);
console.log(`✅ 代理地址: ${proxyConfig.host}:${proxyConfig.port}`);
if (proxyConfig.username) {
console.log(`✅ 认证用户: ${proxyConfig.username}`);
console.log(
`✅ 密码设置: ${"*".repeat(proxyConfig.password?.length || 0)}`
);
} else {
console.log("ℹ️ 无需认证");
}
// 测试代理连接
console.log("\n🚀 代理连接测试:");
console.log("正在测试代理连接...");
const startTime = Date.now();
const isWorking = await testProxyConnection(proxyConfig);
const endTime = Date.now();
if (isWorking) {
console.log(`✅ 代理连接成功! (耗时: ${endTime - startTime}ms)`);
console.log("🎉 代理服务器工作正常，可以使用");
} else {
console.log(`❌ 代理连接失败! (耗时: ${endTime - startTime}ms)`);
console.log("\n🔧 故障排查建议:");
console.log("1. 检查代理服务器地址和端口是否正确");
console.log("2. 检查用户名和密码是否正确");
console.log("3. 检查代理服务器是否在线");
console.log("4. 检查网络防火墙设置");
console.log("5. 尝试其他代理服务器");
}
// 环境变量检查
console.log("\n📋 环境变量检查:");
const envVars = [
"PROXY_URL",
"PROXY_TYPE",
"PROXY_HOST",
"PROXY_PORT",
"PROXY_USERNAME",
"PROXY_PASSWORD",
];
envVars.forEach((varName) => {
const value = process.env[varName];
if (value) {
if (varName.includes("PASSWORD")) {
console.log(`✅ ${varName}: ${"*".repeat(value.length)}`);
} else {
console.log(`✅ ${varName}: ${value}`);
}
} else {
console.log(`❌ ${varName}: 未设置`);
}
});
}
// 如果是直接运行此脚本
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch(console.error);
}
export default main;
================================================
FILE: test_topic_data.js
================================================
import { extractTopicIdFromUrl, processAndSaveTopicData } from './src/topic_data.js';
import fs from 'fs';
import dotenv from 'dotenv';
// Load environment variables
dotenv.config();
if (fs.existsSync(".env.local")) {
const envConfig = dotenv.parse(fs.readFileSync(".env.local"));
for (const k in envConfig) {
process.env[k] = envConfig[k];
}
}
async function testTopicData() {
try {
// Test URL extraction
const testUrls = [
'https://linux.do/t/topic/710103',
'https://linux.do/t/topic/525305/100',
'https://linux.do/t/topic/13716/790'
];
console.log('测试话题ID提取:');
for (const url of testUrls) {
const topicId = extractTopicIdFromUrl(url);
console.log(`URL: ${url} -> Topic ID: ${topicId}`);
}
// Test with an invalid URL
const invalidUrl = 'https://linux.do/latest';
const invalidTopicId = extractTopicIdFromUrl(invalidUrl);
console.log(`Invalid URL: ${invalidUrl} -> Topic ID: ${invalidTopicId}`);
console.log('\n✅ 话题ID提取测试完成');
// Note: We're not testing the full processAndSaveTopicData function here
// because it requires a Puppeteer page instance which is complex to set up
console.log('\n📝 完整的话题数据抓取功能需要在实际运行环境中测试');
} catch (error) {
console.error('测试失败:', error);
}
}
testTopicData();
================================================
FILE: 随笔.md
================================================
# TimeoutError: Navigation timeout of 30000 ms exceeded 为什么puppeteer经常出现这个错误
在使用 Puppeteer 进行网页自动化或爬虫开发时，`TimeoutError: Navigation timeout of 30000 ms exceeded` 错误是一个常见问题。这个错误意味着 Puppeteer 在尝试加载页面或执行某个操作时超出了预设的最大等待时间（默认为 30,000 毫秒，即 30 秒）。这种情况通常由以下几个原因引起：
### 1. 网页加载慢
网页可能因为多种原因加载缓慢：
- 网络连接慢。
- 服务器响应时间长。
- 页面资源（如大型图片、视频或复杂的 JavaScript）多且复杂，加载时间长。
### 2. 重定向
有些网页在加载过程中可能会进行多次重定向，这些重定向可能会导致总加载时间超过设定的超时限制。
### 3. JavaScript 执行延迟
如果网页依赖于复杂的 JavaScript 或外部脚本执行完成才能呈现，且这些脚本执行缓慢，可能会触发超时错误。
### 4. 服务器问题
服务器可能会由于各种原因（例如高负载、维护或配置问题）响应缓慢，导致页面加载延迟。
### 解决策略
要解决或规避这个问题，可以采取以下几种方法：
#### a. 增加超时时间
如果你认为页面需要更长时间加载，可以增加超时设置：
```javascript
await page.goto('http://example.com', {waitUntil: 'networkidle0', timeout: 60000}); // 设置超时时间为60秒
```
#### b. 修改等待条件
Puppeteer 在等待 `networkidle0` 或 `networkidle2` 事件时可能会超时。这些事件分别代表没有网络连接或只有两个网络连接在指定时间内处于空闲状态。你可以改变等待策略以更适应页面的加载方式：
```javascript
await page.goto('http://example.com', {waitUntil: 'domcontentloaded'});
```
这会让 Puppeteer 等待到 `DOM` 内容加载完成即继续执行，而不是等待所有外部资源加载完成。
#### c. 检查和优化页面加载项
如果你对服务器或页面有控制权，尝试优化网页性能。这可能包括减少页面资源大小、优化图片和使用更快的服务器。
#### d. 错误处理
增加错误处理逻辑，当加载超时时采取其他操作，例如重试或记录错误：
```javascript
try {
await page.goto('http://example.com', {waitUntil: 'networkidle0', timeout: 30000});
} catch (error) {
console.error('Navigation timeout exceeded: ' + error.message);
// 可以在此处添加重试逻辑或其他错误处理代码
}
```
#### e. 使用 headless:false 调试
在开发过程中，设置 `headless:false` 来观察 Puppeteer 在浏览器中实际执行的情况可能有助于发现问题原因：
```javascript
const browser = await puppeteer.launch({headless: false});
```
通过上述方法，你可以有效地解决或减少遇到 `TimeoutError: Navigation timeout of 30000 ms exceeded` 错误的情况，从而提高 Puppeteer 脚本的稳定性和效率。
![\":ok_hand:\"](\"https://linux.do/images/emoji/twemoji/ok_hand.png?v=14\" "\\":ok_hand:\\"")
![\":expressionless_face:\"](\"https://linux.do/images/emoji/twemoji/expressionless_face.png?v=14\" "\\":expressionless_face:\\"")