Showing preview only (626K chars total). Download the full file or copy to clipboard to get everything.
Repository: xhunmon/PythonIsTools
Branch: main
Commit: 6354d2d442a0
Files: 158
Total size: 555.3 KB
Directory structure:
gitextract_xtly8u7o/
├── .gitignore
├── 001-Downloader/
│ ├── .gitignore
│ ├── README.md
│ ├── __init__.py
│ ├── config.ini
│ ├── doc/
│ │ ├── mac-sh/
│ │ │ ├── main.spec
│ │ │ └── pyinstaller.sh
│ │ └── win-sh/
│ │ ├── main.spec
│ │ └── pyinstaller.sh
│ ├── douyin/
│ │ └── dy_download.py
│ ├── downloader.py
│ ├── kuaishou/
│ │ └── ks_download.py
│ ├── main.py
│ ├── test/
│ │ ├── bilibili_video_download_v1.py
│ │ ├── ff_video.py
│ │ ├── test_pyinstaller.py
│ │ ├── urls.txt
│ │ └── xhs_download.py
│ ├── type_enum.py
│ ├── ui.py
│ └── utils.py
├── 002-V2rayPool/
│ ├── .gitignore
│ ├── 002-V2rayPool.iml
│ ├── README.md
│ ├── base/
│ │ └── net_proxy.py
│ ├── core/
│ │ ├── client.py
│ │ ├── conf.py
│ │ ├── group.py
│ │ ├── json_template/
│ │ │ ├── client.json
│ │ │ ├── client_socks.json
│ │ │ ├── client_ss.json
│ │ │ ├── client_trojan.json
│ │ │ ├── dyn_port.json
│ │ │ ├── http.json
│ │ │ ├── http2.json
│ │ │ ├── kcp.json
│ │ │ ├── mtproto.json
│ │ │ ├── quic.json
│ │ │ ├── server.json
│ │ │ ├── socks.json
│ │ │ ├── ss.json
│ │ │ ├── stats_settings.json
│ │ │ ├── tcp.json
│ │ │ ├── vless.json
│ │ │ └── ws.json
│ │ ├── profile.py
│ │ └── utils.py
│ ├── db/
│ │ ├── db_main.py
│ │ ├── local.py
│ │ └── net.py
│ ├── doc/
│ │ └── (参考用)config.json
│ └── test_main.py
├── 003-Keywords/
│ ├── .gitignore
│ ├── Necklace/
│ │ └── Necklace.xlsx
│ ├── README.md
│ ├── __init__.py
│ ├── amazon/
│ │ ├── items.py
│ │ ├── middlewares.py
│ │ ├── pipelines.py
│ │ ├── settings.py
│ │ └── spiders/
│ │ ├── __init__.py
│ │ ├── alibaba.py
│ │ ├── amazon.py
│ │ ├── checkip.py
│ │ └── spiders.py
│ ├── google.py
│ ├── main.py
│ ├── mypytrends/
│ │ ├── __init__.py
│ │ ├── dailydata.py
│ │ ├── exceptions.py
│ │ ├── request.py
│ │ └── test_trendReq.py
│ ├── run_api.py
│ ├── scrapy.cfg
│ ├── test_souce1.xlsx
│ ├── test_source.xlsx
│ ├── v2ray_pool/
│ │ ├── __init__.py
│ │ ├── _db-checked.txt
│ │ └── _db-uncheck.txt
│ ├── v2ray_util.py
│ └── women-ring/
│ └── women ring.csv
├── 004-EmailNotify/
│ ├── .gitignore
│ ├── README.md
│ └── main.py
├── 005-PaidSource/
│ ├── .gitignore
│ ├── 005-PaidSource.iml
│ ├── README.md
│ ├── __init__.py
│ ├── chrome.py
│ ├── ff_video.py
│ ├── file_util.py
│ ├── gsearch.py
│ ├── gtransfer.py
│ ├── kaoqin.py
│ ├── keywords.py
│ ├── main.py
│ ├── other_site.py
│ ├── v2ray_pool/
│ │ ├── __init__.py
│ │ ├── _db-checked.txt
│ │ └── _db-uncheck.txt
│ └── v2ray_util.py
├── 006-TikTok/
│ ├── .gitignore
│ ├── 006-TikTok.iml
│ ├── README.md
│ ├── __init__.py
│ ├── dy_review.py
│ ├── file_util.py
│ ├── google_transfer_by_excel.py
│ ├── img_2_webp.py
│ ├── main.py
│ ├── post_autotk.py
│ ├── tikstar.py
│ ├── tt_review.py
│ └── v2ray_pool/
│ └── __init__.py
├── 007-CutVideoAudio/
│ ├── .gitignore
│ ├── 007-CutVideoAudio.iml
│ ├── README.md
│ ├── __init__.py
│ ├── config.ini
│ ├── doc/
│ │ └── mac-sh/
│ │ ├── main.spec
│ │ └── pyinstaller.sh
│ ├── editors.py
│ ├── ff_cut.py
│ ├── ff_util.py
│ ├── ff_util_v2.py
│ ├── main.py
│ ├── type_enum.py
│ ├── ui.py
│ └── utils.py
├── 008-ChatGPT-UI/
│ ├── .gitignore
│ ├── README.md
│ ├── config.ini
│ ├── config.json
│ ├── doc/
│ │ ├── config.json
│ │ └── pyinstaller.sh
│ ├── gpt.py
│ ├── main.py
│ ├── requirements.txt
│ └── utils.py
├── 009-Translate/
│ ├── README.md
│ ├── asset/
│ │ ├── ch.ini
│ │ ├── config.ini
│ │ ├── en.ini
│ │ └── language.json
│ ├── config.py
│ ├── core.py
│ ├── doc/
│ │ └── pyinstaller.sh
│ ├── load_srt.py
│ ├── main.py
│ ├── tran_test.py
│ ├── ui.py
│ └── utils.py
├── 010-YouTubeUpload/
│ ├── README.md
│ ├── main.py
│ ├── tst.py
│ └── youtube.py
├── LICENSE
└── README.md
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitignore
================================================
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
**__pycache__/
**.idea
**dist/
**build
**__pycache__
================================================
FILE: 001-Downloader/.gitignore
================================================
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
================================================
FILE: 001-Downloader/README.md
================================================
# 资源下载器
本项目主要通过网络上开源的项目聚合成了一个跨平台的下载工具，可批量下载抖音、快手视音频资源。下载地址：
MacOS：[Downloader1.0.3.app](https://github.com/xhunmon/PythonIsTools/releases/download/v1.0.3/Downloader1.0.3.app.zip) 下载后解压后使用
Window：[Downloader1.0.3.exe](https://github.com/xhunmon/PythonIsTools/releases/download/v1.0.3/Downloader1.0.3.exe.zip) 下载后解压后使用
效果如图：
#主要知识点
## python GUI(界面)
本文使用tkinter GUI(界面)框架进行界面显示：[./ui.py](ui.py) ，[学习参考](https://www.cnblogs.com/shwee/p/9427975.html) 。
## [pyinstaller](https://pyinstaller.readthedocs.io/en/stable/) 打包
使用pyinstaller把python程序打包成window和mac可执行文件，主要命令如下：
```shell
#① ：生成xxx.spec文件；（去掉命令窗口-w）
pyinstaller -F -i res/logo.ico main.py -w
#②：修改xxx.spec，参考main.spec
#③：再次进行打包，参考installer-mac.sh
pyinstaller -F -i res/logo.ico main.spec -w
```
打包脚本与配置已放在 `doc` 目录下，需要拷贝出根目录进行打包。
注意：
pyinstaller打包工具的版本与python版本、python所需第三方库以及操作系统会存在各种问题，所以需要看日志查找问题。例如：打包后运用，发现导入pyppeteer报错，通过降低版本后能正常使用：pip install pyppeteer==0.2.2
## 项目
项目代码结构非常简单，看ui.py和downloader.py就能知道大概。支持多线程任务下载。如果自己添加其他网站的资源下载，通过增加实现downloader.py和并且在ui.py中start_download增加入口判读即可无缝接入。
================================================
FILE: 001-Downloader/__init__.py
================================================
================================================
FILE: 001-Downloader/config.ini
================================================
# 常用配置模块
[common]
#软件使用截止日期
expired_time=2025/12/15 23:59:59
#app的版本名称
version_name=1.0.4
#app的版本号
version_code=1040
================================================
FILE: 001-Downloader/doc/mac-sh/main.spec
================================================
# -*- mode: python ; coding: utf-8 -*-
block_cipher = None
a = Analysis(['main.py','type_enum.py','ui.py','utils.py','downloader.py','douyin/dy_download.py'],
pathex=['.'],
binaries=[],
datas=[('res/logo.ico', 'images'),('config.ini', '.')],
hiddenimports=[],
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[],
win_no_prefer_redirects=False,
win_private_assemblies=False,
cipher=block_cipher,
noarchive=False)
pyz = PYZ(a.pure, a.zipped_data,
cipher=block_cipher)
exe = EXE(pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
[],
name='main',
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
upx_exclude=[],
runtime_tmpdir=None,
console=False,
disable_windowed_traceback=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None , icon='res/logo.ico')
app = BUNDLE(exe,
name='Downloader.app',
icon='res/logo.ico',
bundle_identifier=None)
================================================
FILE: 001-Downloader/doc/mac-sh/pyinstaller.sh
================================================
#!/bin/bash
pyinstaller -F -i res/logo.ico main.spec main.py -w \
-p type_enum.py \
-p ui.py \
-p utils.py \
-p downloader.py \
-p douyin/dy_download.py \
-p kuaishou/ks_download.py
================================================
FILE: 001-Downloader/doc/win-sh/main.spec
================================================
# -*- mode: python ; coding: utf-8 -*-
block_cipher = None
a = Analysis(['main.py','type_enum.py','ui.py','utils.py','downloader.py','douyin\\dy_download.py'],
pathex=['.'],
binaries=[],
datas=[('res\\logo.ico', 'images'),('config.ini', '.')],
hiddenimports=[],
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[],
win_no_prefer_redirects=False,
win_private_assemblies=False,
cipher=block_cipher,
noarchive=False)
pyz = PYZ(a.pure, a.zipped_data,
cipher=block_cipher)
exe = EXE(pyz,
a.scripts,
a.binaries,
a.zipfiles,
a.datas,
[],
name='main',
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
upx_exclude=[],
runtime_tmpdir=None,
console=False,
disable_windowed_traceback=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None , icon='res\\logo.ico')
app = BUNDLE(exe,
name='Downloader.exe',
icon='res\\logo.ico',
bundle_identifier=None)
================================================
FILE: 001-Downloader/doc/win-sh/pyinstaller.sh
================================================
#!/bin/bash
pyinstaller -F -i res\\logo.ico -w main.spec main.py
-p type_enum.py
-p ui.py
-p utils.py
-p downloader.py
-p douyin\\dy_download.py
-p kuaishou\\ks_download.py
================================================
FILE: 001-Downloader/douyin/dy_download.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 抖音视频下载
@Date :2021/08/14
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import json
import os
import re
import time
import requests
from downloader import Downloader
class DouYin(Downloader):
# 初始化
def __init__(self):
super().__init__()
self.headers = self._headers
# 抓获所有视频
self.end = False
def start(self, url, path):
Downloader.print_ui("开始解析下载链接")
# 读取保存路径
self.save = path
# 读取下载视频个数
self.count = 10
# 读取下载是否下载音频
self.musicarg = True
# 读取用户主页地址
self.user = ''
# 读取单条
self.single = ''
# 读取下载模式 #下载模式选择 like为点赞 post为发布
self.mode = 'post'
# 保存用户名
self.nickname = ''
if '/user/' in url:
self.user = url
else:
self.single = url
# https://www.douyin.com/video/6979067378848042276?extra_params=%7B%22search_id%22%3A%22202109260757420101511740995D070AF5%22%2C%22search_result_id%22%3A%226979067378848042276%22%2C%22search_type%22%3A%22video%22%2C%22search_keyword%22%3A%22%E6%A8%A1%E7%89%B9%22%7D&previous_page=search_result
# try:
# self.single = re.findall(r'(http.+?)\?extra_params', url)[0]
# except:
# self.single = url
if len(self.single) > 0:
self.count = 1
self.parse_single()
else:
self.judge_link()
# 单条数据页面
def parse_single(self):
url = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', self.single)[
0]
r = requests.get(url=url)
key = re.findall('video/(\d+)?', str(r.url))[0]
jx_url = f'https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={key}' # 官方接口
js = json.loads(requests.get(url=jx_url, headers=self.headers).text)
detail = js['item_list'][0]
# 作者信息
author_list = []
# 无水印视频链接
video_list = []
# 作品id
aweme_id = []
# 作者id
nickname = []
max_cursor = 0
author_list.append(str(detail['desc']))
video_list.append(str(detail['video']['play_addr']['url_list'][0]).replace('playwm', 'play'))
aweme_id.append(str(detail['aweme_id']))
nickname.append(str(detail['author']['nickname']))
Downloader.print_ui('开始下载单个视频' + video_list[0])
self.videos_download(author_list, video_list, aweme_id, nickname, max_cursor)
# 匹配粘贴的url地址
def Find(self, string):
# findall() 查找匹配正则表达式的字符串
url = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', string)
Downloader.print_ui('Find url: ' + url)
return url
# 判断个人主页api链接
def judge_link(self):
user_url: str = self.user
Downloader.print_ui('----为您下载多个视频----\r')
key = re.findall('/user/(.*?)$', str(user_url))[0]
if not key:
key = user_url[28:83]
Downloader.print_ui('----' + '用户的sec_id=' + key + '----\r')
# 第一次访问页码
max_cursor = 0
# 构造第一次访问链接
api_post_url = 'https://www.iesdouyin.com/web/api/v2/aweme/%s/?sec_uid=%s&count=%s&max_cursor=%s&aid=1128&_signature=PDHVOQAAXMfFyj02QEpGaDwx1S&dytk=' % (
self.mode, key, str(self.count), max_cursor)
self.get_data(api_post_url, max_cursor)
return api_post_url, max_cursor, key
# 获取第一次api数据
def get_data(self, api_post_url, max_cursor):
# 尝试次数
index = 0
# 存储api数据
result = []
while result == []:
index += 1
Downloader.print_ui('----正在进行第 %d 次尝试----\r' % index)
time.sleep(0.3)
response = requests.get(url=api_post_url, headers=self.headers)
html = json.loads(response.content.decode())
if self.end == False:
# 下一页值
self.nickname = html['aweme_list'][0]['author']['nickname']
Downloader.print_ui('[ 用户 ]:' + str(self.nickname) + '\r')
max_cursor = html['max_cursor']
result = html['aweme_list']
Downloader.print_ui('----抓获数据成功----\r')
# 处理第一页视频信息
self.video_info(result, max_cursor)
else:
max_cursor = html['max_cursor']
self.next_data(max_cursor)
# self.end = True
Downloader.print_ui('----此页无数据，为您跳过----\r')
return result, max_cursor
# 下一页
def next_data(self, max_cursor):
if self.count == 1:
return
user_url = self.user
# 获取用户sec_uid
# key = re.findall('/user/(.*?)\?', str(user_url))[0]
key = re.findall('/user/(.*?)$', str(user_url))[0]
if not key:
key = user_url[28:83]
# 构造下一次访问链接
api_naxt_post_url = 'https://www.iesdouyin.com/web/api/v2/aweme/%s/?sec_uid=%s&count=%s&max_cursor=%s&aid=1128&_signature=RuMN1wAAJu7w0.6HdIeO2EbjDc&dytk=' % (
self.mode, key, str(self.count), max_cursor)
index = 0
result = []
while self.end == False:
# 回到首页，则结束
if max_cursor == 0:
self.end = True
return
index += 1
# Downloader.print_ui('----正在对' + max_cursor + '页进行第 %d 次尝试----\r' % index)
Downloader.print_ui('----正在对{}页进行第 {} 次尝试----\r'.format(max_cursor, index))
time.sleep(3)
response = requests.get(url=api_naxt_post_url, headers=self.headers)
html = json.loads(response.content.decode())
if self.end == False:
# 下一页值
max_cursor = html['max_cursor']
result = html['aweme_list']
Downloader.print_ui('----{}页抓获数据成功----\r'.format(max_cursor))
# 处理下一页视频信息
self.video_info(result, max_cursor)
else:
self.end = True
Downloader.print_ui('----{}页抓获数据失败----\r'.format(max_cursor))
# sys.exit()
# 处理视频信息
def video_info(self, result, max_cursor):
# 作者信息
author_list = []
# 无水印视频链接
video_list = []
# 作品id
aweme_id = []
# 作者id
nickname = []
# 封面大图
# dynamic_cover = []
for i2 in range(len(result)):
try:
author_list.append(str(result[i2]['desc']))
video_list.append(str(result[i2]['video']['play_addr']['url_list'][0]))
aweme_id.append(str(result[i2]['aweme_id']))
nickname.append(str(result[i2]['author']['nickname']))
# dynamic_cover.append(str(result[i2]['video']['dynamic_cover']['url_list'][0]))
except Exception as error:
# Downloader.print_ui2(error)
pass
self.videos_download(author_list, video_list, aweme_id, nickname, max_cursor)
return self, author_list, video_list, aweme_id, nickname, max_cursor
def videos_download(self, author_list, video_list, aweme_id, nickname, max_cursor):
count = len(author_list)
Downloader.add_total_count(count)
for i in range(count):
if count == 1:
# 创建并检测下载目录是否存在
pre_save = os.path.join(self.save, "单条")
else:
pre_save = os.path.join(self.save, nickname[i])
try:
os.makedirs(pre_save)
except:
pass
Downloader.add_downloading_count()
# try:
# jx_url = f'https://www.iesdouyin.com/web/api/v2/aweme/iteminfo/?item_ids={aweme_id[i]}' # 官方接口
# js = json.loads(requests.get(url=jx_url, headers=self.headers).text)
# music_url = str(js['item_list'][0]['music']['play_url']['url_list'][0])
# music_title = str(js['item_list'][0]['music']['author'])
# if self.musicarg == "yes": # 保留音频
# music = requests.get(music_url) # 保存音频
# start = time.time() # 下载开始时间
# size = 0 # 初始化已下载大小
# chunk_size = 1024 # 每次下载的数据大小
# content_size = int(music.headers['content-length']) # 下载文件总大小
# if music.status_code == 200: # 判断是否响应成功
# Downloader.print_ui('[ 音频 ]:' + author_list[i] + '[文件 大小]:{size:.2f} MB'.format(
# size=content_size / chunk_size / 1024)) # 开始下载，显示下载文件大小
# # m_url = pre_save + music_title + '-[' + author_list[i] + '].mp3'
# m_url = os.path.join(pre_save,
# nickname[i] + "-" + music_title + '-[' + author_list[i] + '].mp3')
# Downloader.print_ui("路径：" + m_url)
# with open(m_url, 'wb') as file: # 显示进度条
# for data in music.iter_content(chunk_size=chunk_size):
# file.write(data)
# size += len(data)
# Downloader.print_ui('\r' + music_title + '\n[下载进度]:%s%.2f%%' % (
# '>' * int(size * 50 / content_size), float(size / content_size * 100)))
# end = time.time() # 下载结束时间
# Downloader.print_ui('\n' + music_title + '\n[下载完成]:耗时: %.2f秒\n' % (end - start)) # 输出下载用时时间
# Downloader.add_success_count()
# except Exception as error:
# # Downloader.print_ui2(error)
# Downloader.print_ui('该页音频没有' + str(self.count) + '个\r')
# # Downloader.add_failed_count()
# # break
try:
v_url = os.path.join(pre_save, nickname[i] + "-" + '[' + author_list[i] + '].mp4')
# 如果本地已经有了就跳过
if os.path.exists(v_url):
Downloader.print_ui('{}-已存在！'.format(v_url))
Downloader.add_success_count()
continue
video = requests.get(video_list[i], headers=self.headers) # 保存视频
start = time.time() # 下载开始时间
size = 0 # 初始化已下载大小
chunk_size = 100 # 每次下载的数据大小
content_size = int(video.headers['content-length']) # 下载文件总大小
if video.status_code == 200: # 判断是否响应成功
Downloader.print_ui(
'[ 视频 ]:' + nickname[i] + '-' + author_list[i] + '[文件 大小]:{size:.2f} MB'.format(
size=content_size / 1024 / 1024)) # 开始下载，显示下载文件大小
# v_url = os.path.join(pre_save, nickname[i] + "-" + '[' + author_list[i] + '].mp4')
# v_url = pre_save + '[' + author_list[i] + '].mp4'
Downloader.print_ui("路径：" + v_url)
with open(v_url, 'wb') as file: # 显示进度条
for data in video.iter_content(chunk_size=chunk_size):
file.write(data)
size += len(data)
Downloader.print_ui('\r' + author_list[i] + '\n[下载进度]:%s%.2f%%' % (
'>' * int(size * 50 / content_size), float(size / content_size * 100)))
end = time.time() # 下载结束时间
Downloader.print_ui('\n' + author_list[i] + '\n[下载完成]:耗时: %.2f秒\n' % (end - start)) # 输出下载用时时间
Downloader.add_success_count()
except Exception as error:
# Downloader.print_ui2(error)
Downloader.print_ui('该页视频没有' + str(count) + '个,已为您跳过\r')
Downloader.add_failed_count()
break
self.next_data(max_cursor)
================================================
FILE: 001-Downloader/downloader.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description:downloader.py 所有下载类的基类，负责与UI界面的绑定
@Date :2021/08/14
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import time
from threading import Lock
import requests
from my_fake_useragent import UserAgent
from type_enum import PrintType
from utils import Config
ua = UserAgent(family='chrome')
class Downloader(object):
func_ui_print = None
__mutex_total = Lock()
__mutex_success = Lock()
__mutex_failed = Lock()
__mutex_downloading = Lock()
__count_total = 0
__count_success = 0
__count_failed = 0
__count_downloading = 0
__beijing_time = 0 # 在线北京时间
def __init__(self):
self._headers = {'user-agent': ua.random()}
self.get_beijing_time()
@staticmethod
def print_hint():
"""显示初始提示信息"""
Downloader.print_ui(
"""
使用说明：
1、快手下载用户批量视频如：https://www.kuaishou.com/profile/xxx
2、快手下载单条视频如：https://www.kuaishou.com/short-video/xxx
3、抖音下载用户批量视频如：https://www.douyin.com/user/xxx
4、抖音下载单条视频如：https://www.douyin.com/video/xxx
"""
)
def start(self, url, path):
"""业务逻辑由子类实现"""
pass
@staticmethod
def print_ui(txt):
"""在界面显示内容"""
Downloader.print_all_ui(txt=txt) # 打印日志
@staticmethod
def print_all_ui(txt, print_type: PrintType = PrintType.log):
"""通知ui中func_ui_print更新内容"""
if Downloader.func_ui_print is not None:
Downloader.func_ui_print(txt=txt, print_type=print_type)
@staticmethod
def get_beijing_time():
"""静态方法：获取在线的北京时间"""
if Downloader.__beijing_time > 0:
return Downloader.__beijing_time
try:
response = requests.get(url='http://www.beijing-time.org/t/time.asp', headers={'user-agent': ua.random()})
result = response.text
data = result.split("\r\n")
year = data[1][len("nyear") + 1: len(data[1]) - 1]
month = data[2][len("nmonth") + 1: len(data[2]) - 1]
day = data[3][len("nday") + 1: len(data[3]) - 1]
# wday = data[4][len("nwday")+1 : len(data[4])-1]
hrs = data[5][len("nhrs") + 1: len(data[5]) - 1]
minute = data[6][len("nmin") + 1: len(data[6]) - 1]
sec = data[7][len("nsec") + 1: len(data[7]) - 1]
beijinTimeStr = "%s/%s/%s %s:%s:%s" % (year, month, day, hrs, minute, sec)
beijinTime = time.strptime(beijinTimeStr, "%Y/%m/%d %X")
Downloader.__beijing_time = int(time.mktime(beijinTime))
except:
pass
return Downloader.__beijing_time
@staticmethod
def is_expired():
"""静态方法：判断是否已过期"""
if Downloader.__beijing_time == 0: # 还没获取到时间
return True
expired_time_str = time.strptime(Config.instance().get_expired_time(), "%Y/%m/%d %X")
expired_time_int = int(time.mktime(expired_time_str))
return Downloader.__beijing_time > expired_time_int
@staticmethod
def add_total_count(count=1):
"""静态方法：添加总下载任务数"""
Downloader.__mutex_total.acquire()
Downloader.__count_total += count
Downloader.__mutex_total.release()
Downloader.print_all_ui(txt="预计总数：%d" % Downloader.__count_total, print_type=PrintType.total)
@staticmethod
def get_total_count():
"""静态方法：获取总下载任务数"""
return Downloader.__count_total
@staticmethod
def add_downloading_count():
"""静态方法：添加正在下载任务数"""
Downloader.__mutex_downloading.acquire()
Downloader.__count_downloading += 1
Downloader.__mutex_downloading.release()
Downloader.print_all_ui(txt="正在下载：%d" % Downloader.__count_downloading, print_type=PrintType.downloading)
@staticmethod
def __sub_downloading_count():
"""静态方法：减去正在下载任务数"""
Downloader.__mutex_downloading.acquire()
Downloader.__count_downloading -= 1
Downloader.__mutex_downloading.release()
Downloader.print_all_ui(txt="正在下载：%d" % Downloader.__count_downloading, print_type=PrintType.downloading)
@staticmethod
def get_downloading_count():
"""静态方法：获取正在下载任务数"""
return Downloader.__count_downloading
@staticmethod
def add_success_count():
"""静态方法：添加下载成功任务数"""
Downloader.__mutex_success.acquire()
Downloader.__count_success += 1
Downloader.__mutex_success.release()
# 成功一条，减正在下载的一条
Downloader.__sub_downloading_count()
Downloader.print_all_ui(txt="已完成：%d" % Downloader.__count_success, print_type=PrintType.success)
@staticmethod
def get_success_count():
"""静态方法：获取下载成功任务数"""
return Downloader.__count_success
@staticmethod
def add_failed_count():
"""静态方法：添加下载失败任务数"""
Downloader.__mutex_failed.acquire()
Downloader.__count_failed += 1
Downloader.__mutex_failed.release()
# 失败一条，减正在下载的一条
Downloader.__sub_downloading_count()
Downloader.print_all_ui(txt="已失败：%d" % Downloader.__count_failed, print_type=PrintType.failed)
@staticmethod
def get_failed_count():
"""静态方法：获取下载失败任务数"""
return Downloader.__count_failed
================================================
FILE: 001-Downloader/kuaishou/ks_download.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 快手视频下载
@Date :2021/09/102
@Author :qincji
@Mail :xhunmon@gmail.com
"""
import json
import os
import re
import time
import urllib
import requests
from downloader import Downloader
requestUrl = 'https://video.kuaishou.com/graphql'
class KuaiShou(Downloader):
cookie = 'clientid=3; client_key=65890b29; kpf=PC_WEB; kpn=KUAISHOU_VISION; did=web_3e09c32da1db9d38c0122ffa25ad8b7d'
# 初始化
def __init__(self):
super().__init__()
self.headers = self._headers
# 抓获所有视频
self.end = False
def set_cookie(self, c):
"""当cookie过期时，需要从外界出入"""
KuaiShou.cookie = c
def start(self, url, path):
Downloader.print_ui("开始解析下载链接")
# 读取保存路径
self.save = path
if '/profile/' in url:
self.parse_user(url)
elif '/short-video/' in url:
if 'trendingId' in url:
self.parse_single_trendingId(urllib.parse.unquote(url, encoding="utf-8"))
elif 'streamSource' in url:
self.parse_single_streamSource(urllib.parse.unquote(url, encoding="utf-8"))
else:
Downloader.print_ui('该链接不支持下载')
else:
Downloader.print_ui('该链接不支持下载')
# 单条数据页面
def parse_single_trendingId(self, url):
try:
Downloader.print_ui('----为您下载单个视频----\r')
# userId = re.findall(r'/short-video/(.+?)\?', url)[0].strip()
trendingId = re.findall(r'trendingId=(.+?)&', url)[0].strip()
area = re.findall(r'area=(.+?)$', url)[0].strip()
except:
Downloader.print_ui('地址%s输入错误' % url)
return
links = []
try:
result = self.post_single_trendingId(url, KuaiShou.cookie, trendingId, area)
data = json.loads(result)
feeds = data['data']['hotData']['feeds']
size = len(feeds)
links.append(feeds)
except Exception as e:
Downloader.print_ui(str(e))
return
if size < 1:
Downloader.print_ui('解析地址%s异常' % url)
return
Downloader.add_total_count(size)
for link in links:
self.download(link)
# 单条数据页面
def parse_single_streamSource(self, url):
try:
Downloader.print_ui('----为您下载单个视频----\r')
userId = re.findall(r'/short-video/(.+?)\?', url)[0].strip()
area = re.findall(r'area=(.+?)$', url)[0].strip()
except:
Downloader.print_ui('地址%s输入错误' % url)
return
links = []
try:
result = self.post_single_streamSource(url, KuaiShou.cookie, userId, area)
data = json.loads(result)
feeds = [data['data']['visionVideoDetail'], ]
size = len(feeds)
links.append(feeds)
except Exception as e:
Downloader.print_ui(str(e))
return
if size < 1:
Downloader.print_ui('解析地址%s异常' % url)
return
Downloader.add_total_count(size)
for link in links:
self.download(link)
# 判断个人主页api链接
def parse_user(self, url):
# https://www.kuaishou.com/profile/3xcx5qwycxzxdre
try:
Downloader.print_ui('----为您下载多个视频----\r')
userId = re.findall(r'/profile/(.+?)$', url)[0].strip()
except:
Downloader.print_ui('地址%s输入错误' % url)
return
pcursor = ''
all_count = 0
links = []
while True:
try:
result = self.post_user(userId, KuaiShou.cookie, pcursor)
data = json.loads(result)
feeds = data['data']['visionProfilePhotoList']['feeds']
flen = len(feeds)
pcursor = data['data']['visionProfilePhotoList']['pcursor']
if flen == 0:
break
all_count += flen
links.append(feeds)
except Exception as e:
Downloader.print_ui(str(e))
break
if len(links) < 1:
Downloader.print_ui('解析地址%s异常' % url)
return
Downloader.add_total_count(all_count)
for link in links:
self.download(link)
def post_single_trendingId(self, url, Cookie, trendingId, area):
data = {
"operationName": "hotVideoQuery",
"variables": {
"trendingId": trendingId,
"page": "detail",
"webPageArea": area
},
"query": "query hotVideoQuery($trendingId: String, $page: String, $webPageArea: String) {\n hotData(trendingId: $trendingId, page: $page, webPageArea: $webPageArea) {\n result\n llsid\n expTag\n serverExpTag\n pcursor\n webPageArea\n feeds {\n type\n trendingId\n author {\n id\n name\n headerUrl\n following\n headerUrls {\n url\n __typename\n }\n __typename\n }\n photo {\n id\n duration\n caption\n likeCount\n realLikeCount\n coverUrl\n photoUrl\n coverUrls {\n url\n __typename\n }\n timestamp\n expTag\n animatedCoverUrl\n stereoType\n videoRatio\n __typename\n }\n canAddComment\n llsid\n status\n currentPcursor\n __typename\n }\n __typename\n }\n}\n"
}
headers = {
'Host': 'www.kuaishou.com',
'Connection': 'keep-alive',
'Content-Length': '1261',
'accept': '*/*',
'User-Agent': 'Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/89.0.4389.114Safari/537.36Edg/89.0.774.68',
'content-type': 'application/json',
'Origin': 'https://www.kuaishou.com',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Dest': 'empty',
'Referer': url.encode(encoding='utf-8'),
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6'
# 'Cookie': Cookie,
}
requests.packages.urllib3.disable_warnings()
r = requests.post('https://www.kuaishou.com/graphql', data=json.dumps(data), headers=headers)
r.encoding = r.apparent_encoding
html = r.text
return html
def post_single_streamSource(self, url, Cookie, photoId, area):
data = {
"operationName": "visionVideoDetail",
"variables": {
"photoId": photoId,
"page": "detail",
"webPageArea": area
},
"query": "query visionVideoDetail($photoId: String, $type: String, $page: String, $webPageArea: String) {\n visionVideoDetail(photoId: $photoId, type: $type, page: $page, webPageArea: $webPageArea) {\n status\n type\n author {\n id\n name\n following\n headerUrl\n __typename\n }\n photo {\n id\n duration\n caption\n likeCount\n realLikeCount\n coverUrl\n photoUrl\n liked\n timestamp\n expTag\n llsid\n viewCount\n videoRatio\n stereoType\n croppedPhotoUrl\n manifest {\n mediaType\n businessType\n version\n adaptationSet {\n id\n duration\n representation {\n id\n defaultSelect\n backupUrl\n codecs\n url\n height\n width\n avgBitrate\n maxBitrate\n m3u8Slice\n qualityType\n qualityLabel\n frameRate\n featureP2sp\n hidden\n disableAdaptive\n __typename\n }\n __typename\n }\n __typename\n }\n __typename\n }\n tags {\n type\n name\n __typename\n }\n commentLimit {\n canAddComment\n __typename\n }\n llsid\n danmakuSwitch\n __typename\n }\n}\n"
}
headers = {
'Host': 'www.kuaishou.com',
'Connection': 'keep-alive',
'Content-Length': '1261',
'accept': '*/*',
'User-Agent': 'Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/89.0.4389.114Safari/537.36Edg/89.0.774.68',
'content-type': 'application/json',
'Origin': 'https://www.kuaishou.com',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Dest': 'empty',
'Referer': url.encode(encoding='utf-8'),
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Cookie': Cookie,
}
requests.packages.urllib3.disable_warnings()
r = requests.post('https://www.kuaishou.com/graphql', data=json.dumps(data), headers=headers)
r.encoding = r.apparent_encoding
html = r.text
return html
def post_user(self, userId, Cookie, pcursor):
data = {"operationName": "visionProfilePhotoList",
"variables": {"userId": userId, "pcursor": pcursor, "page": "profile"},
"query": "query visionProfilePhotoList($pcursor: String, $userId: String, $page: String, $webPageArea: String) {\n visionProfilePhotoList(pcursor: $pcursor, userId: $userId, page: $page, webPageArea: $webPageArea) {\n result\n llsid\n webPageArea\n feeds {\n type\n author {\n id\n name\n following\n headerUrl\n headerUrls {\n cdn\n url\n __typename\n }\n __typename\n }\n tags {\n type\n name\n __typename\n }\n photo {\n id\n duration\n caption\n likeCount\n realLikeCount\n coverUrl\n coverUrls {\n cdn\n url\n __typename\n }\n photoUrls {\n cdn\n url\n __typename\n }\n photoUrl\n liked\n timestamp\n expTag\n animatedCoverUrl\n stereoType\n videoRatio\n __typename\n }\n canAddComment\n currentPcursor\n llsid\n status\n __typename\n }\n hostName\n pcursor\n __typename\n }\n}\n"}
failed = {'msg': 'failed...'}
headers = {
'Host': 'video.kuaishou.com',
'Connection': 'keep-alive',
'Content-Length': '1261',
'accept': '*/*',
'User-Agent': 'Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/89.0.4389.114Safari/537.36Edg/89.0.774.68',
'content-type': 'application/json',
'Origin': 'https://video.kuaishou.com',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Dest': 'empty',
'Referer': 'https://video.kuaishou.com/profile/' + userId,
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Cookie': Cookie,
}
requests.packages.urllib3.disable_warnings()
r = requests.post(requestUrl, data=json.dumps(data), headers=headers)
r.encoding = 'UTF-8'
html = r.text
return html
def progressbar(self, url, filepath, filename):
if not os.path.exists(filepath):
os.mkdir(filepath)
start = time.time()
response = requests.get(url, stream=True)
size = 0
chunk_size = 1024
content_size = int(response.headers['content-length'])
if response.status_code == 200:
# print('Start download,[File size]:{size:.2f} MB'.format(size=content_size / chunk_size / 1024))
Downloader.print_ui(('%s Start download,[File size]:{size:.2f} MB' % filename).format(
size=content_size / chunk_size / 1024))
filename = filename.replace("\n", "")
# filepath = filepath + filename
filepath = os.path.join(filepath, filename)
try:
with open(filepath, 'wb') as file:
for data in response.iter_content(chunk_size=chunk_size):
file.write(data)
size += len(data)
Downloader.print_ui('\r' + '%s[下载进度]:%s%.2f%%' % (filename,
'>' * int(size * 50 / content_size),
float(size / content_size * 100)))
# print('\r' + '[下载进度]:%s%.2f%%' % (
# '>' * int(size * 50 / content_size), float(size / content_size * 100)), end=' ')
end = time.time()
# print('Download completed!,times: %.2f秒' % (end - start))
Downloader.print_ui('%s Download completed!,times: %.2f秒' % (filename, end - start))
except:
Downloader.add_failed_count()
Downloader.print_ui('%s [下载失败！！]' % filename)
def download(self, feeds):
author = ''
for feed in feeds:
try:
Downloader.add_downloading_count()
author = feed['author']['name']
filename = feed['photo']['caption'] + '.mp4'
# filepath = self.save + '/' + author + '/'
filepath = os.path.join(self.save, author)
filename_path = os.path.join(filepath, filename)
if not os.path.exists(filename_path):
self.progressbar(feed['photo']['photoUrl'], filepath, filename)
# print(filename + ",下载完成")
Downloader.print_ui('%s--下载完成' % filename)
Downloader.add_success_count()
else:
# print(filename + ",已存在，跳过")
Downloader.print_ui('%s--已存在，跳过' % filename)
Downloader.add_success_count()
except:
Downloader.add_failed_count()
Downloader.print_ui('%s下载失败' % author)
================================================
FILE: 001-Downloader/main.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 程序主入口
@Date :2021/08/14
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import os
import sys
from ui import Ui
# 主模块执行
if __name__ == "__main__":
path = os.path.dirname(os.path.realpath(sys.argv[0]))
# path = os.path.dirname('/Users/Qincji/Documents/zmt/')
app = Ui()
app.set_dir(path)
# to do
app.mainloop()
================================================
FILE: 001-Downloader/test/bilibili_video_download_v1.py
================================================
# !/usr/bin/python
# -*- coding:utf-8 -*-
# time: 2019/04/17--08:12
__author__ = 'Henry'
'''
项目: B站视频下载
版本1: 加密API版,不需要加入cookie,直接即可下载1080p视频
20190422 - 增加多P视频单独下载其中一集的功能
'''
import requests, time, hashlib, urllib.request, re, json
from moviepy.editor import *
import os, sys
# 访问API地址
def get_play_list(start_url, cid, quality):
entropy = 'rbMCKn@KuamXWlPMoJGsKcbiJKUfkPF_8dABscJntvqhRSETg'
appkey, sec = ''.join([chr(ord(i) + 2) for i in entropy[::-1]]).split(':')
params = 'appkey=%s&cid=%s&otype=json&qn=%s&quality=%s&type=' % (appkey, cid, quality, quality)
chksum = hashlib.md5(bytes(params + sec, 'utf8')).hexdigest()
url_api = 'https://interface.bilibili.com/v2/playurl?%s&sign=%s' % (params, chksum)
headers = {
'Referer': start_url, # 注意加上referer
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
}
# print(url_api)
html = requests.get(url_api, headers=headers).json()
# print(json.dumps(html))
video_list = []
for i in html['durl']:
video_list.append(i['url'])
# print(video_list)
return video_list
# 下载视频
'''
urllib.urlretrieve 的回调函数：
def callbackfunc(blocknum, blocksize, totalsize):
@blocknum: 已经下载的数据块
@blocksize: 数据块的大小
@totalsize: 远程文件的大小
'''
def Schedule_cmd(blocknum, blocksize, totalsize):
speed = (blocknum * blocksize) / (time.time() - start_time)
# speed_str = " Speed: %.2f" % speed
speed_str = " Speed: %s" % format_size(speed)
recv_size = blocknum * blocksize
# 设置下载进度条
f = sys.stdout
pervent = recv_size / totalsize
percent_str = "%.2f%%" % (pervent * 100)
n = round(pervent * 50)
s = ('#' * n).ljust(50, '-')
f.write(percent_str.ljust(8, ' ') + '[' + s + ']' + speed_str)
f.flush()
# time.sleep(0.1)
f.write('\r')
def Schedule(blocknum, blocksize, totalsize):
speed = (blocknum * blocksize) / (time.time() - start_time)
# speed_str = " Speed: %.2f" % speed
speed_str = " Speed: %s" % format_size(speed)
recv_size = blocknum * blocksize
# 设置下载进度条
f = sys.stdout
pervent = recv_size / totalsize
percent_str = "%.2f%%" % (pervent * 100)
n = round(pervent * 50)
s = ('#' * n).ljust(50, '-')
print(percent_str.ljust(6, ' ') + '-' + speed_str)
f.flush()
time.sleep(2)
# print('\r')
# 字节bytes转化K\M\G
def format_size(bytes):
try:
bytes = float(bytes)
kb = bytes / 1024
except:
print("传入的字节格式不对")
return "Error"
if kb >= 1024:
M = kb / 1024
if M >= 1024:
G = M / 1024
return "%.3fG" % (G)
else:
return "%.3fM" % (M)
else:
return "%.3fK" % (kb)
# 下载视频
def down_video(video_list, title, start_url, page):
num = 1
print('[正在下载P{}段视频,请稍等...]:'.format(page) + title)
currentVideoPath = os.path.join(sys.path[0], 'bilibili_video', title) # 当前目录作为下载目录
for i in video_list:
opener = urllib.request.build_opener()
# 请求头
opener.addheaders = [
# ('Host', 'upos-hz-mirrorks3.acgvideo.com'), #注意修改host,不用也行
('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:56.0) Gecko/20100101 Firefox/56.0'),
('Accept', '*/*'),
('Accept-Language', 'en-US,en;q=0.5'),
('Accept-Encoding', 'gzip, deflate, br'),
('Range', 'bytes=0-'), # Range 的值要为 bytes=0- 才能下载完整视频
('Referer', start_url), # 注意修改referer,必须要加的!
('Origin', 'https://www.bilibili.com'),
('Connection', 'keep-alive'),
]
urllib.request.install_opener(opener)
# 创建文件夹存放下载的视频
if not os.path.exists(currentVideoPath):
os.makedirs(currentVideoPath)
# 开始下载
if len(video_list) > 1:
urllib.request.urlretrieve(url=i, filename=os.path.join(currentVideoPath, r'{}-{}.mp4'.format(title, num)),
reporthook=Schedule_cmd) # 写成mp4也行 title + '-' + num + '.flv'
else:
urllib.request.urlretrieve(url=i, filename=os.path.join(currentVideoPath, r'{}.mp4'.format(title)),
reporthook=Schedule_cmd) # 写成mp4也行 title + '-' + num + '.flv'
num += 1
# 合并视频
def combine_video(video_list, title):
currentVideoPath = os.path.join(sys.path[0], 'bilibili_video', title) # 当前目录作为下载目录
if not os.path.exists(currentVideoPath):
os.makedirs(currentVideoPath)
if len(video_list) >= 2:
# 视频大于一段才要合并
print('[下载完成,正在合并视频...]:' + title)
# 定义一个数组
L = []
# 访问 video 文件夹 (假设视频都放在这里面)
root_dir = currentVideoPath
# 遍历所有文件
for file in sorted(os.listdir(root_dir), key=lambda x: int(x[x.rindex("-") + 1:x.rindex(".")])):
# 如果后缀名为 .mp4/.flv
if os.path.splitext(file)[1] == '.flv':
# 拼接成完整路径
filePath = os.path.join(root_dir, file)
# 载入视频
video = VideoFileClip(filePath)
# 添加到数组
L.append(video)
# 拼接视频
final_clip = concatenate_videoclips(L)
# 生成目标视频文件
final_clip.to_videofile(os.path.join(root_dir, r'{}.mp4'.format(title)), fps=24, remove_temp=False)
print('[视频合并完成]' + title)
else:
# 视频只有一段则直接打印下载完成
print('[视频合并完成]:' + title)
def getAid(Bid):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
}
url = "https://api.bilibili.com/x/web-interface/view?bvid=" + Bid
print(url)
r = requests.get(url, headers=headers)
j = json.loads(r.text)
# print(j["data"]["aid"])
print(j)
return j["data"]["aid"]
if __name__ == '__main__':
# 用户输入av号或者视频链接地址
print('*' * 30 + 'B站视频下载小助手' + '*' * 30)
start = input('请输入您要下载的B站av号、bv号或者视频链接地址:')
if 'http' in start:
if 'video/BV' in start:
bv = re.findall(r'video/(.*?)\?', start)[0]
start = str(getAid(bv))
print(start)
if start.isdigit() == True: # 如果输入的是av号
# 获取cid的api, 传入aid即可
start_url = 'https://api.bilibili.com/x/web-interface/view?aid=' + start
else:
# https://www.bilibili.com/video/av46958874/?spm_id_from=333.334.b_63686965665f7265636f6d6d656e64.16
start_url = 'https://api.bilibili.com/x/web-interface/view?aid=' + re.search(r'/av(\d+)/*', start).group(1)
# https://www.bilibili.com/video/BV1jL4y1e7Uz?t=7.2
# start_url = 'https://api.bilibili.com/x/web-interface/view?aid=' + re.findall(r'video/(.*?)\?', start)[0]
print(start_url)
# 视频质量
# <accept_format><![CDATA[flv,flv720,flv480,flv360]]></accept_format>
# <accept_description><![CDATA[高清 1080P,高清 720P,清晰 480P,流畅 360P]]></accept_description>
# <accept_quality><![CDATA[80,64,32,16]]></accept_quality>
quality = input('请输入您要下载视频的清晰度(1080p:80;720p:64;480p:32;360p:16)(填写80或64或32或16):')
# 获取视频的cid,title
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
}
html = requests.get(start_url, headers=headers).json()
data = html['data']
video_title = data["title"].replace(" ", "_")
cid_list = []
if '?p=' in start:
# 单独下载分P视频中的一集
p = re.search(r'\?p=(\d+)', start).group(1)
cid_list.append(data['pages'][int(p) - 1])
else:
# 如果p不存在就是全集下载
cid_list = data['pages']
# print(cid_list)
for item in cid_list:
cid = str(item['cid'])
title = item['part']
if not title:
title = video_title
title = re.sub(r'[\/\\:*?"<>|]', '', title) # 替换为空的
print('[下载视频的cid]:' + cid)
print('[下载视频的标题]:' + title)
page = str(item['page'])
start_url = start_url + "/?p=" + page
video_list = get_play_list(start_url, cid, quality)
start_time = time.time()
down_video(video_list, title, start_url, page)
combine_video(video_list, title)
# 如果是windows系统，下载完成后打开下载目录
currentVideoPath = os.path.join(sys.path[0], 'bilibili_video') # 当前目录作为下载目录
if (sys.platform.startswith('win')):
os.startfile(currentVideoPath)
# 分P视频下载测试: https://www.bilibili.com/video/av19516333/
================================================
FILE: 001-Downloader/test/ff_video.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: ffmpeg去掉最后一帧，改变md5
@Date :2022/02/17
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import os
def cute_video(folder):
files = next(os.walk(folder))[2] # 获取文件
for file in files:
file_path = os.path.join(folder, file)
shotname, extension = os.path.splitext(file)
if len(shotname) == 0 or len(extension) == 0:
continue
out_file = os.path.join(folder, 'out-{}{}'.format(shotname, extension))
# 获取时间。输入自己系统安装的ffmpeg，注意斜杠
time = os.popen(
r"/usr/local/ffmpeg/bin/ffmpeg -i {} 2>&1 | grep 'Duration' | cut -d ' ' -f 4 | sed s/,//".format(
file_path)).read().replace('\n', '').replace(' ', '')
if '.' in time:
match_time = time.split('.')[0]
else:
match_time = time
print(match_time)
ts = match_time.split(':')
sec = int(ts[0]) * 60 * 60 + int(ts[1]) * 60 + int(ts[2])
# 从0分0秒100毫秒开始截切（目的就是去头去尾）
os.popen(r"/usr/local/ffmpeg/bin/ffmpeg -ss 0:00.100 -i {} -t {} -c:v copy -c:a copy {}".format(file_path, sec,
out_file))
# 主模块执行
if __name__ == "__main__":
# path = os.path.dirname('/Users/Qincji/Downloads/ffmpeg/')
path = os.path.dirname('需要处理的目录') # 目录下的所有视频
cute_video(path)
================================================
FILE: 001-Downloader/test/test_pyinstaller.py
================================================
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import os
# 测试打包
# import pyppeteer
# import sys
# import asyncio
# from urllib.parse import urlparse, urlunparse, urljoin
# from concurrent.futures import ThreadPoolExecutor
# from concurrent.futures._base import TimeoutError
# from functools import partial
# from typing import Set, Union, List, MutableMapping, Optional
#
# import requests
# from pyquery import PyQuery
#
# from fake_useragent import UserAgent
# from lxml.html.clean import Cleaner
# import lxml
# from lxml import etree
# from lxml.html import HtmlElement
# from lxml.html import tostring as lxml_html_tostring
# from lxml.html.soupparser import fromstring as soup_parse
# from parse import search as parse_search
# from parse import findall, Result
# from w3lib.encoding import html_to_unicode
# from tkinter import *
from tkinter.filedialog import (askdirectory)
print('输入和粗了了')
================================================
FILE: 001-Downloader/test/urls.txt
================================================
================================================
FILE: 001-Downloader/test/xhs_download.py
================================================
import os
import random
import time
import requests
from my_fake_useragent import UserAgent
ua = UserAgent(family='chrome')
pre_save = os.path.join(os.path.curdir, '0216')
'''
'''
def download_url(url, index):
try:
headers = {
'Accept': '*/*',
'Accept-Encoding': 'identity;q=1, *;q=0',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Cookie': 'xhsTrackerId=6970aca9-a496-4f50-cf98-118929f063bf; timestamp2=2022021544322a4e45f1e1dec93beb82; timestamp2.sig=jk1cFo-zHueSZUpZRvlqyJwTFoA1y8ch9t76Bfy28_Q; solar.beaker.session.id=1644906492328060192125; xhsTracker=url=index&searchengine=google',
'Host': 'v.xiaohongshu.com',
'Pragma': 'no-cache',
'Referer': url,
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36'
}
video = requests.get(url, headers=headers) # 保存视频
start = time.time() # 下载开始时间
size = 0 # 初始化已下载大小
chunk_size = 100 # 每次下载的数据大小
content_size = int(video.headers['content-length']) # 下载文件总大小
print(video.status_code)
if video.status_code == 200: # 判断是否响应成功
print(str(index) + '[文件 大小]:{size:.2f} MB'.format(size=content_size / 1024 / 1024)) # 开始下载，显示下载文件大小
v_url = os.path.join(pre_save, '{}.mp4'.format(index))
# v_url = pre_save + '[' + author_list[i] + '].mp4'
with open(v_url, 'wb') as file: # 显示进度条
for data in video.iter_content(chunk_size=chunk_size):
file.write(data)
size += len(data)
# print('\r' + i + '\n[下载进度]:%s%.2f%%' % (
# '>' * int(size * 50 / content_size), float(size / content_size * 100)))
end = time.time() # 下载结束时间
print('\n' + str(index) + '\n[下载完成]:耗时: %.2f秒\n' % (end - start)) # 输出下载用时时间
except Exception as error:
# Downloader.print_ui2(error)
print(error)
print('该页视频没有' + str(index) + ',已为您跳过\r')
if __name__ == '__main__':
ls = []
if not os.path.exists(pre_save):
os.makedirs(pre_save)
with open('../xhs/urls.txt', 'r') as f:
for line in f:
if 'http' in line:
ls.append(line.replace('\n', '').replace(' ', ''))
size = len(ls)
for i in range(0, size):
url = ls[i]
print('{}-{}'.format(i, url))
download_url(url, i)
time.sleep(random.randint(5, 10))
================================================
FILE: 001-Downloader/type_enum.py
================================================
#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
@Description:dy_download.py
@Date :2021/08/14
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
from enum import Enum
class PrintType(Enum):
log = 1
total = 2
downloading = 3
success = 4
failed = 5
================================================
FILE: 001-Downloader/ui.py
================================================
#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
@Description: 用于GUI界面显示
@Date :2021/08/14
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
from tkinter import *
from tkinter.filedialog import (askdirectory)
from douyin.dy_download import DouYin
from downloader import Downloader
from kuaishou.ks_download import KuaiShou
from type_enum import PrintType
from utils import *
# from PIL import Image, ImageTk
class Ui(Frame):
def __init__(self, master=None):
global bg_color
bg_color = '#373434'
Frame.__init__(self, master, bg=bg_color)
self.ui_width = 0
self.pack(expand=YES, fill=BOTH)
self.window_init()
self.createWidgets()
def window_init(self):
self.master.title(
'欢迎使用-自媒体资源下载器' + Config.instance().get_version_name() + '，本程序仅用于学习交流！如有疑问请联系：xhunmon@gmail.com')
self.master.bg = bg_color
width, height = self.master.maxsize()
# self.master.geometry("{}x{}".format(width, height))
self.master.geometry("%dx%d+%d+%d" % (width / 2, height / 2, width / 4, height / 4))
self.ui_width = width / 2
def createWidgets(self):
# fm1
self.fm1 = Frame(self, bg=bg_color)
self.fm1.pack(fill='y', pady=10)
# window没有原生PIL 64位支持
# load = Image.open('res/logo.png')
# load.thumbnail((38, 38), Image.ANTIALIAS)
# initIamge = ImageTk.PhotoImage(load)
# self.panel = Label(self.fm1, image=initIamge, bg=bg_color)
# self.panel.image = initIamge
# self.panel.pack(side=LEFT, fill='y', padx=5)
self.titleLabel = Label(self.fm1, text="资源下载器", font=('微软雅黑', 32), fg="white", bg=bg_color)
self.titleLabel.pack(side=LEFT, fill='y')
# fm2
self.fm2 = Frame(self, bg=bg_color)
self.fm2.pack(side=TOP, fill="y")
self.fm2_right = Frame(self.fm2, bg=bg_color)
self.fm2_right.pack(side=RIGHT, padx=0, pady=10, expand=YES, fill='y')
self.fm2_left = Frame(self.fm2, bg=bg_color)
self.fm2_left.pack(side=LEFT, padx=15, pady=10, expand=YES, fill='x')
self.fm2_left_top = Frame(self.fm2_left, bg=bg_color)
self.fm2_left_bottom = Frame(self.fm2_left, bg=bg_color)
self.downloadBtn = Button(self.fm2_right, text='开始下载', fg="#ffffff", bg=bg_color,
font=('微软雅黑', 18), command=self.start_download)
self.downloadBtn.pack(side=RIGHT)
self.dirEntry = Entry(self.fm2_left_top, font=('微软雅黑', 14), width='72', fg='#ffffff', bg=bg_color, bd=1)
self.dirEntry.config(insertbackground='#ffffff')
# self.set_dir(os.path.dirname(os.path.realpath(sys.argv[0])))
self.dirBtn = Button(self.fm2_left_top, text='选择保存目录：', bg=bg_color, fg='#aaaaaa',
font=('微软雅黑', 12), width='10', command=self.save_dir)
self.dirBtn.pack(side=LEFT)
self.dirEntry.pack(side=LEFT, fill='y')
self.fm2_left_top.pack(side=TOP, fill='x')
self.urlEntry = Entry(self.fm2_left_bottom, font=('微软雅黑', 14), width='72', fg='#ffffff', bg=bg_color, bd=1)
self.urlEntry.config(insertbackground='#ffffff')
self.urlButton = Button(self.fm2_left_bottom, text='清空下载地址：', bg=bg_color, fg='#aaaaaa',
font=('微软雅黑', 12), width='10', command=self.download_url)
self.urlButton.pack(side=LEFT)
self.urlEntry.pack(side=LEFT, fill='y')
self.fm2_left_bottom.pack(side=TOP, pady=10, fill='x')
# fm3 任务数状态
self.fm3 = Frame(self, bg=bg_color, height=6)
self.fm3.pack(side=TOP, fill="x")
self.totalLabel = Label(self.fm3, width=10, text="预计总数：0", font=('微软雅黑', 12), fg="white", bg=bg_color)
self.totalLabel.pack(side=LEFT, fill='y', padx=20)
self.downloadingLabel = Label(self.fm3, width=10, text="正在下载：0", font=('微软雅黑', 12), fg="white", bg=bg_color)
self.downloadingLabel.pack(side=LEFT, fill='y', padx=20)
self.successLabel = Label(self.fm3, width=10, text="已完成：0", font=('微软雅黑', 12), fg="white", bg=bg_color)
self.successLabel.pack(side=LEFT, fill='y', padx=20)
self.failLabel = Label(self.fm3, width=10, text="已失败：0", font=('微软雅黑', 12), fg="white", bg=bg_color)
self.failLabel.pack(side=LEFT, fill='y', padx=20)
# fm4
self.fm4 = Frame(self, bg=bg_color)
self.fm4.pack(side=TOP, expand=YES, fill="both")
self.logLabel = Label(self.fm4, anchor='w', wraplength=self.ui_width - 40, text="", font=('微软雅黑', 12),
fg="white",
bg=bg_color)
self.logLabel.pack(side=TOP, fill='both', padx=20)
# 注册回调
Downloader.func_ui_print = self.func_ui_print
# 判断是否有网络
if Downloader.get_beijing_time() == 0:
self.output("获取数据异常，请检查您的网络！")
else:
Downloader.print_hint()
def save_dir(self):
path = askdirectory()
self.set_dir(path)
def set_dir(self, path):
self.dirEntry.delete(0, END)
self.dirEntry.insert(0, path)
def download_url(self):
ground_truth = ''
self.urlEntry.delete(0, END)
self.urlEntry.insert(0, ground_truth)
def output(self, txt):
self.logLabel.config(text=txt)
def func_ui_print(self, txt, print_type: PrintType = None):
if print_type == PrintType.log:
self.logLabel.config(text=txt)
elif print_type == PrintType.total:
self.totalLabel.config(text=txt)
elif print_type == PrintType.downloading:
self.downloadingLabel.config(text=txt)
elif print_type == PrintType.success:
self.successLabel.config(text=txt)
elif print_type == PrintType.failed:
self.failLabel.config(text=txt)
def start_download(self):
# 判断是否有网络
if Downloader.get_beijing_time() == 0:
self.output("获取数据异常，请检查您的网络！")
return
if Downloader.is_expired():
self.output("授权证书已到期，请联系客服！")
return
url = self.urlEntry.get()
path = self.dirEntry.get()
domain = get_domain(url)
if "kwaicdn" in domain or "kuaishou" in domain:
downloader: KuaiShou = KuaiShou()
# downloader.set_cookie()
else:
downloader: Downloader = DouYin()
downloader_t = threading.Thread(target=downloader.start, args=(url, path))
downloader_t.setDaemon(True) # 设置守护进程，避免界面卡死
downloader_t.start()
================================================
FILE: 001-Downloader/utils.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description:工具类
@Date :2021/08/16
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import threading
import configparser
import os
import re
def get_domain(url: str = None):
"""
获取链接地址的域名
:param url:
:return:
"""
# http://youtube.com/watch
return re.match(r"(http://|https://).*?\/", url, re.DOTALL).group(0)
class Config(object):
"""
配置文件的单例类
"""
_instance_lock = threading.Lock()
def __init__(self):
parent_dir = os.path.dirname(os.path.abspath(__file__))
conf_path = os.path.join(parent_dir, 'config.ini')
self.conf = configparser.ConfigParser()
self.conf.read(conf_path, encoding="utf-8")
@classmethod
def instance(cls, *args, **kwargs):
with Config._instance_lock:
if not hasattr(Config, "_instance"):
Config._instance = Config(*args, **kwargs)
return Config._instance
def get_expired_time(self):
return self.conf.get("common", "expired_time")
def get_version_name(self):
return self.conf.get("common", "version_name")
def get_version_code(self):
return self.conf.get("common", "version_code")
================================================
FILE: 002-V2rayPool/.gitignore
================================================
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
================================================
FILE: 002-V2rayPool/002-V2rayPool.iml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
================================================
FILE: 002-V2rayPool/README.md
================================================
# v2ray节点代理池
学习python爬虫过程中，我们需要一些代理。而本项目则是通过收集网上公开的节点，来实现自己的代理请求的过程！
## v2ray是什么？如何使用？
- [v2ray官方指导](https://www.v2ray.com/index.html)
- [v2ray wiki](https://zh.wikipedia.org/wiki/V2Ray)
## v2ray客户端推荐
- [window：v2rayN](https://github.com/2dust/v2rayN/releases)
- [macOS：V2rayU](https://github.com/yanue/V2rayU/releases)
- [android：v2rayNG](https://github.com/2dust/v2rayNG/releases)
- ios推荐Shadowrocket（俗称小火箭，注意不是shadowrocket VPN），但需要付费的，可以通过找"美区apple id共享2021小火箭"，找共享的账号下载软件，但是会有风险，请一定要注意。
## v2ray服务端推荐
有些应用场景会需要到用到专用的ip代理，如Tik Tok、亚马逊和Facebook等。这是，我们通过购买国外的服务器或者vps来搭建代理服务器，从而实现专有ip代理。
推荐使用[x-ui](https://github.com/vaxilu/x-ui) 进行非常简单的"一键式"搭建开源框架。
#本项目主要知识点
学习本项目需要先了解代理原理，以及v2ray实现的原理。
## 实现思路
## v2ray内核使用
我们找的是v2ray节点，所以这些协议只能运行在v2ray特有程序中。因此，我们要找[v2ray内核](https://github.com/v2ray/v2ray-core/releases) 。 这里就以macOS系统举例说明：
1. 下载[v2ray-core-v4.31.0](https://github.com/v2fly/v2ray-core/releases/download/v4.31.0/v2ray-macos-64.zip)
2. 配置解压目录的路径：
```python
Config.set_v2ray_core_path('xxx/v2ray-macos-64')
```
3. 查看是否能正常启动：
```python
client.Creator().v2ray_start('xxx')
#如果需要开启全局代理
client.Creator().v2ray_start('xxx',True)
```
## 2022-1-11检测可用测试节点(注意去掉后面","开始的内容)：
```shell
ss://YWVzLTI1Ni1nY206MWY2YWNhM2NlYmQyMWE0Y2Q1YTgwNzE4ZWQxNmI3NGNAMTIwLjIzMi4yMTQuMzY6NTAwMg#%F0%9F%87%B8%F0%9F%87%ACSingapore,8.25.96.100,美国 Level3
ss://YWVzLTI1Ni1nY206Y2RCSURWNDJEQ3duZklO@139.99.62.207:8119#github.com/freefq%20-%20%E6%96%B0%E5%8A%A0%E5%9D%A1OVH%201,139.99.62.207,新加坡 OVH
ss://YWVzLTI1Ni1nY206UmV4bkJnVTdFVjVBRHhH@167.88.61.60:7002#github.com/freefq%20-%20%E7%91%9E%E5%85%B8%20%203,167.88.61.60,美国加利福尼亚圣克拉拉
ss://YWVzLTI1Ni1nY206WTZSOXBBdHZ4eHptR0M@38.143.66.71:3389#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%E5%8D%8E%E7%9B%9B%E9%A1%BFCogent%E9%80%9A%E4%BF%A1%E5%85%AC%E5%8F%B8%204,38.143.66.71,美国华盛顿西雅图 Cogent
trojan://e6c36d58-6070-4b55-a437-146e6b53ec57@t1.ssrsub.com:8443#github.com/freefq%20-%20%E5%8A%A0%E6%8B%BF%E5%A4%A7%20%2012,142.47.89.64,加拿大安大略
ss://YWVzLTI1Ni1nY206ZTRGQ1dyZ3BramkzUVk@172.99.190.87:9101#github.com/freefq%20-%20%E7%BE%8E%E5%9B%BD%20%2013,172.99.190.87,美国乔治亚亚特兰大
ss://YWVzLTI1Ni1nY206UENubkg2U1FTbmZvUzI3@46.29.218.6:8091#github.com/freefq%20-%20%E6%8C%AA%E5%A8%81%20%2019,46.29.218.6,挪威
```
注意：本程序虽可跨平台，但因博主能力有限，只在macos系统操作过，无法在更多系统上去尝试和改进，望谅解！
-------
如有可用节点增加，请推荐给博主吧：xhunmon@gmail.com
================================================
FILE: 002-V2rayPool/base/net_proxy.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 网络代理请求的基类
@Date :2021/09/15
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import requests
from my_fake_useragent import UserAgent
class Net(object):
"""
基类，封装常用接口
"""
TIMEOUT = 8
def __init__(self, timeout=8):
Net.TIMEOUT = timeout
self._ua = UserAgent()
self._agent = self._ua.random() # 随机生成的agent
self.USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:65.0) Gecko/20100101 Firefox/65.0"
self._headers = {"user-agent": self.USER_AGENT, 'Connection': 'close'}
# determine
self._proxy = '127.0.0.1:1080'
self._proxies_en = {
'http': 'socks5h://' + self._proxy,
'https': 'socks5h://' + self._proxy,
}
self._proxies_zh = {
'http': 'socks5://' + self._proxy,
'https': 'socks5://' + self._proxy,
}
def get_header(self, headers, key):
key_lower = key.lower()
headers_lower = {k.lower(): v for k, v in headers.items()}
if (key_lower in headers_lower):
return headers_lower[key_lower]
else:
return ''
def update_agent(self):
self.USER_AGENT = self._ua.random() # 随机生成的agent
self._headers = {"user-agent": self.USER_AGENT, 'Connection': 'close'}
def get_urls(self) -> []:
"""需子类实现"""
pass
def request(self, url, allow_redirects=False, verify=False, timeout=TIMEOUT):
"""普通请求"""
return self.__request(url, allow_redirects=allow_redirects, verify=verify, timeout=timeout)
def request_en(self, url, allow_redirects=False, verify=False, timeout=TIMEOUT):
"""国外网站请求，需要开代理"""
return self.__request(url, allow_redirects=allow_redirects, verify=verify, proxies=self._proxies_en,
timeout=timeout)
def request_zh(self, url, allow_redirects=False, verify=False, timeout=TIMEOUT):
"""国内网站请求，需要开代理"""
return self.__request(url, allow_redirects=allow_redirects, verify=verify, proxies=self._proxies_zh,
timeout=timeout)
def __request(self, url, allow_redirects=False, verify=False, proxies=None, timeout=TIMEOUT):
"""最终的请求实现"""
requests.packages.urllib3.disable_warnings()
if proxies:
return requests.get(url=url, headers=self._headers, allow_redirects=allow_redirects, verify=verify,
proxies=proxies, timeout=timeout)
else:
return requests.get(url=url, headers=self._headers, allow_redirects=allow_redirects, verify=verify,
timeout=timeout)
================================================
FILE: 002-V2rayPool/core/client.py
================================================
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import base64
import json
import os
import urllib
from core.conf import Config
from core.group import Vmess, Vless, Socks, SS, Mtproto, Trojan, Group, Dyport
from core.utils import ProtocolType
import core.utils as util
class ClientWriter:
def __init__(self, group):
self.config_factory = Config()
# with open(self.config_factory.get_path('config_path'), 'r') as json_file:
# self.config = json.load(json_file)
self.write_path = self.config_factory.get_path("config_path")
self.template_path = self.config_factory.json_path
self.group = group
self.node = group.node
def load_template(self, template_name):
'''
load special template
'''
with open(self.template_path + "/" + template_name, 'r') as stream_file:
template = json.load(stream_file)
return template
def transform(self):
user_json = None
if type(self.node) == Vmess:
self.client_config = self.load_template('client.json')
user_json = self.client_config["outbounds"][0]["settings"]["vnext"][0]
user_json["users"][0]["id"] = self.node.password
user_json["users"][0]["alterId"] = self.node.alter_id
elif type(self.node) == Vless:
self.client_config = self.load_template('client.json')
user_json = self.client_config["outbounds"][0]["settings"]["vnext"][0]
user_json["users"][0]["id"] = self.node.password
del user_json["users"][0]["alterId"]
del user_json["users"][0]["security"]
user_json["users"][0]["encryption"] = self.node.encryption
if self.node.flow:
user_json["users"][0]["flow"] = self.node.flow
self.client_config["outbounds"][0]["protocol"] = "vless"
elif type(self.node) == Socks:
self.client_config = self.load_template('client_socks.json')
user_json = self.client_config["outbounds"][0]["settings"]["servers"][0]
user_json["users"][0]["user"] = self.node.user_info
user_json["users"][0]["pass"] = self.node.password
elif type(self.node) == SS:
self.client_config = self.load_template('client_ss.json')
user_json = self.client_config["outbounds"][0]["settings"]["servers"][0]
user_json["method"] = self.node.method
user_json["password"] = self.node.password
elif type(self.node) == Trojan:
self.client_config = self.load_template('client_trojan.json')
user_json = self.client_config["outbounds"][0]["settings"]["servers"][0]
user_json["password"] = self.node.password
elif type(self.node) == Mtproto:
print("")
print("MTProto protocol only use Telegram, and can't generate client json!")
print("")
exit(-1)
try:
if isinstance(self.group.port, int):
user_json["port"] = self.group.port
else:
user_json["port"] = int(self.group.port)
user_json["address"] = self.group.ip
self.client_config["inbounds"][0]["listen"] = self.group.listen
self.client_config["inbounds"][0]["port"] = self.group.i_port
except:
print('数据异常，启动失败')
return
# inbounds = self.client_config["inbounds"]
# for inbound in inbounds:
# inbound["listen"] = self.group.listen
# if type(self.node) != SS:
# self.client_config["outbounds"][0]["streamSettings"] = self.config["inbounds"][self.group.index][
# "streamSettings"]
if self.group.tls == 'tls':
self.client_config["outbounds"][0]["streamSettings"]["tlsSettings"] = {}
elif self.group.tls == 'xtls':
self.client_config["outbounds"][0]["streamSettings"]["xtlsSettings"]["serverName"] = self.group.ip
del self.client_config["outbounds"][0]["streamSettings"]["xtlsSettings"]["certificates"]
del self.client_config["outbounds"][0]["streamSettings"]["xtlsSettings"]["alpn"]
del self.client_config["outbounds"][0]["mux"]
def write(self):
'''
写客户端配置文件函数
'''
json_dump = json.dumps(self.client_config, indent=1)
with open(self.write_path, 'w') as write_json_file:
write_json_file.writelines(json_dump)
# print("{0}({1})".format("save json success!", self.write_path))
class Creator(object):
"""
生成代理json并启动
"""
def __init__(self):
self.__thread = None
self.__main_pid = os.getpid()
def parse_vmess(self, vmesslink):
"""返回：{'v': '2', 'ps': 'https://git.io/v9999 圣何塞sv2', 'add': 'sv2.free3333.xyz', 'port': '26707', 'id': '8f7a28a6-002a-11ec-b64a-00163cf00cd9', 'aid': '0', 'net': 'tcp', 'type': 'none', 'host': '', 'path': '', 'tls': '', 'sni': ''}"""
if vmesslink.startswith(ProtocolType.VMESS):
bs = vmesslink[len(ProtocolType.VMESS):]
# paddings
blen = len(bs)
if blen % 4 > 0:
bs += "=" * (4 - blen % 4)
vms = base64.b64decode(bs).decode()
return json.loads(vms)
else:
raise Exception("vmess link invalid")
def parse_trojan(self, link):
link = urllib.parse.unquote(link)
trStr = link[link.find("//") + 2:]
password = trStr[:trStr.find('@')]
trStr = trStr[trStr.find('@') + 1:]
sni = trStr[:trStr.find(':')]
trStr = trStr[trStr.find(':') + 1:]
port = trStr[:trStr.find('#')]
name = trStr[trStr.find('#') + 1:]
node = {
"name": name,
"server": sni,
"port": port,
"type": "trojan",
"password": password,
"sni": sni
}
return node
def parse_ss(self, sslink):
RETOBJ = {
"v": "2",
"ps": "",
"add": "",
"port": "",
"id": "",
"aid": "",
"net": "shadowsocks",
"type": "",
"host": "",
"path": "",
"tls": ""
}
if sslink.startswith(ProtocolType.SS):
info = sslink[len(ProtocolType.SS):]
if info.rfind("#") > 0:
info, _ps = info.split("#", 2)
RETOBJ["ps"] = urllib.parse.unquote(_ps)
if info.find("@") < 0:
# old style link
# paddings
blen = len(info)
if blen % 4 > 0:
info += "=" * (4 - blen % 4)
info = base64.b64decode(info).decode()
atidx = info.rfind("@")
method, password = info[:atidx].split(":", 2)
addr, port = info[atidx + 1:].split(":", 2)
else:
atidx = info.rfind("@")
addr, port = info[atidx + 1:].split(":", 2)
info = info[:atidx]
blen = len(info)
if blen % 4 > 0:
info += "=" * (4 - blen % 4)
info = base64.b64decode(info).decode()
method, password = info.split(":", 2)
RETOBJ["add"] = addr
RETOBJ["port"] = port
RETOBJ["aid"] = method
RETOBJ["id"] = password
return RETOBJ
def generateAndWrite(self, url: str):
# listen="127.0.0.1",
group = Group(None, 1024, end_port=None, tls="none", tfo="open", dyp=Dyport(), index=0)
if url.startswith(ProtocolType.VMESS):
_json = self.parse_vmess(url.strip())
node = Vmess(uuid=_json['id'], alter_id=int(_json['aid']), network=_json['net'], user_number=1,
path=_json['path'] if 'path' in _json else None,
host=_json['host'], header=None, email=None,
quic=None)
group.port = _json['port']
group.tls = _json['tls']
group.ip = _json['add']
group.protocol = node.__class__.__name__
group.node = node
print(_json)
elif url.startswith(ProtocolType.SS):
_json = self.parse_ss(url.strip())
# {'v': '2', 'ps': 'github.com/freefq - 罗马尼亚 10', 'add': '194.110.115.83', 'port': '43893', 'id': 'YyCBeDdYX4cadHpCkkmdJLq8', 'aid': 'aes-256-gcm', 'net': 'shadowsocks', 'type': '', 'host': '', 'path': '', 'tls': ''}
node = SS(0, _json['id'], _json['aid'], None)
group.port = _json['port']
group.tls = _json['tls']
group.ip = _json['add']
group.protocol = node.__class__.__name__
group.node = node
print(_json)
elif url.startswith(ProtocolType.TROJAN):
_json = self.parse_trojan(url.strip())
node = Trojan(0, _json['password'], None)
group.port = _json['port']
group.tls = ''
group.ip = _json['sni']
group.protocol = node.__class__.__name__
group.node = node
print(_json)
else:
print('无效地址：%s' % url)
return
cw = ClientWriter(group)
cw.transform()
cw.write()
def __kill_threading(self):
pids = os.popen("ps aux |grep v2ray |awk '{print $2}'").read().split('\n')
pid_all = []
for pid in pids:
temp = pid.strip()
if len(temp) > 1 and temp != 'PID' and temp != '0' and temp != str(self.__main_pid) and temp not in pid_all:
pid_all.append(temp)
for pid in pid_all:
try:
import subprocess
# subprocess.check_output("kill %d" % int(pid))
a = os.popen("kill %d" % int(pid)).read()
except Exception as e:
pass
util.sys_proxy_off()
def __child_thread(self, url: str, isSysOn=False):
self.generateAndWrite(url)
# 执行就可，不需要知道结果
if Config.get_v2ray_core_path() is None:
raise Exception('请先调用#Config.set_v2ray_core_path 设置路径')
v2ray_path = os.path.join(Config.get_v2ray_core_path(), 'v2ray')
config_path = os.path.join(Config.get_v2ray_core_path(), 'config.json')
os.popen("%s -config %s >/dev/null 2>&1" % (v2ray_path, config_path))
print("%s -config %s >/dev/null 2>&1" % (v2ray_path, config_path))
if isSysOn:
util.sys_v2ray_on()
def v2ray_start(self, url: str, isSysOn=False):
self.__kill_threading()
self.__child_thread(url, isSysOn)
def v2ray_start_with_log(self, url: str, isSysOn=False):
try:
self.v2ray_start(url, isSysOn)
except Exception as e:
print(e)
print("启动异常：%s" % url)
return False
return True
================================================
FILE: 002-V2rayPool/core/conf.py
================================================
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import configparser
import os
class Config:
__v2ray_core_path = None
__v2ray_node_path = None
def __init__(self):
self.config = configparser.ConfigParser()
parent_dir = os.path.dirname(os.path.abspath(__file__))
self.config_path = os.path.join(Config.__v2ray_core_path, 'config.json')
self.json_path = os.path.join(parent_dir, 'json_template')
# self.config.read(self.config_path)
def get_path(self, key):
# return self.config.get('path', key)
return self.config_path
def get_data(self, key):
return self.config.get('data', key)
def set_data(self, key, value):
self.config.set('data', key, value)
self.config.write(open(self.config_path, "w"))
@staticmethod
def set_v2ray_core_path(dir: str):
"""设置当前v2ray_core程序的目录"""
Config.__v2ray_core_path = dir
@staticmethod
def get_v2ray_core_path():
"""获取当前v2ray_core程序的目录"""
return Config.__v2ray_core_path
@staticmethod
def set_v2ray_node_path(dir: str):
"""设置当前v2ray保存节点的目录"""
Config.__v2ray_node_path = dir
@staticmethod
def get_v2ray_node_path():
"""获取当前v2ray保存节点的目录"""
return Config.__v2ray_node_path
================================================
FILE: 002-V2rayPool/core/group.py
================================================
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import base64
import json
from urllib.parse import quote
__author__ = 'qincji'
class Dyport(object):
def __init__(self, status=False, aid=0):
self.status = status
self.aid = aid
class Quic(object):
def __init__(self, security="none", key="", header="none"):
self.security = security
self.key = key
self.header = header
class User(object):
def __init__(self, user_number, password, user_info=None):
"""
user_info可能是email, 也可能user_name, 具体取决于group的protocol
password: id或者密码
"""
self.__password = password
self.user_info = user_info
self.user_number = user_number
@property
def password(self):
return self.__password
class SS(User):
def __init__(self, user_number, password, method, user_info):
super(SS, self).__init__(user_number, password, user_info)
self.method = method
def __str__(self):
if self.user_info:
return "Email: {self.user_info}\nMethod: {self.method}\nPassword: {password}\n".format(self=self,
password=self.password)
else:
return "Method: {self.method}\nPassword: {password}\n".format(self=self, password=self.password)
def link(self, ip, port, tls):
ss_origin_url = "{0}:{1}@{2}:{3}".format(self.method, self.password, ip, port)
return "ss://{}".format(bytes.decode(base64.b64encode(bytes(ss_origin_url, 'utf-8'))))
def stream(self):
return "shadowsocks"
class Trojan(User):
def __init__(self, user_number, password, email):
super(Trojan, self).__init__(user_number, password, email)
def __str__(self):
if self.user_info:
return "Email: {self.user_info}\nPassword: {password}\n".format(self=self, password=self.password)
else:
return "Password: {password}\n".format(password=self.password)
def link(self, ip, port, tls):
return "trojan://{0}@{1}:{2}".format(self.password, ip, port)
def stream(self):
return "trojan"
class Mtproto(User):
def __str__(self):
if self.user_info:
return "Email: {}\nSecret: {}\n".format(self.user_info, self.password)
else:
return "Secret: {}\n".format(self.password)
def link(self, ip, port, tls):
return "tg://proxy?server={0}&port={1}&secret={2}".format(ip, port, self.password)
def stream(self):
return "mtproto"
class Socks(User):
def __str__(self):
return "User: {0}\nPass: {1}\nUDP: true\n".format(self.user_info, self.password)
def link(self, ip, port, tls):
if tls == "tls":
return "HTTPS Socks5 don't support telegram share link"
else:
return "tg://socks?server={0}&port={1}&user={2}&pass={3}".format(ip, port, self.user_info, self.password)
def stream(self):
return "socks"
class Vless(User):
def __init__(self, uuid, user_number, encryption=None, email=None, network=None, path=None, host=None, header=None,
flow="", serviceName="", mode=""):
super(Vless, self).__init__(user_number, uuid, email)
self.encryption = encryption
self.path = path
self.host = host
self.header = header
self.network = network
self.flow = flow
self.serviceName = serviceName
self.mode = mode
def __str__(self):
email = ""
if self.user_info:
email = "Email: {}".format(self.user_info)
result = '''
{email}
ID: {password}
Encryption: {self.encryption}
Network: {network}
'''.format(self=self, password=self.password, email=email, network=self.stream()).strip() + "\n"
return result
def stream(self):
if self.network == "ws":
return "WebSocket host: {0}, path: {1}".format(self.host, self.path)
elif self.network == "tcp":
return "tcp"
elif self.network == "grpc":
return "grpc serviceName: {}, mode: {}".format(self.serviceName, self.mode)
elif self.network == "kcp":
result = "kcp"
if self.header and self.header != 'none':
result = "{} {}".format(result, self.header)
if self.path != "":
result = "{} seed: {}".format(result, self.path)
return result
def link(self, ip, port, tls):
result_link = "vless://{s.password}@{ip}:{port}?encryption={s.encryption}".format(s=self, ip=ip, port=port)
if tls == "tls":
result_link += "&security=tls"
elif tls == "xtls":
result_link += "&security=xtls&flow={}".format(self.flow)
if self.network == "ws":
result_link += "&type=ws&host={0}&path={1}".format(self.host, quote(self.path))
elif self.network == "tcp":
result_link += "&type=tcp"
elif self.network == "grpc":
result_link += "&type=grpc&serviceName={}&mode={}".format(self.serviceName, self.mode)
elif self.network == "kcp":
result_link += "&type=kcp&headerType={0}&seed={1}".format(self.header, self.path)
return result_link
class Vmess(User):
def __init__(self, uuid, alter_id: int, network: str, user_number, *, path=None, host=None, header=None, email=None,
quic=None):
super(Vmess, self).__init__(user_number, uuid, email)
self.alter_id = alter_id
self.network = network
self.path = path
self.host = host
self.header = header
self.quic = quic
if quic:
self.header = quic.header
self.host = quic.security
self.path = quic.key
def stream(self):
network = ""
if self.network == "quic":
network = "Quic\n{}".format(self.quic)
elif self.network == "h2":
network = "HTTP/2 path: {}".format(self.path)
elif self.network == "ws":
network = "WebSocket host: {0}, path: {1}".format(self.host, self.path)
elif self.network == "tcp":
if self.host:
network = "tcp host: {0}".format(self.host)
else:
network = "tcp"
elif self.network == "kcp":
network = "kcp"
if self.header and self.header != 'none':
network = "{} {}".format(network, self.header)
if self.path != "":
network = "{} seed: {}".format(network, self.path)
return network
def __str__(self):
email = ""
if self.user_info:
email = "Email: {}".format(self.user_info)
result = '''
{email}
UUID: {uuid}
Alter ID: {self.alter_id}
Network: {network}
'''.format(self=self, uuid=self.password, email=email, network=self.stream()).strip() + "\n"
return result
def link(self, ip, port, tls):
json_dict = {
"v": "2",
"ps": "",
"add": ip,
"port": port,
"aid": self.alter_id,
"type": self.header,
"net": self.network,
"path": self.path,
"host": self.host,
"id": self.password,
"tls": tls
}
json_data = json.dumps(json_dict)
result_link = "vmess://{}".format(bytes.decode(base64.b64encode(bytes(json_data, 'utf-8'))))
return result_link
class Group(object):
def __init__(self, ip, port: str, *, end_port=None, tfo=None, tls="none", i_port='1080', listen="0.0.0.0",
dyp=Dyport(),
index=0,
tag='A'):
self.ip = ip
self.port = port
self.end_port = end_port
self.tag = tag
self.node = None
self.tfo = tfo
self.tls = tls
self.dyp = dyp
self.protocol = None
self.index = index
self.listen = listen
self.i_port = i_port
# port = 101
# key = str(port) if isinstance(port, int) else port
# print(key)
# if isinstance(key, int):
# print('key is int')
# if isinstance(key, str):
# print('key is str')
================================================
FILE: 002-V2rayPool/core/json_template/client.json
================================================
{
"log": {
"access": "",
"error": "",
"loglevel": "info"
},
"inbounds": [
{
"port": 1080,
"listen": "0.0.0.0",
"protocol": "socks",
"settings": {
"auth": "noauth",
"udp": true,
"ip": "127.0.0.1",
"clients": null
},
"streamSettings": null
},
{
"listen": "127.0.0.1",
"protocol": "http",
"settings": {
"timeout": 360
},
"port": "1087"
}
],
"outbounds": [
{
"protocol": "vmess",
"settings": {
"vnext": [
{
"address": "123.ocm",
"port": 1234,
"users": [
{
"id": "cc4f8d5b-967b-4557-a4b6-bde92965bc27",
"alterId": 0,
"security": "aes-128-gcm"
}
]
}
]
},
"streamSettings": {
"security": "",
"tlsSettings": {},
"wsSettings": {},
"httpSettings": {},
"network": "tcp",
"kcpSettings": {},
"tcpSettings": {},
"quicSettings": {}
},
"mux": {
"enabled": true
}
},
{
"protocol": "freedom",
"settings": {
"response": null
},
"tag": "direct"
}
],
"dns": {
"servers": [
"8.8.8.8",
"8.8.4.4",
"localhost"
]
},
"routing": {
"domainStrategy": "IPIfNonMatch",
"rules": [
{
"type": "field",
"ip": ["geoip:private"],
"outboundTag": "direct"
},
{
"type": "field",
"domain": ["geosite:cn"],
"outboundTag": "direct"
},
{
"type": "field",
"domain": ["geoip:cn"],
"outboundTag": "direct"
}
]
}
}
================================================
FILE: 002-V2rayPool/core/json_template/client_socks.json
================================================
{
"log": {
"access": "",
"error": "",
"loglevel": "info"
},
"inbounds": [
{
"port": 1080,
"listen": "0.0.0.0",
"protocol": "socks",
"settings": {
"auth": "noauth",
"udp": true,
"ip": "127.0.0.1",
"clients": null
},
"streamSettings": null
},
{
"listen": "127.0.0.1",
"protocol": "http",
"settings": {
"timeout": 360
},
"port": "1087"
}
],
"outbounds":
[
{
"protocol": "socks",
"settings": {
"servers": [
{
"address": "123.ocm",
"port": 1234,
"users": [
{
"user": "hello",
"pass": "3.1415"
}
]
}
]
},
"streamSettings": {
"security": "",
"tlsSettings": {},
"wsSettings": {},
"httpSettings": {},
"network": "tcp",
"kcpSettings": {},
"tcpSettings": {},
"quicSettings": {}
},
"mux": {
"enabled": true
}
},
{
"protocol": "freedom",
"settings": {
"response": null
},
"tag": "direct"
}
],
"routing": {
"domainStrategy": "IPIfNonMatch",
"rules": [
{
"type": "field",
"ip": ["geoip:private"],
"outboundTag": "direct"
},
{
"type": "field",
"domain": ["geosite:cn"],
"outboundTag": "direct"
},
{
"type": "field",
"domain": ["geoip:cn"],
"outboundTag": "direct"
}
]
}
}
================================================
FILE: 002-V2rayPool/core/json_template/client_ss.json
================================================
{
"log": {
"access": "",
"error": "",
"loglevel": "info"
},
"inbounds": [
{
"port": 1080,
"listen": "0.0.0.0",
"protocol": "socks",
"settings": {
"auth": "noauth",
"udp": true,
"ip": "127.0.0.1",
"clients": null
},
"streamSettings": null
},
{
"listen": "127.0.0.1",
"protocol": "http",
"settings": {
"timeout": 360
},
"port": "1087"
}
],
"outbounds": [
{
"protocol": "shadowsocks",
"settings": {
"servers": [
{
"address": "serveraddr.com",
"method": "aes-128-gcm",
"ota": false,
"password": "sspasswd",
"port": 1024
}
]
}
},
{
"protocol": "freedom",
"settings": {
"response": null
},
"tag": "direct"
}
],
"routing": {
"domainStrategy": "IPIfNonMatch",
"rules": [
{
"type": "field",
"ip": ["geoip:private"],
"outboundTag": "direct"
},
{
"type": "field",
"domain": ["geosite:cn"],
"outboundTag": "direct"
},
{
"type": "field",
"domain": ["geoip:cn"],
"outboundTag": "direct"
}
]
}
}
================================================
FILE: 002-V2rayPool/core/json_template/client_trojan.json
================================================
{
"log": {
"access": "",
"error": "",
"loglevel": "info"
},
"inbounds": [
{
"port": 1080,
"listen": "0.0.0.0",
"protocol": "socks",
"settings": {
"auth": "noauth",
"udp": true,
"ip": "127.0.0.1",
"clients": null
},
"streamSettings": null
},
{
"listen": "127.0.0.1",
"protocol": "http",
"settings": {
"timeout": 360
},
"port": "1087"
}
],
"outbounds": [
{
"protocol": "trojan",
"settings": {
"servers": [
{
"address": "serveraddr.com",
"port": 443,
"password": "passwd"
}
]
},
"streamSettings": {
"security": "tls",
"network": "tcp"
}
},
{
"protocol": "freedom",
"settings": {
"response": null
},
"tag": "direct"
}
],
"routing": {
"domainStrategy": "IPIfNonMatch",
"rules": [
{
"type": "field",
"ip": ["geoip:private"],
"outboundTag": "direct"
},
{
"type": "field",
"domain": ["geosite:cn"],
"outboundTag": "direct"
},
{
"type": "field",
"domain": ["geoip:cn"],
"outboundTag": "direct"
}
]
}
}
================================================
FILE: 002-V2rayPool/core/json_template/dyn_port.json
================================================
{
"protocol": "vmess",
"port": "10000-20000",
"tag": "dynamicPort",
"settings": {
"default": {
"alterId": 32
}
},
"allocate": {
"strategy": "random",
"concurrency": 3,
"refresh": 5
}
}
================================================
FILE: 002-V2rayPool/core/json_template/http.json
================================================
{
"network": "tcp",
"security": "none",
"tlsSettings": {},
"httpSettings": {},
"tcpSettings": {
"header": {
"type": "http",
"request": {
"version": "1.1",
"method": "GET",
"path": [
"/"
],
"headers": {
"Host": [
""
],
"User-Agent": [
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.75 Safari/537.36",
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_0_2 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/53.0.2785.109 Mobile/14A456 Safari/601.1.46"
],
"Accept-Encoding": [
"gzip, deflate"
],
"Connection": [
"keep-alive"
],
"Pragma": "no-cache"
}
},
"response": {
"version": "1.1",
"status": "200",
"reason": "OK",
"headers": {
"Content-Type": [
"application/octet-stream",
"video/mpeg"
],
"Transfer-Encoding": [
"chunked"
],
"Connection": [
"keep-alive"
],
"Pragma": "no-cache"
}
}
}
},
"kcpSettings": {},
"wsSettings": {},
"quicSettings": {}
}
================================================
FILE: 002-V2rayPool/core/json_template/http2.json
================================================
{
"network": "h2",
"security": "tls",
"tlsSettings": {},
"tcpSettings": {},
"httpSettings": {
"path": "/ray/"
},
"kcpSettings": {},
"wsSettings": {},
"quicSettings": {}
}
================================================
FILE: 002-V2rayPool/core/json_template/kcp.json
================================================
{
"network": "kcp",
"security": "none",
"tlsSettings": {},
"tcpSettings": {},
"httpSettings": {},
"kcpSettings": {
"mtu": 1350,
"tti": 50,
"uplinkCapacity": 100,
"downlinkCapacity": 100,
"congestion": false,
"readBufferSize": 2,
"writeBufferSize": 2,
"header": {
"type": "none"
}
},
"wsSettings": {},
"quicSettings": {}
}
================================================
FILE: 002-V2rayPool/core/json_template/mtproto.json
================================================
{
"mtproto-in": {
"tag": "tg-in",
"port": 5829,
"protocol": "mtproto",
"settings": {
"users": [{"secret": "b0cbcef5a486d9636472ac27f8e11a9d"}]
}
},
"mtproto-out": {
"tag": "tg-out",
"protocol": "mtproto",
"settings": {}
},
"routing-bind": {
"type": "field",
"inboundTag": ["tg-in"],
"outboundTag": "tg-out"
}
}
================================================
FILE: 002-V2rayPool/core/json_template/quic.json
================================================
{
"network": "quic",
"security": "none",
"tlsSettings": {},
"tcpSettings": {},
"kcpSettings": {},
"httpSettings": {},
"wsSettings": {},
"quicSettings": {
"security": "none",
"key": "",
"header": {
"type": "none"
}
}
}
================================================
FILE: 002-V2rayPool/core/json_template/server.json
================================================
{
"log": {
"access": "/var/log/v2ray/access.log",
"error": "/var/log/v2ray/error.log",
"loglevel": "info"
},
"inbounds": [
{
"port": 999999999,
"protocol": "vmess",
"settings": {
"clients": [
{
"id": "cc4f8d5b-967b-4557-a4b6-bde92965bc27",
"alterId": 0
}
]
},
"streamSettings": {
"security": "none",
"tlsSettings": {},
"wsSettings": {},
"httpSettings": {},
"network": "",
"kcpSettings": {},
"tcpSettings": {},
"quicSettings": {}
}
}
],
"outbounds": [
{
"protocol": "freedom",
"settings": {}
},
{
"protocol": "blackhole",
"settings": {},
"tag": "blocked"
}
],
"routing": {
"rules": [
{
"type": "field",
"ip": [
"0.0.0.0/8",
"10.0.0.0/8",
"100.64.0.0/10",
"169.254.0.0/16",
"172.16.0.0/12",
"192.0.0.0/24",
"192.0.2.0/24",
"192.168.0.0/16",
"198.18.0.0/15",
"198.51.100.0/24",
"203.0.113.0/24",
"::1/128",
"fc00::/7",
"fe80::/10"
],
"outboundTag": "blocked"
}
]
}
}
================================================
FILE: 002-V2rayPool/core/json_template/socks.json
================================================
{
"auth": "password",
"accounts": [
{
"user": "hello",
"pass": "socks"
}
],
"udp": true
}
================================================
FILE: 002-V2rayPool/core/json_template/ss.json
================================================
{
"port": 1024,
"protocol": "shadowsocks",
"settings": {
"method": "aes-128-gcm",
"password": "password",
"network":"tcp,udp"
}
}
================================================
FILE: 002-V2rayPool/core/json_template/stats_settings.json
================================================
{
"stats": {},
"api": {
"services": [
"StatsService"
],
"tag": "api"
},
"policy": {
"levels": {
"0": {
"statsUserDownlink": true,
"statsUserUplink": true
}
},
"system": {
"statsInboundUplink": true,
"statsInboundDownlink": true
}
},
"routingRules": {
"inboundTag": [
"api"
],
"outboundTag": "api",
"type": "field"
},
"dokodemoDoor": {
"listen": "127.0.0.1",
"port": 10085,
"protocol": "dokodemo-door",
"settings": {
"address": "127.0.0.1"
},
"tag": "api"
}
}
================================================
FILE: 002-V2rayPool/core/json_template/tcp.json
================================================
{
"network": "tcp",
"security": "none",
"tlsSettings": {},
"tcpSettings": {},
"kcpSettings": {},
"wsSettings": {},
"httpSettings": {},
"quicSettings": {},
"grpcSettings": {}
}
================================================
FILE: 002-V2rayPool/core/json_template/vless.json
================================================
{
"clients": [
{
"id": "d4e321ea-e118-11ea-a265-42010a8c0002"
}
],
"decryption": "none",
"fallbacks": [
{
"dest": 80
}
]
}
================================================
FILE: 002-V2rayPool/core/json_template/ws.json
================================================
{
"network": "ws",
"security": "none",
"tlsSettings": {},
"tcpSettings": {},
"kcpSettings": {},
"httpSettings": {},
"wsSettings": {
"path": "",
"headers": {
"Host": ""
}
},
"quicSettings": {}
}
================================================
FILE: 002-V2rayPool/core/profile.py
================================================
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import json
import os
from conf import Config
from group import SS, Socks, Vmess, Vless, Mtproto, Quic, Group, Dyport, Trojan
class Stats:
def __init__(self, status=False, door_port=0):
self.status = status
self.door_port = door_port
def __str__(self):
return "open" if self.status else "close"
class Profile:
def __init__(self):
self.path = Config().get_path('config_path')
self.group_list = []
self.stats = None
self.ban_bt = False
self.user_number = 0
self.network = "ipv4"
self.modify_time = os.path.getmtime(self.path)
self.read_json()
def __str__(self):
result = ""
for group in self.group_list:
result = "{}{}".format(result, group)
result = result + "Tip: The same group's node protocol, port, tls are the same."
return result
def read_json(self):
with open(self.path, 'r') as json_file:
self.config = json.load(json_file)
# 读取配置文件大框架
conf_inbounds = self.config["inbounds"]
conf_rules = self.config["routing"]["rules"]
stats = Stats()
if "stats" in self.config:
stats.status = True
for inbound in conf_inbounds:
if "protocol" in inbound and inbound["protocol"] == "dokodemo-door":
stats.door_port = inbound["port"]
break
self.stats = stats
for rule in conf_rules:
if "protocol" in rule and "bittorrent" in rule["protocol"]:
self.ban_bt = True
# local_ip = get_ip()
local_ip = ''
if ":" in local_ip:
self.network = "ipv6"
group_ascii = 64 # before 'A' ascii code
for index, json_part in enumerate(conf_inbounds):
group = self.parse_group(json_part, index, local_ip)
if group != None:
group_ascii = group_ascii + 1
if group_ascii > 90:
group.tag = str(group_ascii)
else:
group.tag = chr(group_ascii)
self.group_list.append(group)
del self.config
def parse_group(self, part_json, group_index, local_ip):
dyp, quic, end_port, tfo, header, tls, path, host, conf_ip, serviceName, mode = Dyport(), None, None, None, "", "", "", "", local_ip, "", "gun"
protocol = part_json["protocol"]
if protocol == 'dokodemo-door' or (protocol == "vmess" and "streamSettings" not in part_json):
return
conf_settings = part_json["settings"]
port_info = str(part_json["port"]).split("-", 2)
if "domain" in part_json and part_json["domain"]:
conf_ip = part_json["domain"]
if len(port_info) == 2:
port, end_port = port_info
else:
port = port_info[0]
if "detour" in conf_settings:
dynamic_port_tag = conf_settings["detour"]["to"]
for inbound in self.config["inbounds"]:
if "tag" in inbound and inbound["tag"] == dynamic_port_tag:
dyp.aid = inbound["settings"]["default"]["alterId"]
dyp.status = True
break
if protocol in ("vmess", "vless", "socks", "trojan"):
conf_stream = part_json["streamSettings"]
tls = conf_stream["security"]
if "sockopt" in conf_stream and "tcpFastOpen" in conf_stream["sockopt"]:
tfo = "open" if conf_stream["sockopt"]["tcpFastOpen"] else "close"
if "httpSettings" in conf_stream and conf_stream["httpSettings"]:
path = conf_stream["httpSettings"]["path"]
elif "wsSettings" in conf_stream and conf_stream["wsSettings"]:
host = conf_stream["wsSettings"]["headers"]["Host"]
path = conf_stream["wsSettings"]["path"]
elif "tcpSettings" in conf_stream and conf_stream["tcpSettings"]:
host = conf_stream["tcpSettings"]["header"]["request"]["headers"]["Host"]
header = "http"
if conf_stream["network"] == "kcp" and "header" in conf_stream["kcpSettings"]:
header = conf_stream["kcpSettings"]["header"]["type"]
if "seed" in conf_stream["kcpSettings"]:
path = conf_stream["kcpSettings"]["seed"]
if conf_stream["network"] == "quic" and conf_stream["quicSettings"]:
quic_settings = conf_stream["quicSettings"]
quic = Quic(quic_settings["security"], quic_settings["key"], quic_settings["header"]["type"])
if conf_stream["network"] == "grpc" and conf_stream["grpcSettings"]:
serviceName = conf_stream["grpcSettings"]["serviceName"]
if "multiMode" in conf_stream["grpcSettings"] and conf_stream["grpcSettings"]["multiMode"]:
mode = "multi"
group = Group(conf_ip, port, end_port=end_port, tls=tls, tfo=tfo, dyp=dyp, index=group_index)
if protocol == "shadowsocks":
self.user_number = self.user_number + 1
email = conf_settings["email"] if 'email' in conf_settings else ''
ss = SS(self.user_number, conf_settings["password"], conf_settings["method"], email)
group.node = ss
group.protocol = ss.__class__.__name__
return group
elif protocol in ("vmess", "vless", "trojan"):
clients = conf_settings["clients"]
elif protocol == "socks":
clients = conf_settings["accounts"]
elif protocol == "mtproto":
clients = conf_settings["users"]
for client in clients:
email, node, flow = "", None, ""
self.user_number = self.user_number + 1
if "email" in client and client["email"]:
email = client["email"]
if protocol == "vmess":
node = Vmess(client["id"], client["alterId"], conf_stream["network"], self.user_number, path=path,
host=host, header=header, email=email, quic=quic)
elif protocol == "socks":
node = Socks(self.user_number, client["pass"], user_info=client["user"])
elif protocol == "mtproto":
node = Mtproto(self.user_number, client["secret"], user_info=email)
elif protocol == "vless":
if tls == "xtls":
flow = client["flow"]
node = Vless(client["id"], self.user_number, conf_settings["decryption"], email, conf_stream["network"],
path, host, header, flow, serviceName, mode)
elif protocol == "trojan":
node = Trojan(self.user_number, client["password"], email)
if not group.protocol:
group.protocol = node.__class__.__name__
group.node = node
return group
================================================
FILE: 002-V2rayPool/core/utils.py
================================================
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import random
import re
import os
import socket
import string
import sys
import termios
import tty
import urllib.request
import signal
import subprocess
from enum import Enum, unique
class ProtocolType(object):
TROJAN = 'trojan://'
SS = 'ss://'
VMESS = 'vmess://'
@unique
class StreamType(Enum):
TCP = 'tcp'
TCP_HOST = 'tcp_host'
SOCKS = 'socks'
SS = 'ss'
MTPROTO = 'mtproto'
H2 = 'h2'
WS = 'ws'
QUIC = 'quic'
KCP = 'kcp'
KCP_UTP = 'utp'
KCP_SRTP = 'srtp'
KCP_DTLS = 'dtls'
KCP_WECHAT = 'wechat'
KCP_WG = 'wireguard'
VLESS_KCP = 'vless_kcp'
VLESS_UTP = 'vless_utp'
VLESS_SRTP = 'vless_srtp'
VLESS_DTLS = 'vless_dtls'
VLESS_WECHAT = 'vless_wechat'
VLESS_WG = 'vless_wireguard'
VLESS_TCP = 'vless_tcp'
VLESS_TLS = 'vless_tls'
VLESS_WS = 'vless_ws'
VLESS_GRPC = 'vless_grpc'
VLESS_XTLS = 'vless_xtls'
TROJAN = 'trojan'
def header_type_list():
return ("none", "srtp", "utp", "wechat-video", "dtls", "wireguard")
def ss_method():
return ("aes-256-gcm", "aes-128-gcm", "chacha20-poly1305")
def xtls_flow():
return ("", "xtls-rprx-origin", "xtls-rprx-direct")
def get_ip():
"""
获取本地ip
"""
my_ip = ""
try:
my_ip = urllib.request.urlopen('http://api.ipify.org').read()
except Exception:
my_ip = urllib.request.urlopen('http://icanhazip.com').read()
return bytes.decode(my_ip).strip()
def port_is_use(port):
"""
判断端口是否占用
"""
tcp_use, udp_use = False, False
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
u = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.settimeout(3)
tcp_use = s.connect_ex(('127.0.0.1', int(port))) == 0
try:
u.bind(('127.0.0.1', int(port)))
except:
udp_use = True
finally:
u.close()
return tcp_use or udp_use
def random_port(start_port, end_port):
while True:
random_port = random.randint(start_port, end_port)
if not port_is_use(random_port):
return random_port
def is_email(email):
"""
判断是否是邮箱格式
"""
str = r'^[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+){0,4}@[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+){0,4}$'
return re.match(str, email)
def is_ipv4(ip):
try:
socket.inet_pton(socket.AF_INET, ip)
except AttributeError: # no inet_pton here, sorry
try:
socket.inet_aton(ip)
except socket.error:
return False
return ip.count('.') == 3
except socket.error: # not a valid ip
return False
return True
def is_ipv6(ip):
try:
socket.inet_pton(socket.AF_INET6, ip)
except socket.error: # not a valid ip
return False
return True
def check_ip(ip):
return is_ipv4(ip) or is_ipv6(ip)
def bytes_2_human_readable(number_of_bytes, precision=1):
"""
流量bytes转换为kb, mb, gb等单位
"""
if number_of_bytes < 0:
raise ValueError("!!! number_of_bytes can't be smaller than 0 !!!")
step_to_greater_unit = 1024.
number_of_bytes = float(number_of_bytes)
unit = 'bytes'
if (number_of_bytes / step_to_greater_unit) >= 1:
number_of_bytes /= step_to_greater_unit
unit = 'KB'
if (number_of_bytes / step_to_greater_unit) >= 1:
number_of_bytes /= step_to_greater_unit
unit = 'MB'
if (number_of_bytes / step_to_greater_unit) >= 1:
number_of_bytes /= step_to_greater_unit
unit = 'GB'
if (number_of_bytes / step_to_greater_unit) >= 1:
number_of_bytes /= step_to_greater_unit
unit = 'TB'
number_of_bytes = round(number_of_bytes, precision)
return str(number_of_bytes) + ' ' + unit
def random_email():
domain = ['163', 'qq', 'sina', '126', 'gmail', 'outlook', 'icloud']
core_email = "@{}.com".format(random.choice(domain))
return ''.join(random.sample(string.ascii_letters + string.digits, 8)) + core_email
def readchar(prompt=""):
if prompt:
sys.stdout.write(prompt)
sys.stdout.flush()
fd = sys.stdin.fileno()
old_settings = termios.tcgetattr(fd)
try:
tty.setraw(sys.stdin.fileno())
ch = sys.stdin.read(1)
finally:
termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
print(ch)
return ch.strip()
def kill_all_v2ray():
pids = os.popen("ps aux |grep v2ray |awk '{print $2}'").read().split('\n')
for pid in pids:
try:
import subprocess
# subprocess.check_output("kill %d" % int(pid))
a = os.popen("kill %d" % int(pid)).read()
except Exception as e:
pass
sys_proxy_off()
# netstat -nlp | grep :1080 | awk '{print $7}' | awk -F\" / \" '{ print $1 }'
def kill_process_by_port(port):
try:
pids = os.popen("pgrep -f v2ray|xargs kill -9").read().split('\n')
print(pids)
except:
pass
def sys_proxy_on(proxy, port):
''''控制macOS系统代理'''
os.system('networksetup -setwebproxy wi-fi %s %d' % (proxy, port)) # http
os.system('networksetup -setsecurewebproxy wi-fi %s %d' % (proxy, port)) # https
os.system('networksetup -setsocksfirewallproxy wi-fi %s %d' % (proxy, port)) # socks
def sys_v2ray_on():
# proxy_on("127.0.0.1", 1080)
'''端口要对应起v2ray开启的，具体要看写入config.json文件中inbounds节点部分'''
os.system('networksetup -setwebproxy wi-fi 127.0.0.1 1087')
os.system('networksetup -setsecurewebproxy wi-fi 127.0.0.1 1087')
os.system('networksetup -setsocksfirewallproxy wi-fi 127.0.0.1 1080')
def sys_proxy_off():
os.system('networksetup -setwebproxystate wi-fi off')
os.system('networksetup -setsecurewebproxystate wi-fi off')
os.system('networksetup -setsocksfirewallproxystate wi-fi off')
================================================
FILE: 002-V2rayPool/db/db_main.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 数据相关接口封装
@Date :2021/08/30
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import random
from core import client
from db.local import DbLocal, DbEnable
from db.net import *
class DBManage(object):
def init(self):
self.dbLocal = DbLocal()
self.check = PYCheck()
self.dbEnable = DbEnable().get()
def __add_urls_de_dup(self, all_urls: [], new_urls: []) -> []:
"""合并数组，并且去重，去空"""
if not new_urls: # 为 [] 或者 None
return all_urls
for url in new_urls:
temp = url.strip().replace('\n', '')
if temp in all_urls or len(temp) < 20:
continue
all_urls.append(temp)
def start_random_v2ray_by_local(self, isSysOn=False):
"""从本地随机启动一个可用的proxy"""
urls = self.load_enable_urls_by_local()
for url in urls:
if client.Creator().v2ray_start_with_log(random.choice(urls), isSysOn) is False:
time.sleep(1)
continue
time.sleep(2)
ips = PYCheck().get_curren_ip()
if not ips:
print('无效地址：%s' % url)
continue
print('代理开启成功')
time.sleep(1)
return True
return False
def load_urls_and_save_auto(self):
"""首先通过不需要代理的网页获取节点，当代理有可用时，开启代理，获取需要代理获取的网页"""
self.dbLocal.clear_local()
all_urls = self.load_urls_by_not_proxy()
proxy_url = None
for url in all_urls:
if client.Creator().v2ray_start_with_log(url) is False:
time.sleep(1)
continue
time.sleep(2)
ips = PYCheck().get_curren_ip()
if not ips:
print('无效地址：%s' % url)
continue
proxy_url = url
break
if proxy_url is None:
raise Exception("无代理可用，退出！")
print("获得可用代理地址：%s" % proxy_url)
proxy_urls = self.load_urls_by_net_with_proxy(proxy_url=proxy_url)
all_urls = all_urls + proxy_urls
self.check_and_save(all_urls, append=False)
def load_urls_by_not_proxy(self, save_local=True):
all_urls = []
# 1. 先把不需要代理的先请求下来
self.__add_urls_de_dup(all_urls, PNTWGithubV2ray().get_urls())
print("获取https://hub.xn--gzu630h.xn--kpry57d/freefq/free后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PNSsfree().get_urls())
print("获取https://view.ssfree.ru/后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PNFreevpnX().get_urls())
print("获取https://freevpn-x.com/后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PNGithubIwxf().get_urls())
print("获取https://github.com/iwxf/free-v2ray/blob/master/README.md 后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PNFreeV2ray().get_urls())
print("获取https://view.freev2ray.org/ 后数目：%d" % len(all_urls))
if save_local: # 保存到本地
self.dbLocal.save_urls(all_urls)
return all_urls
def load_urls_by_net_with_proxy(self, proxy_url=None, save_local=True):
all_urls = []
if save_local: # 保存到本地
self.dbLocal.save_urls(all_urls)
if not proxy_url:
proxy_url = 'ss://YWVzLTI1Ni1nY206NGVqSjhuNWRkTHVZRFVIR1hKcmUydWZK@212.102.40.68:48938#github.com/freefq%20-%20%E6%84%8F%E5%A4%A7%E5%88%A9%20%201'
creator = client.Creator()
creator.v2ray_start(proxy_url)
time.sleep(2)
self.__add_urls_de_dup(all_urls, PYIvmess().get_urls())
print("获取https://t.me/s/ivmess 后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PYFlyingboat().get_urls())
print("获取https://t.me/s/flyingboat 后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PYFreevpnnet().get_urls())
print("获取https://www.freevpnnet.com/ 后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PYMerlinblog().get_urls())
print("获取https://merlinblog.xyz/wiki/freess.html 后数目：%d" % len(all_urls))
# __add_urls_de_dup(all_urls, PYFreeFq().get_urls())
self.__add_urls_de_dup(all_urls, PYFreeFq().download_urls(f_day=1)) # 前2天的地址
print("获取从https://freefq.com/ 后数目：%d" % len(all_urls))
if save_local: # 保存到本地
self.dbLocal.save_urls(all_urls)
return all_urls
def load_urls_by_net(self, proxy_url=None, save_local=True, need_proxy=True):
"""
通过网络获取最新的节点，但是需要代理
:param proxy_url: 代理url，默认的如果失效了则回去失败
:param save_local: 是否保存到本地
:param need_proxy: 如果程序本身就在外网跑，就不需要开启代理获取了
:return:
"""
all_urls = []
# 1. 先把不需要代理的先请求下来
self.__add_urls_de_dup(all_urls, PNTWGithubV2ray().get_urls())
print("获取https://hub.xn--gzu630h.xn--kpry57d/freefq/free后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PNSsfree().get_urls())
print("获取https://view.ssfree.ru/后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PNFreevpnX().get_urls())
print("获取https://freevpn-x.com/后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PNGithubIwxf().get_urls())
print("获取https://github.com/iwxf/free-v2ray/blob/master/README.md 后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PNFreeV2ray().get_urls())
print("获取https://view.freev2ray.org/ 后数目：%d" % len(all_urls))
print("准备开启代理获取...")
if need_proxy:
if not proxy_url:
proxy_url = 'ss://YWVzLTI1Ni1nY206NGVqSjhuNWRkTHVZRFVIR1hKcmUydWZK@212.102.40.68:48938#github.com/freefq%20-%20%E6%84%8F%E5%A4%A7%E5%88%A9%20%201'
# 需要代理
creator = client.Creator()
creator.v2ray_start(proxy_url)
time.sleep(2)
self.__add_urls_de_dup(all_urls, PYIvmess().get_urls())
print("获取https://t.me/s/ivmess 后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PYFlyingboat().get_urls())
print("获取https://t.me/s/flyingboat 后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PYFreevpnnet().get_urls())
print("获取https://www.freevpnnet.com/ 后数目：%d" % len(all_urls))
self.__add_urls_de_dup(all_urls, PYMerlinblog().get_urls())
print("获取https://merlinblog.xyz/wiki/freess.html 后数目：%d" % len(all_urls))
# __add_urls_de_dup(all_urls, PYFreeFq().get_urls())
self.__add_urls_de_dup(all_urls, PYFreeFq().download_urls(f_day=1)) # 前2天的地址
print("获取从https://freefq.com/ 后数目：%d" % len(all_urls))
if save_local: # 保存到本地
self.dbLocal.save_urls(all_urls, append=False)
return all_urls
def load_unchecked_urls_by_local(self):
"""获取本地未校验过的url"""
self.dbLocal.get_urls(False)
urls = self.dbLocal.get_checked_urls()
return urls
def load_enable_urls_by_local(self):
"""获取已检测过的url"""
return self.dbEnable.get_urls()
def check_url_single(self, url: str):
client.Creator().v2ray_start(url)
time.sleep(2)
ips = PYCheck().get_curren_ip()
if not ips:
print('地址无效！')
return False
print('检查地址结果：%s' % url)
print(ips)
return True
def check_and_save(self, urls: [], append=True):
"""检测url是否可用，并且保存到本地"""
if not append:
self.dbEnable.clear_local()
all_infos = self.dbEnable.get_infos()
new_infos = []
size = len(urls)
for i in range(size):
try:
if i % 30 == 0: # 每三十个更新一次
self.dbLocal.save_urls(urls=urls, append=False) # 更新剩下的
url = urls.pop()
in_all = False
for item in all_infos:
if url in item:
in_all = True
break
if in_all:
print('取出地址已存在：%s' % url)
continue
if client.Creator().v2ray_start_with_log(url) is False:
time.sleep(1)
continue
time.sleep(2)
ips = PYCheck().get_curren_ip()
if not ips:
print('地址无效！')
continue
ip, add = str(ips[0]), ips[1]
hase_item = False # 已存在
for item in all_infos:
if ip in item:
hase_item = True
break
if hase_item:
print('ip=%s已存在！' % ip)
continue
info = r'%s,%s,%s' % (url.strip(), ip.strip(), add.strip().replace('\n', ''))
print('%s！总共：%d |待检测：%d |可用：%d' % ('地址有效', size, len(urls), len(all_infos)))
new_infos.append(info)
all_infos.append(info)
self.dbEnable.save_urls(new_infos) # 写入已通过的
new_infos.clear()
except Exception as e:
print(e)
# 最后
print('%s！总共：%d |待检测：%d |可用：%d' % ('全部检测完毕！', size, len(urls), len(all_infos)))
self.dbEnable.save_urls(new_infos)
self.dbLocal.clear_local()
================================================
FILE: 002-V2rayPool/db/local.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 数据库相关类
@Date :2021/08/25
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import os
import shutil
from concurrent.futures import ThreadPoolExecutor
from threading import Lock
from core.conf import Config
class DbLocal(object):
"""
加载本地文件的url，并进行检测是否合法
"""
def __init__(self):
path = Config.get_v2ray_node_path()
parent_dir = os.path.dirname(os.path.abspath(__file__))
if path:
parent_dir = path
if not os.path.exists(parent_dir):
os.mkdir(parent_dir)
self.__save_path = os.path.join(parent_dir, '_db-uncheck.txt')
self.__get_path = os.path.join(parent_dir, '_db-uncheck.txt')
if not os.path.isfile(self.__save_path):
open(self.__save_path, mode='w', encoding="utf-8").write('')
if not os.path.isfile(self.__get_path):
open(self.__get_path, mode='w', encoding="utf-8").write('')
self.__reset()
def __reset(self):
"""恢复默认状态"""
self.__checked_urls = []
self.__load_urls = []
self.__checked_finish = False
self.__checked_count = 0
def __check_url(self, url: str):
"""在异步中检测url是否合法等"""
self.__checked_count += 1
# print('check_url: %s' % url)
# 如果检测通过
temp = url.strip()
if len(temp) > 10 and (temp.startswith('ss://') or temp.startswith('vmess://')):
self.__checked_urls.append(temp)
if len(self.__load_urls) == self.__checked_count:
self.__checked_finish = True
print('已经完成检测')
def is_checked_finished(self) -> bool:
"""是否已经完成检测"""
return self.__checked_finish
def get_checked_urls(self) -> []:
"""已通过检测的url"""
return self.__checked_urls
def get_urls(self, is_check=True) -> bool:
"""
开始加载本地的链接，异步处理结果。
:param get_path: 本地路径
:return: True：加载成功
"""
self.__reset()
get_path = self.__get_path
if not os.path.isfile(get_path):
return False
try:
with open(get_path, mode='r') as f:
for url in f:
if url and url not in self.__load_urls:
self.__load_urls.append(url)
except Exception as e:
print(e)
finally:
f.close()
size = len(self.__load_urls)
if size == 0:
print('本地无数据')
return False
if is_check:
# 创建线程池，传入max_workers参数来设置线程池中最多能同时运行的线程数目
executor = ThreadPoolExecutor(max_workers=3)
for url in self.__load_urls:
executor.submit(self.__check_url, url).done()
else:
self.__checked_urls = self.__load_urls
self.__checked_finish = True
print("已加载数目：%d" % size)
return True
def clear_local(self):
"""通过写入空字符实现清除内容"""
try:
with open(self.__save_path, mode='w', encoding="utf-8") as f:
f.write('')
f.close()
except Exception as e:
print(e)
def save_urls(self, urls, append=True):
"""
保存节点到本地
:param urls:
:param append: 添加到末尾
:return:
"""
if not urls:
return
all_url = []
if not append: # 清空之后再继续
self.clear_local()
all_url = urls
else: # 把本地的取出来，然后再进行去重
for url in urls:
if url not in all_url:
all_url.append(url)
try:
with open(self.__save_path, mode='r') as f:
for url in f:
if url and url not in all_url:
all_url.append(url)
except Exception as e:
print(e)
finally:
f.close()
size = len(all_url)
print('目前本地总共%d条' % size)
try:
self.clear_local()
with open(self.__save_path, mode='a', encoding="utf-8") as f:
for url in all_url:
url = url.strip().replace('\n', '')
if len(url) > 20:
f.write(url + '\n')
f.close()
except Exception as e:
print(e)
class DbEnable(object):
"""
单例模式，提供可用的v2ray对象
"""
_instance_lock = Lock()
def __init__(self):
self.__urls = []
self.__index = 0
self.__END = '.back'
self.__mutex = Lock()
self.__default_url = 'ss://YWVzLTI1Ni1nY206WWd1c0gyTVdBOFBXYzNwMlZEc1I3QVZ2@81.19.223.189:31764#github.com/freefq%20-%20%E8%8B%B1%E5%9B%BD%20%208'
self.__config_path = ''
path = Config.get_v2ray_node_path()
# 获取本地文件实例？
parent_dir = os.path.dirname(os.path.abspath(__file__))
if path:
parent_dir = path
if not os.path.exists(parent_dir):
os.mkdir(parent_dir)
self.__path = os.path.join(parent_dir, '_db-checked.txt')
if not os.path.isfile(self.__path):
open(self.__path, mode='w', encoding="utf-8").write('')
@classmethod
def get(cls, *args, **kwargs):
with DbEnable._instance_lock:
if not hasattr(DbEnable, "_instance"):
DbEnable._instance = DbEnable(*args, **kwargs)
return DbEnable._instance
def init(self, config_path, def_url=None):
"""
初始化所需要参数
:param config_path: 配置文件路径
:param def_url: 默认使用的 v2ray链接
:return:
"""
if def_url is not None:
self.__default_url = def_url
if len(config_path) < len('(参考用)config.json'):
return False
if not os.path.isfile(config_path):
return False
self.__config_path = config_path
return self.__back_config_json()
def __back_config_json(self):
"""备份配置文件"""
if not os.path.isfile(self.__config_path):
return False
back_path = self.__config_path + self.__END
try:
shutil.copy(self.__config_path, back_path)
except Exception as e:
print(e)
return False
return True
def __restore_config_json(self):
"""恢复配置文件"""
back_path = self.__config_path + self.__END
if not os.path.isfile(back_path):
return False
try:
if os.path.isfile(self.__config_path):
os.remove(self.__config_path)
shutil.copy(back_path, self.__config_path)
except Exception as e:
print(e)
return False
return True
def add_url(self, url: str):
self.__add(url)
def add_urls(self, url: []):
self.__adds(url)
def select_by_order(self):
"""选择下一个代理，并且更新配置文件，让他起作用"""
url = self.get_by_order()
return True
def get_by_order(self):
"""从队列中顺序获取一个v2ray协议实例"""
def_url = self.__default_url
if self.__check_and_enable():
def_url = self.__get(self.__index)
return def_url
def __check_and_enable(self):
"""检查是否已到结束，如果是，下一个从0开始"""
if len(self.__urls) == 0:
return False
if len(self.__urls) <= self.__index + 1:
self.__index = -1
self.__index += 1
return True
def __add(self, url):
self.__mutex.acquire()
self.__urls.append(url)
self.__mutex.release()
def __adds(self, urls: []):
self.__mutex.acquire()
self.__urls.extend(urls)
self.__mutex.release()
def __get(self, index):
self.__mutex.acquire()
url = self.__urls[index]
self.__mutex.release()
return url
def clear_local(self):
"""通过写入空字符实现清除内容"""
try:
with open(self.__path, mode='w', encoding="utf-8") as f:
f.write('')
f.close()
except Exception as e:
print(e)
def get_urls(self) -> []:
"""获取截取后的url"""
urls = []
try:
with open(self.__path, mode='r') as f:
for url in f:
url = url.strip().replace('\n', '')
if len(url) > 20:
urls.append(url.split(',')[0])
except Exception as e:
print(e)
return urls
def get_infos(self) -> []:
"""获取所有信息，包括ip和地址"""
infos = []
try:
with open(self.__path, mode='r') as f:
for info in f:
info = info.strip().replace('\n', '')
if len(info) > 20:
infos.append(info)
except Exception as e:
print(e)
return infos
def de_duplication(self):
"""去重"""
infos = self.get_infos()
new_infos = []
for info in infos:
if len(info.strip()) <= 0: # 去空行
continue
_in = False
for n in new_infos:
if info.split(',')[1] in n:
_in = True
break
if not _in:
new_infos.append(info)
print('去重前数目：%d，去重后的数目：%d' % (len(infos), len(new_infos)))
try:
self.clear_local()
with open(self.__path, mode='a', encoding="utf-8") as f:
for url in new_infos:
url = url.strip().replace('\n', '')
if len(url) > 20:
f.write(url + '\n')
f.close()
except Exception as e:
print(e)
def save_urls(self, urls, append=True):
"""
保存节点到本地
:param urls:
:param append: 添加到末尾
:return:
"""
if not urls:
return
all_url = []
if not append: # 清空之后再继续
self.clear_local()
all_url = urls
else: # 把本地的取出来，然后再进行去重
for url in urls:
if url not in all_url:
all_url.append(url)
try:
with open(self.__path, mode='r') as f:
for url in f:
if url and url not in all_url:
all_url.append(url)
except Exception as e:
print(e)
finally:
f.close()
size = len(all_url)
print('目前本地总共已检测可用%d条' % size)
try:
self.clear_local()
with open(self.__path, mode='a', encoding="utf-8") as f:
for url in all_url:
url = url.strip().replace('\n', '')
if len(url) > 20:
f.write(url + '\n')
f.close()
except Exception as e:
print(e)
================================================
FILE: 002-V2rayPool/db/net.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 从网络获取
@Date :2021/08/30
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import json
from base.net_proxy import Net
import re
import time
import chardet
def re_vmess_ss_trojan(pattern, html) -> []:
"""传入规则：r'xxx%sxxx', %s为固定的：(vmess://.+?|ss://.+?|trojan://.+?)"""
results = re.findall(pattern % '(vmess://.+?|ss://.+?|trojan://.+?)', html, re.DOTALL)
urls = []
for i in results:
temp: str = i.strip()
if len(temp) < 20 and temp in results: # 凭感觉
continue
# 如果是换行的
if '\n' in temp:
items = temp.split('\n')
for j in items:
temp_j: str = j.strip()
if len(temp_j) < 20 and temp_j in results: # 凭感觉
continue
value_j = temp_j.replace('\n', '')
if len(value_j) > 10:
urls.append(value_j)
continue
value = temp.replace('\n', '')
if len(value) > 10:
urls.append(value)
return urls
class PYCheck(Net):
def get_curren_ip(self, url='https://ip.cn/api/index?ip=&type=0'):
"""获取内容"""
try:
r = self.request_zh(url)
if r.status_code == 200:
charset = chardet.detect(r.content)
content = r.content.decode(charset['encoding'])
r.encoding = r.apparent_encoding
# {"rs":1,"code":0,"address":"德国 Hessen ","ip":"51.38.122.98","isDomain":0}
results = json.loads(content)
if not results:
return None
return [results['ip'], results['address']]
elif r.status_code == 301 or r.status_code == 302 or r.status_code == 303:
location = r.headers['Location']
time.sleep(1)
return self.get_curren_ip(location)
except Exception as e:
print(e)
return None
class PNFreeV2ray(Net):
"""
https://view.freev2ray.org/
"""
def get_urls(self) -> []:
try:
r = self.request(r'https://view.freev2ray.org/')
if r.status_code != 200:
return None
r.encoding = r.apparent_encoding
return re_vmess_ss_trojan(r'"%s"', r.text)
except Exception as e:
print(e)
return None
class PNTWGithubV2ray(Net):
"""
# https://hub.xn--gzu630h.xn--kpry57d/freefq/free
"""
def get_urls(self) -> []:
try:
r = self.request(r'https://hub.xn--gzu630h.xn--kpry57d/freefq/free')
if r.status_code != 200:
return None
r.encoding = r.apparent_encoding
return re_vmess_ss_trojan(r'"%s"', r.text)
except Exception as e:
print(e)
return None
class PNSsfree(Net):
"""
https://view.ssfree.ru/
"""
def get_urls(self) -> []:
try:
r = self.request(r'https://view.ssfree.ru/')
if r.status_code != 200:
return None
r.encoding = r.apparent_encoding
html = r.text
return re_vmess_ss_trojan(r'"%s"', html)
except Exception as e:
print(e)
return None
class PNFreevpnX(Net):
"""
https://freevpn-x.com/
"""
def get_urls(self) -> []:
"""
获取当前页面中文本的url
:param date: 如：2021/08/29
:return:
"""
url2 = r'https://url.cr/api/user.ashx?do=freevpn&ip=127.0.0.1&uuid=C5E0C9BA-FECB-44ED-9BD8-90C55365E11B&_=%d' % (
time.time() * 1000)
try:
r = self.request(url2)
if r.status_code != 200:
return None
r.encoding = r.apparent_encoding
html = r.text
results = html.split('\n')
urls = []
for i in results:
temp = i.strip()
if len(temp) > 10:
urls.append(temp)
return urls
except Exception as e:
print(e)
return None
class PNGithubIwxf(Net):
"""
https://github.com/iwxf/free-v2ray/blob/master/README.md
"""
def get_urls(self) -> []:
"""
获取当前页面中文本的url
:param date: 如：2021/08/29
:return:
"""
try:
r = self.request(r'https://github.com/iwxf/free-v2ray/blob/master/README.md')
if r.status_code != 200:
return None
r.encoding = r.apparent_encoding
html = r.text
return re_vmess_ss_trojan(r'<pre><code>%s</code></pre>', html)
except Exception as e:
print(e)
return None
class PYFreevpnnet(Net):
"""
https://www.freevpnnet.com/
需要代理
"""
def get_urls(self) -> []:
try:
r = self.request_en(r'https://www.freevpnnet.com/')
if r.status_code != 200:
return None
r.encoding = r.apparent_encoding
html = r.text
return re_vmess_ss_trojan(r'>%s<', html)
except Exception as e:
print(e)
return None
class PYMerlinblog(Net):
"""
https://merlinblog.xyz/wiki/freess.html
需要代理
"""
def get_urls(self) -> []:
try:
r = self.request_en(r'https://merlinblog.xyz/wiki/freess.html')
if r.status_code != 200:
return None
r.encoding = r.apparent_encoding
html = r.text
return re_vmess_ss_trojan('%s<', html)
except Exception as e:
print(e)
return None
class PYFlyingboat(Net):
"""
https://t.me/s/flyingboat
需要代理
"""
def get_urls(self) -> []:
try:
r = self.request_en(r'https://t.me/s/flyingboat')
if r.status_code != 200:
return None
r.encoding = r.apparent_encoding
html = r.text
return re_vmess_ss_trojan('>%s<', html)
except Exception as e:
print(e)
return None
class PYIvmess(Net):
"""
https://t.me/s/ivmess
需要代理
"""
def get_urls(self) -> []:
try:
r = self.request_en(r'https://t.me/s/ivmess')
if r.status_code != 200:
return None
r.encoding = r.apparent_encoding
html = r.text
return re_vmess_ss_trojan('>%s<', html)
except Exception as e:
print(e)
return None
class PYFreeFq(Net):
"""
从https://freefq.com/获取免费节点，规则：https://freefq.com/v2ray/2021/08/30/v2ray.html
"""
def __get_content_url(self, date) -> str:
"""
获取当前页面中文本的url
:param date: 如：2021/08/29
:return:
"""
try:
r = self.request_en(r'https://freefq.com/v2ray/%s/v2ray.html' % date)
if r.status_code != 200:
return None
r.encoding = r.apparent_encoding
return self.__get_url_by_content_html(r.text)
except Exception as e:
print(e)
return None
def __get_url_by_content_html(self, html: str):
"""从文本中通过正则获取节点所在文本的url"""
results = re.findall(r'<td>.+?</td>', html)
tag = None
pre = 'https://www.freefq.com/d/file/v2ray'
for url in results:
if pre in url:
tag = url
break
if not tag:
print('无法获取节点url：%s' % results)
return None
try:
url = re.findall(r'%s.+?\.htm' % pre, tag)[0]
except Exception as e:
print(e)
return None
return url
def __get_url_by_detail_html(self, html: str):
"""截取文本中的节点url"""
# results = re.findall(r'(trojan://.+?|vmess://.+?|ss://.+?)<br>', html)
results = re_vmess_ss_trojan(r'%s<br>', html)
urls = []
for url in results:
temp: str = url.strip()
if len(temp) < 20 and temp in results: # 凭感觉
continue
urls.append(url)
return urls
def __get_detail_urls(self, url: str) -> []:
"""获取内容"""
try:
r = self.request_en(url)
if r.status_code != 200:
return None
r.encoding = r.apparent_encoding
# return self.__get_url_by_detail_html(r.text)
return re_vmess_ss_trojan(r'%s<br>', r.text)
except Exception as e:
print(e)
return None
def download_urls(self, f_day=1) -> []:
"""
开始获取
:param f_day: 需要获取往前的天数
:return:
"""
DAY = 24 * 60 * 60
l_time = time.time()
all_url = []
for day in range(1, f_day + 1): # 需要从前一天开始
temp_day = time.strftime("%Y/%m/%d", time.localtime(l_time - (day * DAY)))
url = self.__get_content_url(temp_day)
if not url:
continue
urls = self.__get_detail_urls(url)
if not urls: # 如果是None或者[]
continue
for temp in urls:
if temp not in all_url:
all_url.append(temp)
time.sleep(1)
return all_url
def get_urls(self) -> []:
return self.download_urls()
================================================
FILE: 002-V2rayPool/doc/(参考用)config.json
================================================
{
"log": {
"access": "",
"error": "",
"loglevel": "info"
},
"inbounds": [
{
"port": "1080",
"listen": "0.0.0.0",
"protocol": "socks",
"settings": {
"auth": "noauth",
"udp": true,
"ip": "127.0.0.1",
"clients": null
},
"streamSettings": null
},
{
"listen": "127.0.0.1",
"protocol": "http",
"settings": {
"timeout": 360
},
"port": "1087"
}
],
"outbounds": [
{
"protocol": "shadowsocks",
"settings": {
"servers": [
{
"address": "167.88.61.60",
"method": "aes-256-gcm",
"ota": false,
"password": "RexnBgU7EV5ADxG",
"port": 7002
}
]
}
},
{
"protocol": "freedom",
"settings": {
"response": null
},
"tag": "direct"
}
],
"routing": {
"domainStrategy": "IPIfNonMatch",
"rules": [
{
"type": "field",
"ip": [
"geoip:private"
],
"outboundTag": "direct"
},
{
"type": "field",
"domain": [
"geosite:cn"
],
"outboundTag": "direct"
},
{
"type": "field",
"domain": [
"geoip:cn"
],
"outboundTag": "direct"
}
]
}
}
================================================
FILE: 002-V2rayPool/test_main.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 主要入口
@Date :2021/08/25
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
from core import utils
from core.conf import Config
from db.db_main import *
EXIT_NUM = 100
if __name__ == '__main__':
utils.kill_all_v2ray()
Config.set_v2ray_core_path('/Users/Qincji/Desktop/develop/soft/intalled/v2ray-macos-64') # v2ray内核存放路径
Config.set_v2ray_node_path('/Users/Qincji/Desktop/develop/py/project/PythonIsTools/002-V2rayPool') # 保存获取到节点的路径
proxy_url = 'vmess://ew0KICAidiI6ICIyIiwNCiAgInBzIjogIkBTU1JTVUItVjUyLeS7mOi0ueaOqOiNkDpzdW8ueXQvc3Nyc3ViIiwNCiAgImFkZCI6ICIxMTIuMzMuMzIuMTM2IiwNCiAgInBvcnQiOiAiMTAwMDMiLA0KICAiaWQiOiAiNjVjYWM1NmQtNDE1NS00M2M4LWJhZTAtZjM2OGNiMjFmNzcxIiwNCiAgImFpZCI6ICIxIiwNCiAgInNjeSI6ICJhdXRvIiwNCiAgIm5ldCI6ICJ0Y3AiLA0KICAidHlwZSI6ICJub25lIiwNCiAgImhvc3QiOiAiMTEyLjMzLjMyLjEzNiIsDQogICJwYXRoIjogIiIsDQogICJ0bHMiOiAiIiwNCiAgInNuaSI6ICIiDQp9'
dbm = DBManage()
dbm.init() # 必须初始化
if dbm.check_url_single(proxy_url):
urls = dbm.load_urls_by_net(proxy_url=proxy_url)
dbm.check_and_save(urls, append=False)
# print(urls)
# urls = dbm.load_unchecked_urls_by_local()
# dbm.check_and_save(urls, append=False)
# urls = dbm.load_enable_urls_by_local()
# dbm.load_urls_and_save_auto()
utils.kill_all_v2ray()
================================================
FILE: 003-Keywords/.gitignore
================================================
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
================================================
FILE: 003-Keywords/README.md
================================================
# 通过google trends查找相关关键词，并且生成趋势
## 效果
例子：通过`women ring`关键词查找出有1千个相关关键词：[women ring.csv](women-ring/women ring.csv)
以及其生成关键词趋势，如：[swarovski rings.jpg](women-ring/swarovski rings.jpg)
## 实现
1. 使用[pytrends](https://github.com/GeneralMills/pytrends) 开源库。
2. 使用[002-V2rayPool](../002-V2rayPool) 代理（可选择第三方代理）。
3. 实现入口请参照：[main.py](main.py)
> 注：项目架构是使用[Scrapy](https://www.osgeo.cn/scrapy/intro/overview.html) 实现的，可实现amazon关键词查询等。
================================================
FILE: 003-Keywords/__init__.py
================================================
================================================
FILE: 003-Keywords/amazon/items.py
================================================
# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
class AmazonItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass
================================================
FILE: 003-Keywords/amazon/middlewares.py
================================================
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
import requests
from scrapy import signals
# useful for handling different item types with a single interface
from scrapy.http import TextResponse
class AmazonSpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, or item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Request or item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn’t have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class AmazonDownloaderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
return None
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info('Spider opened: %s' % spider.name)
class AmazonProxyMiddleware(object):
def process_request(self, request, spider):
print('执行AmazonProxyMiddleware……')
# Set the location of the proxy
proxy_url = "socks5://127.0.0.1:1080"
request.meta['proxy'] = proxy_url
# 考虑socks代理，使用requests库进行请求
if proxy_url.startswith('socks'):
url = request.url
method = request.method
headers = {key: request.headers[key] for key in request.headers}
body = request.body
cookies = request.cookies
timeout = request.meta.get('download_timeout', 10)
proxies = {'http': proxy_url,
'https': proxy_url}
resp = requests.request(method, url,
data=body,
headers=headers,
cookies=cookies,
verify=False, timeout=timeout, proxies=proxies)
resp.headers['content-encoding'] = None
response = TextResponse(url=url, headers=resp.headers, body=resp.content,
request=request, encoding=resp.encoding)
return response
return None
================================================
FILE: 003-Keywords/amazon/pipelines.py
================================================
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
class AmazonPipeline:
def process_item(self, item, spider):
return item
================================================
FILE: 003-Keywords/amazon/settings.py
================================================
# Scrapy settings for amazon project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://docs.scrapy.org/en/latest/topics/settings.html
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'amazon'
SPIDER_MODULES = ['amazon.spiders']
NEWSPIDER_MODULE = 'amazon.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'amazon (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 2
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
#COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
#}
# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'amazon.middlewares.AmazonSpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
# # 'amazon.middlewares.AmazonDownloaderMiddleware': 543,
# 'amazon.middlewares.AmazonProxyMiddleware': 543
# }
# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
#ITEM_PIPELINES = {
# 'amazon.pipelines.AmazonPipeline': 300,
#}
# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
================================================
FILE: 003-Keywords/amazon/spiders/__init__.py
================================================
# This package will contain the spiders of your Scrapy project
#
# Please refer to the documentation for information on how to create and manage
# your spiders.
================================================
FILE: 003-Keywords/amazon/spiders/alibaba.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 亚马逊相关关键词获取
@Date :2021/09/24
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import re
from urllib.parse import quote_plus
import scrapy
from my_fake_useragent import UserAgent
class AlibabaSpider(scrapy.Spider):
name = 'alibaba'
allowed_domains = ['alibaba.com']
results = []
keywords = []
headers = {
'Host': 'www.alibaba.com',
'User-Agent': UserAgent().random()
}
def start_requests(self):
print('alibaba start_requests')
"""
start_requests做为程序的入口，可以重写，自定义第一批请求
"""
start_urls = ['https://www.alibaba.com/trade/search?fsb=y&IndexArea=product_en&CatId=&SearchText={k}'.format(
k=quote_plus(k)) for k in self.keywords]
# , meta={'proxy': 'socks5h://127.0.0.1:1080/'}
for url in start_urls:
yield scrapy.Request(url, headers=self.headers,
callback=self.parse)
def parse(self, response):
print('alibaba parse')
with open('alibaba.html', mode='w') as f:
f.write(response.text)
================================================
FILE: 003-Keywords/amazon/spiders/amazon.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 亚马逊相关关键词获取
@Date :2021/09/24
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import re
from urllib.parse import quote_plus
import scrapy
from my_fake_useragent import UserAgent
class AmazonSpider(scrapy.Spider):
name = 'amazon'
allowed_domains = ['amazon.com']
results = []
keywords = []
headers = {
'Host': 'www.amazon.com',
'User-Agent': UserAgent().random()
}
def start_requests(self):
print('amazon start_requests')
"""
start_requests做为程序的入口，可以重写，自定义第一批请求
"""
start_urls = ['https://www.amazon.com/s?k={k}'.format(k=quote_plus(k)) for k in self.keywords]
# , meta={'proxy': 'socks5h://127.0.0.1:1080/'}
for url in start_urls:
yield scrapy.Request(url, headers=self.headers,
callback=self.parse)
def parse(self, response):
print('amazon parse')
temps = re.findall(r'<span class="a-size-base a-color-base s-line-clamp-2">(.+?)</span>', response.text,
re.DOTALL)
deal = [x.replace('\n', '').strip() for x in temps]
print(deal)
self.results += deal
================================================
FILE: 003-Keywords/amazon/spiders/checkip.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 检查当前代理是否起作用
@Date :2021/09/24
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import re
import scrapy
from my_fake_useragent import UserAgent
class CheckIpSpider(scrapy.Spider):
name = 'ip138'
allowed_domains = ['ip138.com']
ips = None
headers = {
'User-Agent': UserAgent().random()
}
def start_requests(self):
print('CheckIpSpider start_requests')
"""
start_requests做为程序的入口，可以重写，自定义第一批请求
"""
start_urls = ['https://2021.ip138.com']
# , meta={'proxy': 'socks5h://127.0.0.1:1080/'}
for url in start_urls:
yield scrapy.Request(url, headers=self.headers,
callback=self.parse)
def parse(self, response):
print('CheckIpSpider parse')
results = re.findall(r'\[<a.+?>(.+?)</a>.+?：(.+?)\n</p>', response.text, re.DOTALL)
print(results)
self.ips = results
================================================
FILE: 003-Keywords/amazon/spiders/spiders.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 爬虫页面集合
@Date :2021/09/26
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import re
from urllib.parse import quote_plus
import scrapy
from my_fake_useragent import UserAgent
class CheckIpSpider(scrapy.Spider):
"""
检查当前代理ip信息
"""
name = 'ip138'
allowed_domains = ['ip138.com']
ips = None
headers = {
'User-Agent': UserAgent().random()
}
def start_requests(self):
print('CheckIpSpider start_requests')
"""
start_requests做为程序的入口，可以重写，自定义第一批请求
"""
start_urls = ['https://2021.ip138.com']
# , meta={'proxy': 'socks5h://127.0.0.1:1080/'}
for url in start_urls:
yield scrapy.Request(url, headers=self.headers,
callback=self.parse)
def parse(self, response):
print('CheckIpSpider parse')
results = re.findall(r'\[<a.+?>(.+?)</a>.+?：(.+?)\n</p>', response.text, re.DOTALL)
print(results)
self.ips = results
class AmazonSpider(scrapy.Spider):
"""
https://www.amazon.com/s?k=？？
亚马逊页面获取搜索词相关：
"""
name = 'amazon'
allowed_domains = ['amazon.com']
results = []
keywords = []
headers = {
'Host': 'www.amazon.com',
'User-Agent': UserAgent().random()
}
def start_requests(self):
print('amazon start_requests')
"""
start_requests做为程序的入口，可以重写，自定义第一批请求
"""
start_urls = ['https://www.amazon.com/s?k={k}'.format(k=quote_plus(k)) for k in self.keywords]
# , meta={'proxy': 'socks5h://127.0.0.1:1080/'}
for url in start_urls:
yield scrapy.Request(url, headers=self.headers,
callback=self.parse)
def parse(self, response):
print('amazon parse')
temps = re.findall(r'<span class="a-size-base a-color-base s-line-clamp-2">(.+?)</span>', response.text,
re.DOTALL)
deal = [x.replace('\n', '').strip() for x in temps]
print(deal)
self.results += deal
class EtsySpider(scrapy.Spider):
"""需要连接外网
https://www.etsy.com/market/
"""
name = 'etsy'
allowed_domains = ['etsy.com']
results = []
keywords = []
headers = {
'User-Agent': UserAgent().random()
}
def start_requests(self):
print('etsy start_requests')
"""
start_requests做为程序的入口，可以重写，自定义第一批请求
"""
start_urls = ['https://www.etsy.com/market/{k}'.format(k=quote_plus(k)) for k in self.keywords]
# , meta={'proxy': 'socks5h://127.0.0.1:1080/'}
for url in start_urls:
yield scrapy.Request(url, headers=self.headers,
callback=self.parse)
def parse(self, response):
print('etsy parse')
with open('etsy.html', mode='w') as f:
f.write(response.text)
f.close()
temps = re.findall(r'<span class="a-size-base a-color-base s-line-clamp-2">(.+?)</span>', response.text,
re.DOTALL)
deal = [x.replace('\n', '').strip() for x in temps]
print(deal)
self.results += deal
class LakesideSpider(scrapy.Spider):
"""一个商城网站
https://www.lakeside.com/browse/Clothing-Accessories
"""
name = 'lakeside'
allowed_domains = ['lakeside.com']
results = []
keywords = []
headers = {
'User-Agent': UserAgent().random()
}
def start_requests(self):
print('lakeside start_requests')
"""
start_requests做为程序的入口，可以重写，自定义第一批请求
"""
start_urls = ['https://www.lakeside.com/browse/{k}'.format(k=quote_plus(k)) for k in self.keywords]
# , meta={'proxy': 'socks5h://127.0.0.1:1080/'}
for url in start_urls:
yield scrapy.Request(url, headers=self.headers,
callback=self.parse)
def parse(self, response):
print('lakeside parse')
with open('lakeside.html', mode='w') as f:
f.write(response.text)
f.close()
# temps = re.findall(r'<span class="a-size-base a-color-base s-line-clamp-2">(.+?)</span>', response.text,
# re.DOTALL)
# deal = [x.replace('\n', '').strip() for x in temps]
# print(deal)
# self.results += deal
class WordtrackerSpider(scrapy.Spider):
"""
https://www.wordtracker.com/search?query=food%20bags
"""
name = 'wordtracker'
allowed_domains = ['wordtracker.com']
results = []
keywords = []
headers = {
'User-Agent': UserAgent().random()
}
def start_requests(self):
print('wordtracker start_requests')
"""
start_requests做为程序的入口，可以重写，自定义第一批请求
"""
start_urls = ['https://www.wordtracker.com/search?query={k}'.format(k=quote_plus(k)) for k in self.keywords]
# , meta={'proxy': 'socks5h://127.0.0.1:1080/'}
for url in start_urls:
yield scrapy.Request(url, headers=self.headers,
callback=self.parse)
def parse(self, response):
print('wordtracker parse')
with open('wordtracker.html', mode='w') as f:
f.write(response.text)
f.close()
temps = re.findall(r'<span class="a-size-base a-color-base s-line-clamp-2">(.+?)</span>', response.text,
re.DOTALL)
deal = [x.replace('\n', '').strip() for x in temps]
print(deal)
self.results += deal
================================================
FILE: 003-Keywords/google.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: google相关获取
@Date :2021/10/08
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
import xlsxwriter
import os.path
import matplotlib.pyplot as plt
from mypytrends.request import TrendReq
class GoogleTrend(object):
def __init__(self):
self.data = {}
self.max_column = 0
def search(self, keyword, path, hl='en-US', proxies=False, retries=2, timeframe='2019-10-01 2022-01-01'):
if not os.path.exists(path):
os.makedirs(path)
csv_file = os.path.join(path, "%s.xlsx" % keyword)
workbook = xlsxwriter.Workbook(csv_file)
# 设置整个工作薄的格式
workbook.formats[0].set_align('vcenter') # 单元格垂直居中
# workbook.formats[0].set_text_wrap() # 自动换行
worksheet = workbook.add_worksheet('sheet1')
i_row, i_column = 0, 0
first_row = ['keyword', 'no', 'top keyword', 'top range', 'rising keyword', 'rising range']
self.max_column = len(first_row)
for i in range(self.max_column):
worksheet.write(i_row, i, first_row[i])
i_row += 1
tr = self.__get_req(hl=hl, proxies=proxies, retries=retries)
i_row = self.__search_trends(i_row, worksheet, path, keyword, timeframe, tr)
first_data = self.__search_related_queries(keyword, timeframe, tr)
tops = first_data[keyword]['top']
risings = first_data[keyword]['rising']
top_size = len(tops)
rising_size = len(risings)
max_len = top_size if top_size > rising_size else rising_size
for i in range(max_len):
top_key, top_range, rising_key, rising_range = None, None, None, None
if i < top_size:
top_key = tops[i]['keyword']
top_range = tops[i]['range']
if i < rising_size:
rising_key = risings[i]['keyword']
rising_range = risings[i]['range']
max_datas = [keyword, i, top_key, top_range, rising_key, rising_range]
for j in range(len(max_datas)):
worksheet.write(i_row, j, max_datas[j])
i_row += 1
# self.__save_line(csv_file, [keyword, i, top_key, top_range, rising_key, rising_range])
# self.__save_line(csv_file, ['', '']) # 换行
i_row += 1
for top in tops:
top_key = top['keyword']
try:
i_row = self.__sub_search(top_key, i_row, worksheet, path, csv_file, timeframe, tr)
# self.__save_line(csv_file, ['', '']) # 换行
i_row += 1
except:
pass
for rising in risings:
rising_key = rising['keyword']
try:
i_row = self.__sub_search(rising_key, i_row, worksheet, path, csv_file, timeframe, tr)
# self.__save_line(csv_file, ['', '']) # 换行
i_row += 1
except:
pass
workbook.close()
def __sub_search(self, i_row, worksheet, keyword, path, csv_file, timeframe, tr):
i_row = self.__search_trends(i_row, worksheet, path, keyword, timeframe, tr)
first_data = self.__search_related_queries(keyword, timeframe, tr)
tops = first_data[keyword]['top']
risings = first_data[keyword]['rising']
top_size = len(tops)
rising_size = len(risings)
max_len = top_size if top_size > rising_size else rising_size
for i in range(max_len):
top_key, top_range, rising_key, rising_range = None, None, None, None
if i < top_size:
top_key = tops[i]['keyword']
top_range = tops[i]['range']
try:
i_row = self.__search_trends(i_row, worksheet, path, top_key, timeframe, tr)
except:
pass
if i < rising_size:
rising_key = risings[i]['keyword']
rising_range = risings[i]['range']
try:
i_row = self.__search_trends(i_row, worksheet, path, rising_key, timeframe, tr)
except:
pass
# self.__save_line(csv_file, [keyword, i, top_key, top_range, rising_key, rising_range])
max_datas = [keyword, i, top_key, top_range, rising_key, rising_range]
for j in range(len(max_datas)):
worksheet.write(i_row, j, max_datas[j])
i_row += 1
return i_row
def __search_related_queries(self, keyword, timeframe, tr: TrendReq) -> {}:
tr.build_payload([keyword, ], cat=0, timeframe=timeframe, geo='', gprop='')
related = tr.related_queries()
r_value = [related[key] for key in related][0]
r_top = r_value['top']
r_rising = r_value['rising']
tops = []
risings = []
print('---------top--------')
for index, row in r_top.iterrows():
print(index, row["query"], row["value"])
tops.append({"keyword": row["query"], "range": row["value"]})
print('---------rising--------')
for index, row in r_rising.iterrows():
print(index, row["query"], row["value"])
risings.append({"keyword": row["query"], "range": row["value"]})
return {keyword: {"top": tops, "rising": risings}}
def __search_trends(self, i_row, worksheet, path, keyword, timeframe, tr: TrendReq):
tr.build_payload([keyword, ], cat=0, timeframe=timeframe, geo='', gprop='')
trends = tr.interest_over_time()
x_data = []
y_data = []
year = ''
month = ''
temp_value = 0
count = 0
for time, row in trends.iterrows():
value = row[keyword]
date = str(time)
t = date.split(' ')[0] if ' ' in date else date
y = t.split('-')[0]
m = t.split('-')[1]
if month != m and month != '':
y_data.append(int(temp_value / count))
x_data.append(year[2:] + "-" + month)
year = ''
month = ''
count = 0
temp_value = 0
continue
year = y
month = m
count += 1
temp_value += value
y_data.append(int(temp_value / count))
x_data.append(year[2:] + "-" + month)
print(y_data)
print(x_data)
print('%s : %d - %d' % (keyword, len(y_data), len(x_data)))
# self.__draw_graph(x_data, y_data, 'pci.jpg', keyword)
img_path = os.path.join(path, "%s.jpg" % keyword)
self.__draw_histogram(x_data, y_data, img_path, keyword)
worksheet.insert_image(i_row - 1, self.max_column, img_path,
{'x_scale': 0.2, 'y_scale': 0.2, 'object_position': 1})
i_row += 1
return i_row
def __draw_histogram(self, x: [], y: [], path, title, x_name='date', y_name='trends'):
plt.figure(dpi=60)
plt.ylim(0, 100)
plt.style.use('ggplot')
plt.bar(x, y, label=title)
# 显示图例（使绘制生效）
plt.legend()
# 横坐标名称
plt.xlabel(x_name)
# 纵坐标名称
plt.ylabel(y_name)
# 横坐标显示倒立
plt.xticks(rotation=90)
# 保存图片到本地
plt.savefig(path)
# 显示图片
# plt.show()
def __draw_graph(self, x: [], y: [], path, title, x_name='date', y_name='trends'):
plt.figure()
'''绘制第一条数据线
1、节点为圆圈
2、线颜色为红色
3、标签名字为y1-data
'''
plt.ylim(0, 100)
plt.plot(x, y, marker='o', color='r', label=title)
# 显示图例（使绘制生效）
plt.legend()
# 横坐标名称
plt.xlabel(x_name)
# 纵坐标名称
plt.ylabel(y_name)
# 横坐标显示倒立
plt.xticks(rotation=90)
# 保存图片到本地
plt.savefig(path)
# 显示图片
# plt.show()
def __save_line(self, path, line, mode='a'):
with open(path, mode=mode, encoding='utf-8', newline='') as csvfile:
writer = csv.writer(csvfile)
writer.writerow(line)
csvfile.close()
def __get_req(self, hl='en-US', proxies=False, retries=2) -> TrendReq:
if proxies:
return TrendReq(hl=hl, tz=360, timeout=(10, 35), proxies=['socks5h://127.0.0.1:1080', ], retries=retries,
backoff_factor=0.1, requests_args={'verify': False})
else:
return TrendReq(hl=hl, tz=360, timeout=(10, 35), retries=retries, backoff_factor=0.1,
requests_args={'verify': False})
================================================
FILE: 003-Keywords/main.py
================================================
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@Description: 关键词获取
@Date :2021/09/22
@Author :xhunmon
@Mail :xhunmon@gmail.com
"""
# from amazon import run_api
import os
import time
import xlwt
import run_api
import v2ray_util as v2util
from google import GoogleTrend
from openpyxl import load_workbook
def Write_Img():
import xlsxwriter
book = xlsxwriter.Workbook('test_source.xlsx')
sheet = book.get_worksheet_by_name('Sheet1')
# sheet = book.add_worksheet('demo')
# sheet.insert_image(0, 5, 'Necklace/Necklace.jpg', {'x_scale': 0.2, 'y_scale': 0.2, 'object_position': 1})
print(sheet)
book.close()
def read_xlsl():
import pandas as pd
df = pd.read_excel('test_source.xlsx', sheet_name='Sheet1')
data = df.values
print(data)
print('\n')
df = pd.read_excel('test_souce1.xlsx', sheet_name='2021xuqiu')
data = df.values
print(data)
if __name__ == "__main__":
# v2util.restart_v2ray()
# 获取amazon中相关词，代理需要看：middlewares.py，
# results = run_api.crawl_amazon(['plastic packaging', ])
# 把通过google trends查出关键词相关词，以及其词的趋势图，如：
# GoogleTrend().search('Necklace', 'Necklace', proxies=True, timeframe='2021-01-01 2022-01-01')
# v2util.kill_all_v2ray()
# Write_Img()
read_xlsl()
================================================
FILE: 003-Keywords/mypytrends/__init__.py
================================================
================================================
FILE: 003-Keywords/mypytrends/dailydata.py
================================================
from datetime import date, timedelta
from functools import partial
from time import sleep
from calendar import monthrange
import pandas as pd
from mypytrends.exceptions import ResponseError
from mypytrends.request import TrendReq
def get_last_date_of_month(year: int, month: int) -> date:
"""Given a year and a month returns an instance of the date class
containing the last day of the corresponding month.
Source: https://stackoverflow.com/questions/42950/get-last-day-of-the-month-in-python
"""
return date(year, month, monthrange(year, month)[1])
def convert_dates_to_timeframe(start: date, stop: date) -> str:
"""Given two dates, returns a stringified version of the interval between
the two dates which is used to retrieve data for a specific time frame
from Google Trends.
"""
return f"{start.strftime('%Y-%m-%d')} {stop.strftime('%Y-%m-%d')}"
def _fetch_data(pytrends, build_payload, timeframe: str) -> pd.DataFrame:
"""Attempts to fecth data and retries in case of a ResponseError."""
attempts, fetched = 0, False
while not fetched:
try:
build_payload(timeframe=timeframe)
except ResponseError as err:
print(err)
print(f'Trying again in {60 + 5 * attempts} seconds.')
sleep(60 + 5 * attempts)
attempts += 1
if attempts > 3:
print('Failed after 3 attemps, abort fetching.')
break
else:
fetched = True
return pytrends.interest_over_time()
def get_daily_data(word: str,
start_year: int,
start_mon: int,
stop_year: int,
stop_mon: int,
geo: str = 'US',
verbose: bool = True,
wait_time: float = 5.0) -> pd.DataFrame:
"""Given a word, fetches daily search volume data from Google Trends and
returns results in a pandas DataFrame.
Details: Due to the way Google Trends scales and returns data, special
care needs to be taken to make the daily data comparable over different
months. To do that, we download daily data on a month by month basis,
and also monthly data. The monthly data is downloaded in one go, so that
the monthly values are comparable amongst themselves and can be used to
scale the daily data. The daily data is scaled by multiplying the daily
value by the monthly search volume divided by 100.
For a more detailed explanation see http://bit.ly/trendsscaling
Args:
word (str): Word to fetch daily data for.
start_year (int): the start year
start_mon (int): start 1st day of the month
stop_year (int): the end year
stop_mon (int): end at the last day of the month
geo (str): geolocation
verbose (bool): If True, then prints the word and current time frame
we are fecthing the data for.
Returns:
complete (pd.DataFrame): Contains 4 columns.
The column named after the word argument contains the daily search
volume already scaled and comparable through time.
The column f'{word}_unscaled' is the original daily data fetched
month by month, and it is not comparable across different months
(but is comparable within a month).
The column f'{word}_monthly' contains the original monthly data
fetched at once. The values in this column have been backfilled
so that there are no NaN present.
The column 'scale' contains the scale used to obtain the scaled
daily data.
"""
# Set up start and stop dates
start_date = date(start_year, start_mon, 1)
stop_date = get_last_date_of_month(stop_year, stop_mon)
# Start pytrends for US region
pytrends = TrendReq(hl='en-US', tz=360)
# Initialize build_payload with the word we need data for
build_payload = partial(pytrends.build_payload,
kw_list=[word], cat=0, geo=geo, gprop='')
# Obtain monthly data for all months in years [start_year, stop_year]
monthly = _fetch_data(pytrends, build_payload,
convert_dates_to_timeframe(start_date, stop_date))
# Get daily data, month by month
results = {}
# if a timeout or too many requests error occur we need to adjust wait time
current = start_date
while current < stop_date:
last_date_of_month = get_last_date_of_month(current.year, current.month)
timeframe = convert_dates_to_timeframe(current, last_date_of_month)
if verbose:
print(f'{word}:{timeframe}')
results[current] = _fetch_data(pytrends, build_payload, timeframe)
current = last_date_of_month + timedelta(days=1)
sleep(wait_time) # don't go too fast or Google will send 429s
daily = pd.concat(results.values()).drop(columns=['isPartial'])
complete = daily.join(monthly, lsuffix='_unscaled', rsuffix='_monthly')
# Scale daily data by monthly weights so the data is comparable
complete[f'{word}_monthly'].ffill(inplace=True) # fill NaN values
complete['scale'] = complete[f'{word}_monthly'] / 100
complete[word] = complete[f'{word}_unscaled'] * complete.scale
return complete
================================================
FILE: 003-Keywords/mypytrends/exceptions.py
================================================
class ResponseError(Exception):
"""Something was wrong with the response from Google"""
def __init__(self, message, response):
super(Exception, self).__init__(message)
# pass response so it can be handled upstream
self.response = response
================================================
FILE: 003-Keywords/mypytrends/request.py
================================================
import json
import sys
import time
from datetime import datetime, timedelta
import pandas as pd
import requests
from pandas.io.json._normalize import nested_to_record
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from mypytrends import exceptions
from urllib.parse import quote
class TrendReq(object):
"""
Google Trends API
"""
GET_METHOD = 'get'
POST_METHOD = 'post'
GENERAL_URL = 'https://trends.google.com/trends/api/explore'
INTEREST_OVER_TIME_URL = 'https://trends.google.com/trends/api/widgetdata/multiline'
INTEREST_BY_REGION_URL = 'https://trends.google.com/trends/api/widgetdata/comparedgeo'
RELATED_QUERIES_URL = 'https://trends.google.com/trends/api/widgetdata/relatedsearches'
TRENDING_SEARCHES_URL = 'https://trends.google.com/trends/hottrends/visualize/internal/data'
TOP_CHARTS_URL = 'https://trends.google.com/trends/api/topcharts'
SUGGESTIONS_URL = 'https://trends.google.com/trends/api/autocomplete/'
CATEGORIES_URL = 'https://trends.google.com/trends/api/explore/pickers/category'
TODAY_SEARCHES_URL = 'https://trends.google.com/trends/api/dailytrends'
ERROR_CODES = (500, 502, 504, 429)
def __init__(self, hl='en-US', tz=360, geo='', timeout=(2, 5), proxies='',
retries=0, backoff_factor=0, requests_args=None):
"""
Initialize default values for params
"""
# google rate limit
self.google_rl = 'You have reached your quota limit. Please try again later.'
self.results = None
# set user defined options used globally
self.tz = tz
self.hl = hl
self.geo = geo
self.kw_list = list()
self.timeout = timeout
self.proxies = proxies # add a proxy option
self.retries = retries
self.backoff_factor = backoff_factor
self.proxy_index = 0
self.requests_args = requests_args or {}
self.cookies = self.GetGoogleCookie()
# intialize widget payloads
self.token_payload = dict()
self.interest_over_time_widget = dict()
self.interest_by_region_widget = dict()
self.related_topics_widget_list = list()
self.related_queries_widget_list = list()
def GetGoogleCookie(self):
"""
Gets google cookie (used for each and every proxy; once on init otherwise)
Removes proxy from the list on proxy error
"""
while True:
if "proxies" in self.requests_args:
try:
return dict(filter(lambda i: i[0] == 'NID', requests.get(
'https://trends.google.com/?geo={geo}'.format(
geo=self.hl[-2:]),
timeout=self.timeout,
**self.requests_args
).cookies.items()))
except:
continue
else:
if len(self.proxies) > 0:
proxy = {'https': self.proxies[self.proxy_index]}
else:
proxy = ''
try:
return dict(filter(lambda i: i[0] == 'NID', requests.get(
'https://trends.google.com/?geo={geo}'.format(
geo=self.hl[-2:]),
timeout=self.timeout,
proxies=proxy,
**self.requests_args
).cookies.items()))
except requests.exceptions.ProxyError:
print('Proxy error. Changing IP')
if len(self.proxies) > 1:
self.proxies.remove(self.proxies[self.proxy_index])
else:
print('No more proxies available. Bye!')
raise
continue
def GetNewProxy(self):
"""
Increment proxy INDEX; zero on overflow
"""
if self.proxy_index < (len(self.proxies) - 1):
self.proxy_index += 1
else:
self.proxy_index = 0
def _get_data(self, url, method=GET_METHOD, trim_chars=0, **kwargs):
"""Send a request to Google and return the JSON response as a Python object
:param url: the url to which the request will be sent
:param method: the HTTP method ('get' or 'post')
:param trim_chars: how many characters should be trimmed off the beginning of the content of the response
before this is passed to the JSON parser
:param kwargs: any extra key arguments passed to the request builder (usually query parameters or data)
:return:
"""
s = requests.session()
# Retries mechanism. Activated when one of statements >0 (best used for proxy)
if self.retries > 0 or self.backoff_factor > 0:
retry = Retry(total=self.retries, read=self.retries,
connect=self.retries,
backoff_factor=self.backoff_factor,
status_forcelist=TrendReq.ERROR_CODES,
method_whitelist=frozenset(['GET', 'POST']))
s.mount('https://', HTTPAdapter(max_retries=retry))
s.headers.update({'accept-language': self.hl})
if len(self.proxies) > 0:
self.cookies = self.GetGoogleCookie()
s.proxies.update({'https': self.proxies[self.proxy_index]})
if method == TrendReq.POST_METHOD:
response = s.post(url, timeout=self.timeout,
cookies=self.cookies, **kwargs,
**self.requests_args) # DO NOT USE retries or backoff_factor here
else:
response = s.get(url, timeout=self.timeout, cookies=self.cookies,
**kwargs, **self.requests_args) # DO NOT USE retries or backoff_factor here
# check if the response contains json and throw an exception otherwise
# Google mostly sends 'application/json' in the Content-Type header,
# but occasionally it sends 'application/javascript
# and sometimes even 'text/javascript
if response.status_code == 200 and 'application/json' in \
response.headers['Content-Type'] or \
'application/javascript' in response.headers['Content-Type'] or \
'text/javascript' in response.headers['Content-Type']:
# trim initial characters
# some responses start with garbage characters, like ")]}',"
# these have to be cleaned before being passed to the json parser
content = response.text[trim_chars:]
# parse json
self.GetNewProxy()
return json.loads(content)
else:
# error
raise exceptions.ResponseError(
'The request failed: Google returned a '
'response with code {0}.'.format(response.status_code),
response=response)
def build_payload(self, kw_list, cat=0, timeframe='today 5-y', geo='',
gprop=''):
"""Create the payload for related queries, interest over time and interest by region"""
if gprop not in ['', 'images', 'news', 'youtube', 'froogle']:
raise ValueError('gprop must be empty (to indicate web), images, news, youtube, or froogle')
self.kw_list = kw_list
self.geo = geo or self.geo
self.token_payload = {
'hl': self.hl,
'tz': self.tz,
'req': {'comparisonItem': [], 'category': cat, 'property': gprop}
}
# build out json for each keyword
for kw in self.kw_list:
keyword_payload = {'keyword': kw, 'time': timeframe,
'geo': self.geo}
self.token_payload['req']['comparisonItem'].append(keyword_payload)
# requests will mangle this if it is not a string
self.token_payload['req'] = json.dumps(self.token_payload['req'])
# get tokens
self._tokens()
return
def _tokens(self):
"""Makes request to Google to get API tokens for interest over time, interest by region and related queries"""
# make the request and parse the returned json
widget_dicts = self._get_data(
url=TrendReq.GENERAL_URL,
method=TrendReq.GET_METHOD,
params=self.token_payload,
trim_chars=4,
)['widgets']
# order of the json matters...
first_region_token = True
# clear self.related_queries_widget_list and self.related_topics_widget_list
# of old keywords'widgets
self.related_queries_widget_list[:] = []
self.related_topics_widget_list[:] = []
# assign requests
gitextract_xtly8u7o/ ├── .gitignore ├── 001-Downloader/ │ ├── .gitignore │ ├── README.md │ ├── __init__.py │ ├── config.ini │ ├── doc/ │ │ ├── mac-sh/ │ │ │ ├── main.spec │ │ │ └── pyinstaller.sh │ │ └── win-sh/ │ │ ├── main.spec │ │ └── pyinstaller.sh │ ├── douyin/ │ │ └── dy_download.py │ ├── downloader.py │ ├── kuaishou/ │ │ └── ks_download.py │ ├── main.py │ ├── test/ │ │ ├── bilibili_video_download_v1.py │ │ ├── ff_video.py │ │ ├── test_pyinstaller.py │ │ ├── urls.txt │ │ └── xhs_download.py │ ├── type_enum.py │ ├── ui.py │ └── utils.py ├── 002-V2rayPool/ │ ├── .gitignore │ ├── 002-V2rayPool.iml │ ├── README.md │ ├── base/ │ │ └── net_proxy.py │ ├── core/ │ │ ├── client.py │ │ ├── conf.py │ │ ├── group.py │ │ ├── json_template/ │ │ │ ├── client.json │ │ │ ├── client_socks.json │ │ │ ├── client_ss.json │ │ │ ├── client_trojan.json │ │ │ ├── dyn_port.json │ │ │ ├── http.json │ │ │ ├── http2.json │ │ │ ├── kcp.json │ │ │ ├── mtproto.json │ │ │ ├── quic.json │ │ │ ├── server.json │ │ │ ├── socks.json │ │ │ ├── ss.json │ │ │ ├── stats_settings.json │ │ │ ├── tcp.json │ │ │ ├── vless.json │ │ │ └── ws.json │ │ ├── profile.py │ │ └── utils.py │ ├── db/ │ │ ├── db_main.py │ │ ├── local.py │ │ └── net.py │ ├── doc/ │ │ └── (参考用)config.json │ └── test_main.py ├── 003-Keywords/ │ ├── .gitignore │ ├── Necklace/ │ │ └── Necklace.xlsx │ ├── README.md │ ├── __init__.py │ ├── amazon/ │ │ ├── items.py │ │ ├── middlewares.py │ │ ├── pipelines.py │ │ ├── settings.py │ │ └── spiders/ │ │ ├── __init__.py │ │ ├── alibaba.py │ │ ├── amazon.py │ │ ├── checkip.py │ │ └── spiders.py │ ├── google.py │ ├── main.py │ ├── mypytrends/ │ │ ├── __init__.py │ │ ├── dailydata.py │ │ ├── exceptions.py │ │ ├── request.py │ │ └── test_trendReq.py │ ├── run_api.py │ ├── scrapy.cfg │ ├── test_souce1.xlsx │ ├── test_source.xlsx │ ├── v2ray_pool/ │ │ ├── __init__.py │ │ ├── _db-checked.txt │ │ └── _db-uncheck.txt │ ├── v2ray_util.py │ └── women-ring/ │ └── women ring.csv ├── 004-EmailNotify/ │ ├── .gitignore │ ├── README.md │ └── main.py ├── 005-PaidSource/ │ ├── .gitignore │ ├── 005-PaidSource.iml │ ├── README.md │ ├── __init__.py │ ├── chrome.py │ ├── ff_video.py │ ├── file_util.py │ ├── gsearch.py │ ├── gtransfer.py │ ├── kaoqin.py │ ├── keywords.py │ ├── main.py │ ├── other_site.py │ ├── v2ray_pool/ │ │ ├── __init__.py │ │ ├── _db-checked.txt │ │ └── _db-uncheck.txt │ └── v2ray_util.py ├── 006-TikTok/ │ ├── .gitignore │ ├── 006-TikTok.iml │ ├── README.md │ ├── __init__.py │ ├── dy_review.py │ ├── file_util.py │ ├── google_transfer_by_excel.py │ ├── img_2_webp.py │ ├── main.py │ ├── post_autotk.py │ ├── tikstar.py │ ├── tt_review.py │ └── v2ray_pool/ │ └── __init__.py ├── 007-CutVideoAudio/ │ ├── .gitignore │ ├── 007-CutVideoAudio.iml │ ├── README.md │ ├── __init__.py │ ├── config.ini │ ├── doc/ │ │ └── mac-sh/ │ │ ├── main.spec │ │ └── pyinstaller.sh │ ├── editors.py │ ├── ff_cut.py │ ├── ff_util.py │ ├── ff_util_v2.py │ ├── main.py │ ├── type_enum.py │ ├── ui.py │ └── utils.py ├── 008-ChatGPT-UI/ │ ├── .gitignore │ ├── README.md │ ├── config.ini │ ├── config.json │ ├── doc/ │ │ ├── config.json │ │ └── pyinstaller.sh │ ├── gpt.py │ ├── main.py │ ├── requirements.txt │ └── utils.py ├── 009-Translate/ │ ├── README.md │ ├── asset/ │ │ ├── ch.ini │ │ ├── config.ini │ │ ├── en.ini │ │ └── language.json │ ├── config.py │ ├── core.py │ ├── doc/ │ │ └── pyinstaller.sh │ ├── load_srt.py │ ├── main.py │ ├── tran_test.py │ ├── ui.py │ └── utils.py ├── 010-YouTubeUpload/ │ ├── README.md │ ├── main.py │ ├── tst.py │ └── youtube.py ├── LICENSE └── README.md
SYMBOL INDEX (647 symbols across 69 files)
FILE: 001-Downloader/douyin/dy_download.py
class DouYin (line 20) | class DouYin(Downloader):
method __init__ (line 22) | def __init__(self):
method start (line 28) | def start(self, url, path):
method parse_single (line 64) | def parse_single(self):
method Find (line 89) | def Find(self, string):
method judge_link (line 96) | def judge_link(self):
method get_data (line 116) | def get_data(self, api_post_url, max_cursor):
method next_data (line 146) | def next_data(self, max_cursor):
method video_info (line 185) | def video_info(self, result, max_cursor):
method videos_download (line 214) | def videos_download(self, author_list, video_list, aweme_id, nickname,...
FILE: 001-Downloader/downloader.py
class Downloader (line 21) | class Downloader(object):
method __init__ (line 33) | def __init__(self):
method print_hint (line 38) | def print_hint():
method start (line 50) | def start(self, url, path):
method print_ui (line 55) | def print_ui(txt):
method print_all_ui (line 60) | def print_all_ui(txt, print_type: PrintType = PrintType.log):
method get_beijing_time (line 66) | def get_beijing_time():
method is_expired (line 90) | def is_expired():
method add_total_count (line 99) | def add_total_count(count=1):
method get_total_count (line 107) | def get_total_count():
method add_downloading_count (line 112) | def add_downloading_count():
method __sub_downloading_count (line 120) | def __sub_downloading_count():
method get_downloading_count (line 128) | def get_downloading_count():
method add_success_count (line 133) | def add_success_count():
method get_success_count (line 143) | def get_success_count():
method add_failed_count (line 148) | def add_failed_count():
method get_failed_count (line 158) | def get_failed_count():
FILE: 001-Downloader/kuaishou/ks_download.py
class KuaiShou (line 23) | class KuaiShou(Downloader):
method __init__ (line 27) | def __init__(self):
method set_cookie (line 33) | def set_cookie(self, c):
method start (line 37) | def start(self, url, path):
method parse_single_trendingId (line 54) | def parse_single_trendingId(self, url):
method parse_single_streamSource (line 81) | def parse_single_streamSource(self, url):
method parse_user (line 107) | def parse_user(self, url):
method post_single_trendingId (line 139) | def post_single_trendingId(self, url, Cookie, trendingId, area):
method post_single_streamSource (line 170) | def post_single_streamSource(self, url, Cookie, photoId, area):
method post_user (line 201) | def post_user(self, userId, Cookie, pcursor):
method progressbar (line 228) | def progressbar(self, url, filepath, filename):
method download (line 260) | def download(self, feeds):
FILE: 001-Downloader/test/bilibili_video_download_v1.py
function get_play_list (line 20) | def get_play_list(start_url, cid, quality):
function Schedule_cmd (line 50) | def Schedule_cmd(blocknum, blocksize, totalsize):
function Schedule (line 68) | def Schedule(blocknum, blocksize, totalsize):
function format_size (line 87) | def format_size(bytes):
function down_video (line 106) | def down_video(video_list, title, start_url, page):
function combine_video (line 139) | def combine_video(video_list, title):
function getAid (line 171) | def getAid(Bid):
FILE: 001-Downloader/test/ff_video.py
function cute_video (line 12) | def cute_video(folder):
FILE: 001-Downloader/test/xhs_download.py
function download_url (line 16) | def download_url(url, index):
FILE: 001-Downloader/type_enum.py
class PrintType (line 13) | class PrintType(Enum):
FILE: 001-Downloader/ui.py
class Ui (line 22) | class Ui(Frame):
method __init__ (line 23) | def __init__(self, master=None):
method window_init (line 32) | def window_init(self):
method createWidgets (line 41) | def createWidgets(self):
method save_dir (line 114) | def save_dir(self):
method set_dir (line 118) | def set_dir(self, path):
method download_url (line 122) | def download_url(self):
method output (line 127) | def output(self, txt):
method func_ui_print (line 130) | def func_ui_print(self, txt, print_type: PrintType = None):
method start_download (line 142) | def start_download(self):
FILE: 001-Downloader/utils.py
function get_domain (line 15) | def get_domain(url: str = None):
class Config (line 25) | class Config(object):
method __init__ (line 31) | def __init__(self):
method instance (line 38) | def instance(cls, *args, **kwargs):
method get_expired_time (line 44) | def get_expired_time(self):
method get_version_name (line 47) | def get_version_name(self):
method get_version_code (line 50) | def get_version_code(self):
FILE: 002-V2rayPool/base/net_proxy.py
class Net (line 14) | class Net(object):
method __init__ (line 20) | def __init__(self, timeout=8):
method get_header (line 37) | def get_header(self, headers, key):
method update_agent (line 45) | def update_agent(self):
method get_urls (line 49) | def get_urls(self) -> []:
method request (line 53) | def request(self, url, allow_redirects=False, verify=False, timeout=TI...
method request_en (line 57) | def request_en(self, url, allow_redirects=False, verify=False, timeout...
method request_zh (line 62) | def request_zh(self, url, allow_redirects=False, verify=False, timeout...
method __request (line 67) | def __request(self, url, allow_redirects=False, verify=False, proxies=...
FILE: 002-V2rayPool/core/client.py
class ClientWriter (line 15) | class ClientWriter:
method __init__ (line 16) | def __init__(self, group):
method load_template (line 26) | def load_template(self, template_name):
method transform (line 34) | def transform(self):
method write (line 103) | def write(self):
class Creator (line 113) | class Creator(object):
method __init__ (line 118) | def __init__(self):
method parse_vmess (line 122) | def parse_vmess(self, vmesslink):
method parse_trojan (line 135) | def parse_trojan(self, link):
method parse_ss (line 154) | def parse_ss(self, sslink):
method generateAndWrite (line 200) | def generateAndWrite(self, url: str):
method __kill_threading (line 241) | def __kill_threading(self):
method __child_thread (line 257) | def __child_thread(self, url: str, isSysOn=False):
method v2ray_start (line 269) | def v2ray_start(self, url: str, isSysOn=False):
method v2ray_start_with_log (line 273) | def v2ray_start_with_log(self, url: str, isSysOn=False):
FILE: 002-V2rayPool/core/conf.py
class Config (line 8) | class Config:
method __init__ (line 12) | def __init__(self):
method get_path (line 19) | def get_path(self, key):
method get_data (line 23) | def get_data(self, key):
method set_data (line 26) | def set_data(self, key, value):
method set_v2ray_core_path (line 31) | def set_v2ray_core_path(dir: str):
method get_v2ray_core_path (line 36) | def get_v2ray_core_path():
method set_v2ray_node_path (line 41) | def set_v2ray_node_path(dir: str):
method get_v2ray_node_path (line 46) | def get_v2ray_node_path():
FILE: 002-V2rayPool/core/group.py
class Dyport (line 10) | class Dyport(object):
method __init__ (line 11) | def __init__(self, status=False, aid=0):
class Quic (line 16) | class Quic(object):
method __init__ (line 17) | def __init__(self, security="none", key="", header="none"):
class User (line 23) | class User(object):
method __init__ (line 24) | def __init__(self, user_number, password, user_info=None):
method password (line 34) | def password(self):
class SS (line 38) | class SS(User):
method __init__ (line 39) | def __init__(self, user_number, password, method, user_info):
method __str__ (line 43) | def __str__(self):
method link (line 50) | def link(self, ip, port, tls):
method stream (line 54) | def stream(self):
class Trojan (line 58) | class Trojan(User):
method __init__ (line 59) | def __init__(self, user_number, password, email):
method __str__ (line 62) | def __str__(self):
method link (line 68) | def link(self, ip, port, tls):
method stream (line 71) | def stream(self):
class Mtproto (line 75) | class Mtproto(User):
method __str__ (line 76) | def __str__(self):
method link (line 82) | def link(self, ip, port, tls):
method stream (line 85) | def stream(self):
class Socks (line 89) | class Socks(User):
method __str__ (line 90) | def __str__(self):
method link (line 93) | def link(self, ip, port, tls):
method stream (line 99) | def stream(self):
class Vless (line 103) | class Vless(User):
method __init__ (line 104) | def __init__(self, uuid, user_number, encryption=None, email=None, net...
method __str__ (line 116) | def __str__(self):
method stream (line 128) | def stream(self):
method link (line 143) | def link(self, ip, port, tls):
class Vmess (line 160) | class Vmess(User):
method __init__ (line 161) | def __init__(self, uuid, alter_id: int, network: str, user_number, *, ...
method stream (line 175) | def stream(self):
method __str__ (line 196) | def __str__(self):
method link (line 208) | def link(self, ip, port, tls):
class Group (line 227) | class Group(object):
method __init__ (line 228) | def __init__(self, ip, port: str, *, end_port=None, tfo=None, tls="non...
FILE: 002-V2rayPool/core/profile.py
class Stats (line 10) | class Stats:
method __init__ (line 11) | def __init__(self, status=False, door_port=0):
method __str__ (line 15) | def __str__(self):
class Profile (line 19) | class Profile:
method __init__ (line 20) | def __init__(self):
method __str__ (line 30) | def __str__(self):
method read_json (line 37) | def read_json(self):
method parse_group (line 77) | def parse_group(self, part_json, group_index, local_ip):
FILE: 002-V2rayPool/core/utils.py
class ProtocolType (line 17) | class ProtocolType(object):
class StreamType (line 24) | class StreamType(Enum):
function header_type_list (line 53) | def header_type_list():
function ss_method (line 57) | def ss_method():
function xtls_flow (line 61) | def xtls_flow():
function get_ip (line 65) | def get_ip():
function port_is_use (line 77) | def port_is_use(port):
function random_port (line 95) | def random_port(start_port, end_port):
function is_email (line 102) | def is_email(email):
function is_ipv4 (line 110) | def is_ipv4(ip):
function is_ipv6 (line 124) | def is_ipv6(ip):
function check_ip (line 132) | def check_ip(ip):
function bytes_2_human_readable (line 136) | def bytes_2_human_readable(number_of_bytes, precision=1):
function random_email (line 169) | def random_email():
function readchar (line 175) | def readchar(prompt=""):
function kill_all_v2ray (line 192) | def kill_all_v2ray():
function kill_process_by_port (line 205) | def kill_process_by_port(port):
function sys_proxy_on (line 213) | def sys_proxy_on(proxy, port):
function sys_v2ray_on (line 220) | def sys_v2ray_on():
function sys_proxy_off (line 228) | def sys_proxy_off():
FILE: 002-V2rayPool/db/db_main.py
class DBManage (line 16) | class DBManage(object):
method init (line 17) | def init(self):
method __add_urls_de_dup (line 22) | def __add_urls_de_dup(self, all_urls: [], new_urls: []) -> []:
method start_random_v2ray_by_local (line 32) | def start_random_v2ray_by_local(self, isSysOn=False):
method load_urls_and_save_auto (line 49) | def load_urls_and_save_auto(self):
method load_urls_by_not_proxy (line 72) | def load_urls_by_not_proxy(self, save_local=True):
method load_urls_by_net_with_proxy (line 89) | def load_urls_by_net_with_proxy(self, proxy_url=None, save_local=True):
method load_urls_by_net (line 113) | def load_urls_by_net(self, proxy_url=None, save_local=True, need_proxy...
method load_unchecked_urls_by_local (line 156) | def load_unchecked_urls_by_local(self):
method load_enable_urls_by_local (line 162) | def load_enable_urls_by_local(self):
method check_url_single (line 166) | def check_url_single(self, url: str):
method check_and_save (line 177) | def check_and_save(self, urls: [], append=True):
FILE: 002-V2rayPool/db/local.py
class DbLocal (line 17) | class DbLocal(object):
method __init__ (line 22) | def __init__(self):
method __reset (line 38) | def __reset(self):
method __check_url (line 45) | def __check_url(self, url: str):
method is_checked_finished (line 57) | def is_checked_finished(self) -> bool:
method get_checked_urls (line 61) | def get_checked_urls(self) -> []:
method get_urls (line 65) | def get_urls(self, is_check=True) -> bool:
method clear_local (line 99) | def clear_local(self):
method save_urls (line 108) | def save_urls(self, urls, append=True):
class DbEnable (line 148) | class DbEnable(object):
method __init__ (line 154) | def __init__(self):
method get (line 173) | def get(cls, *args, **kwargs):
method init (line 179) | def init(self, config_path, def_url=None):
method __back_config_json (line 195) | def __back_config_json(self):
method __restore_config_json (line 207) | def __restore_config_json(self):
method add_url (line 221) | def add_url(self, url: str):
method add_urls (line 224) | def add_urls(self, url: []):
method select_by_order (line 227) | def select_by_order(self):
method get_by_order (line 232) | def get_by_order(self):
method __check_and_enable (line 239) | def __check_and_enable(self):
method __add (line 248) | def __add(self, url):
method __adds (line 253) | def __adds(self, urls: []):
method __get (line 258) | def __get(self, index):
method clear_local (line 264) | def clear_local(self):
method get_urls (line 273) | def get_urls(self) -> []:
method get_infos (line 286) | def get_infos(self) -> []:
method de_duplication (line 299) | def de_duplication(self):
method save_urls (line 325) | def save_urls(self, urls, append=True):
FILE: 002-V2rayPool/db/net.py
function re_vmess_ss_trojan (line 18) | def re_vmess_ss_trojan(pattern, html) -> []:
class PYCheck (line 43) | class PYCheck(Net):
method get_curren_ip (line 44) | def get_curren_ip(self, url='https://ip.cn/api/index?ip=&type=0'):
class PNFreeV2ray (line 66) | class PNFreeV2ray(Net):
method get_urls (line 71) | def get_urls(self) -> []:
class PNTWGithubV2ray (line 83) | class PNTWGithubV2ray(Net):
method get_urls (line 88) | def get_urls(self) -> []:
class PNSsfree (line 100) | class PNSsfree(Net):
method get_urls (line 105) | def get_urls(self) -> []:
class PNFreevpnX (line 118) | class PNFreevpnX(Net):
method get_urls (line 123) | def get_urls(self) -> []:
class PNGithubIwxf (line 149) | class PNGithubIwxf(Net):
method get_urls (line 154) | def get_urls(self) -> []:
class PYFreevpnnet (line 172) | class PYFreevpnnet(Net):
method get_urls (line 178) | def get_urls(self) -> []:
class PYMerlinblog (line 191) | class PYMerlinblog(Net):
method get_urls (line 197) | def get_urls(self) -> []:
class PYFlyingboat (line 210) | class PYFlyingboat(Net):
method get_urls (line 216) | def get_urls(self) -> []:
class PYIvmess (line 229) | class PYIvmess(Net):
method get_urls (line 235) | def get_urls(self) -> []:
class PYFreeFq (line 248) | class PYFreeFq(Net):
method __get_content_url (line 253) | def __get_content_url(self, date) -> str:
method __get_url_by_content_html (line 269) | def __get_url_by_content_html(self, html: str):
method __get_url_by_detail_html (line 288) | def __get_url_by_detail_html(self, html: str):
method __get_detail_urls (line 300) | def __get_detail_urls(self, url: str) -> []:
method download_urls (line 313) | def download_urls(self, f_day=1) -> []:
method get_urls (line 336) | def get_urls(self) -> []:
FILE: 003-Keywords/amazon/items.py
class AmazonItem (line 9) | class AmazonItem(scrapy.Item):
FILE: 003-Keywords/amazon/middlewares.py
class AmazonSpiderMiddleware (line 12) | class AmazonSpiderMiddleware:
method from_crawler (line 18) | def from_crawler(cls, crawler):
method process_spider_input (line 24) | def process_spider_input(self, response, spider):
method process_spider_output (line 31) | def process_spider_output(self, response, result, spider):
method process_spider_exception (line 39) | def process_spider_exception(self, response, exception, spider):
method process_start_requests (line 46) | def process_start_requests(self, start_requests, spider):
method spider_opened (line 55) | def spider_opened(self, spider):
class AmazonDownloaderMiddleware (line 59) | class AmazonDownloaderMiddleware:
method from_crawler (line 65) | def from_crawler(cls, crawler):
method process_request (line 71) | def process_request(self, request, spider):
method process_response (line 83) | def process_response(self, request, response, spider):
method process_exception (line 92) | def process_exception(self, request, exception, spider):
method spider_opened (line 102) | def spider_opened(self, spider):
class AmazonProxyMiddleware (line 106) | class AmazonProxyMiddleware(object):
method process_request (line 107) | def process_request(self, request, spider):
FILE: 003-Keywords/amazon/pipelines.py
class AmazonPipeline (line 11) | class AmazonPipeline:
method process_item (line 12) | def process_item(self, item, spider):
FILE: 003-Keywords/amazon/spiders/alibaba.py
class AlibabaSpider (line 16) | class AlibabaSpider(scrapy.Spider):
method start_requests (line 26) | def start_requests(self):
method parse (line 39) | def parse(self, response):
FILE: 003-Keywords/amazon/spiders/amazon.py
class AmazonSpider (line 16) | class AmazonSpider(scrapy.Spider):
method start_requests (line 26) | def start_requests(self):
method parse (line 37) | def parse(self, response):
FILE: 003-Keywords/amazon/spiders/checkip.py
class CheckIpSpider (line 15) | class CheckIpSpider(scrapy.Spider):
method start_requests (line 23) | def start_requests(self):
method parse (line 34) | def parse(self, response):
FILE: 003-Keywords/amazon/spiders/spiders.py
class CheckIpSpider (line 16) | class CheckIpSpider(scrapy.Spider):
method start_requests (line 27) | def start_requests(self):
method parse (line 38) | def parse(self, response):
class AmazonSpider (line 45) | class AmazonSpider(scrapy.Spider):
method start_requests (line 59) | def start_requests(self):
method parse (line 70) | def parse(self, response):
class EtsySpider (line 79) | class EtsySpider(scrapy.Spider):
method start_requests (line 91) | def start_requests(self):
method parse (line 102) | def parse(self, response):
class LakesideSpider (line 114) | class LakesideSpider(scrapy.Spider):
method start_requests (line 126) | def start_requests(self):
method parse (line 137) | def parse(self, response):
class WordtrackerSpider (line 149) | class WordtrackerSpider(scrapy.Spider):
method start_requests (line 161) | def start_requests(self):
method parse (line 172) | def parse(self, response):
FILE: 003-Keywords/google.py
class GoogleTrend (line 17) | class GoogleTrend(object):
method __init__ (line 18) | def __init__(self):
method search (line 22) | def search(self, keyword, path, hl='en-US', proxies=False, retries=2, ...
method __sub_search (line 79) | def __sub_search(self, i_row, worksheet, keyword, path, csv_file, time...
method __search_related_queries (line 111) | def __search_related_queries(self, keyword, timeframe, tr: TrendReq) -...
method __search_trends (line 129) | def __search_trends(self, i_row, worksheet, path, keyword, timeframe, ...
method __draw_histogram (line 169) | def __draw_histogram(self, x: [], y: [], path, title, x_name='date', y...
method __draw_graph (line 187) | def __draw_graph(self, x: [], y: [], path, title, x_name='date', y_nam...
method __save_line (line 209) | def __save_line(self, path, line, mode='a'):
method __get_req (line 215) | def __get_req(self, hl='en-US', proxies=False, retries=2) -> TrendReq:
FILE: 003-Keywords/main.py
function Write_Img (line 21) | def Write_Img():
function read_xlsl (line 31) | def read_xlsl():
FILE: 003-Keywords/mypytrends/dailydata.py
function get_last_date_of_month (line 12) | def get_last_date_of_month(year: int, month: int) -> date:
function convert_dates_to_timeframe (line 21) | def convert_dates_to_timeframe(start: date, stop: date) -> str:
function _fetch_data (line 29) | def _fetch_data(pytrends, build_payload, timeframe: str) -> pd.DataFrame:
function get_daily_data (line 48) | def get_daily_data(word: str,
FILE: 003-Keywords/mypytrends/exceptions.py
class ResponseError (line 1) | class ResponseError(Exception):
method __init__ (line 4) | def __init__(self, message, response):
FILE: 003-Keywords/mypytrends/request.py
class TrendReq (line 18) | class TrendReq(object):
method __init__ (line 35) | def __init__(self, hl='en-US', tz=360, geo='', timeout=(2, 5), proxies...
method GetGoogleCookie (line 62) | def GetGoogleCookie(self):
method GetNewProxy (line 100) | def GetNewProxy(self):
method _get_data (line 109) | def _get_data(self, url, method=GET_METHOD, trim_chars=0, **kwargs):
method build_payload (line 161) | def build_payload(self, kw_list, cat=0, timeframe='today 5-y', geo='',
method _tokens (line 185) | def _tokens(self):
method interest_over_time (line 214) | def interest_over_time(self):
method interest_by_region (line 267) | def interest_by_region(self, resolution='COUNTRY', inc_low_vol=False,
method related_topics (line 316) | def related_topics(self):
method related_queries (line 365) | def related_queries(self):
method trending_searches (line 412) | def trending_searches(self, pn='united_states'):
method today_searches (line 425) | def today_searches(self, pn='US'):
method top_charts (line 443) | def top_charts(self, date, hl='en-US', tz=300, geo='GLOBAL'):
method suggestions (line 470) | def suggestions(self, keyword):
method categories (line 486) | def categories(self):
method get_historical_interest (line 500) | def get_historical_interest(self, keywords, year_start=2018, month_sta...
FILE: 003-Keywords/mypytrends/test_trendReq.py
class TestTrendReq (line 7) | class TestTrendReq(TestCase):
method test__get_data (line 9) | def test__get_data(self):
method test_build_payload (line 17) | def test_build_payload(self):
method test__tokens (line 23) | def test__tokens(self):
method test_interest_over_time (line 28) | def test_interest_over_time(self):
method test_interest_over_time_images (line 33) | def test_interest_over_time_images(self):
method test_interest_over_time_news (line 38) | def test_interest_over_time_news(self):
method test_interest_over_time_youtube (line 43) | def test_interest_over_time_youtube(self):
method test_interest_over_time_froogle (line 48) | def test_interest_over_time_froogle(self):
method test_interest_over_time_bad_gprop (line 53) | def test_interest_over_time_bad_gprop(self):
method test_interest_by_region (line 58) | def test_interest_by_region(self):
method test_related_topics (line 63) | def test_related_topics(self):
method test_related_queries (line 68) | def test_related_queries(self):
method test_trending_searches (line 73) | def test_trending_searches(self):
method test_top_charts (line 78) | def test_top_charts(self):
method test_suggestions (line 83) | def test_suggestions(self):
method test_ispartial_dtype (line 88) | def test_ispartial_dtype(self):
method test_ispartial_dtype_timeframe_all (line 94) | def test_ispartial_dtype_timeframe_all(self):
FILE: 003-Keywords/run_api.py
function check_ip (line 22) | def check_ip():
function crawl_amazon (line 29) | def crawl_amazon(keywords: []):
function crawl_alibaba (line 37) | def crawl_alibaba(keywords: []):
FILE: 003-Keywords/v2ray_util.py
function search_node (line 15) | def search_node():
function restart_v2ray (line 33) | def restart_v2ray(isSysOn=False):
function kill_all_v2ray (line 48) | def kill_all_v2ray():
FILE: 004-EmailNotify/main.py
class Email (line 22) | class Email(object):
method __init__ (line 23) | def __init__(self, from_user, pwd, to_other):
method sendMsg (line 33) | def sendMsg(self, title="", content=""):
class Coin (line 56) | class Coin(object):
method __init__ (line 57) | def __init__(self, coin="", diff=3.0, sleep=3, reload=1 * 60 * 60, ema...
method start (line 73) | def start(self):
FILE: 005-PaidSource/chrome.py
function get_cookie (line 27) | async def get_cookie(page):
function main (line 41) | async def main():
class Book118 (line 81) | class Book118(Net):
method recycling (line 85) | def recycling(headers, aids, title):
method recycling_name (line 107) | def recycling_name(headers, aids):
method load_page (line 123) | def load_page(url):
method parse_page (line 133) | def parse_page(content):
FILE: 005-PaidSource/ff_video.py
function cute_video (line 12) | def cute_video(folder):
FILE: 005-PaidSource/file_util.py
function file_name (line 22) | def file_name(file_dir):
function deal_one_page (line 32) | def deal_one_page():
function copy_doc (line 51) | def copy_doc():
function html_cover_doc (line 74) | def html_cover_doc(in_path, out_path):
function svg_cover_jpg (line 82) | def svg_cover_jpg(src, dst):
function html_cover_excel (line 97) | def html_cover_excel(content, out_path):
function write_to_html (line 109) | def write_to_html(content, file_path):
function write_json (line 123) | def write_json(content, file_path):
function read_json (line 133) | def read_json(file_path):
function write (line 141) | def write(content, file_path):
function read (line 151) | def read(file_path) -> str:
function get_next_folder (line 163) | def get_next_folder(dst, day_diff, folder, max_size):
FILE: 005-PaidSource/gsearch.py
class GSearch (line 23) | class GSearch(Net):
method search_page (line 24) | def search_page(self, url, pause=3):
method parse_html (line 51) | def parse_html(self, html):
method get_html (line 77) | def get_html(self, url):
method conver_to_doc (line 83) | def conver_to_doc(self, in_name, out_name):
method download_and_merge_page (line 90) | def download_and_merge_page(self, urls, name):
method get_full_urls (line 125) | def get_full_urls(self, html):
method get_full_titles (line 156) | def get_full_titles(self, html):
method format_common_url (line 175) | def format_common_url(self, search, domain='www.google.com', start=0):
method format_full_url (line 180) | def format_full_url(self, domain, as_q='', as_epq='', as_oq='', as_eq=...
FILE: 005-PaidSource/gtransfer.py
class GTransfer (line 27) | class GTransfer(Net):
method search_page (line 28) | def search_page(self, url, pause=3):
method transfer (line 52) | def transfer(self, content):
method init_param (line 78) | def init_param(self, file_name):
FILE: 005-PaidSource/kaoqin.py
function get_days (line 13) | def get_days(year, month): # 获取输出日期的列明
function parse_excel (line 25) | def parse_excel(csv_file, out_file, names, dates):
FILE: 005-PaidSource/keywords.py
class Keywords (line 16) | class Keywords(Net):
method get_keys_by_net (line 19) | def get_keys_by_net(self) -> []:
method get_keys_by_local (line 35) | def get_keys_by_local(self) -> []:
method get_titles_by_local (line 41) | def get_titles_by_local(self) -> []:
method get_titles_by_net (line 47) | def get_titles_by_net(self, key):
function test (line 75) | def test():
FILE: 005-PaidSource/main.py
function start_task (line 20) | def start_task():
function start_proxy_task (line 77) | def start_proxy_task():
function start_task2 (line 86) | def start_task2():
function test_titles (line 138) | def test_titles():
function test_task (line 144) | def test_task():
function test_get_title (line 178) | def test_get_title():
FILE: 005-PaidSource/other_site.py
class Cncic (line 22) | class Cncic(Net):
method start_task (line 25) | def start_task(self):
method load_list (line 74) | def load_list(self, cat, paged=1) -> []:
method load_page (line 88) | def load_page(self, url):
method parse_page (line 101) | def parse_page(self, page):
method parse_list (line 112) | def parse_list(self, page):
class Ceicdata (line 127) | class Ceicdata(Net):
method start_task_1 (line 130) | def start_task_1(self):
method start_task2 (line 166) | def start_task2():
method load_page (line 214) | def load_page(self, url):
method parse_main_1 (line 228) | def parse_main_1(self, page):
method parse_main_2 (line 241) | def parse_main_2(self, page):
method parse_page_1 (line 254) | def parse_page_1(self, page):
method parse_page_2 (line 265) | def parse_page_2(self, page):
class Cnnic (line 311) | class Cnnic(Net):
method start_task (line 315) | def start_task():
method load_page (line 342) | def load_page(self, url):
method parse_page (line 359) | def parse_page(self, page):
method download (line 376) | def download(self, url, path):
class Othersite (line 395) | class Othersite(Net):
method __init__ (line 396) | def __init__(self):
method start_task (line 401) | def start_task():
method load_page (line 493) | def load_page(self, url):
method parse_home (line 508) | def parse_home(self, page):
method parse_list (line 516) | def parse_list(self, page):
method parse_details (line 531) | def parse_details(self, page):
method download_img (line 558) | def download_img(self, src, sku, i):
FILE: 005-PaidSource/v2ray_util.py
function search_node (line 15) | def search_node():
function restart_v2ray (line 33) | def restart_v2ray(isSysOn=False):
function kill_all_v2ray (line 48) | def kill_all_v2ray():
FILE: 006-TikTok/dy_review.py
function review_douyin (line 18) | def review_douyin(): # 评论
FILE: 006-TikTok/file_util.py
function file_name (line 22) | def file_name(file_dir):
function deal_one_page (line 32) | def deal_one_page():
function copy_doc (line 51) | def copy_doc():
function html_cover_doc (line 74) | def html_cover_doc(in_path, out_path):
function svg_cover_jpg (line 82) | def svg_cover_jpg(src, dst):
function html_cover_excel (line 97) | def html_cover_excel(content, out_path):
function write_to_html (line 109) | def write_to_html(content, file_path):
function write_json (line 123) | def write_json(content, file_path):
function read_json (line 133) | def read_json(file_path):
function write (line 141) | def write(content, file_path):
function read (line 151) | def read(file_path) -> str:
function get_next_folder (line 163) | def get_next_folder(dst, day_diff, folder, max_size):
FILE: 006-TikTok/google_transfer_by_excel.py
class GTransfer (line 24) | class GTransfer(object):
method __init__ (line 25) | def __init__(self, file):
method request (line 32) | def request(self, url, allow_redirects=False, verify=False, proxies=No...
method search_page (line 42) | def search_page(self, url, pause=3):
method transfer (line 66) | def transfer(self, content):
method transfer_list (line 91) | def transfer_list(self, lists):
FILE: 006-TikTok/img_2_webp.py
function get_contain_pics (line 14) | def get_contain_pics(folder, content):
function convert_pic_to_webp (line 26) | def convert_pic_to_webp(folder, sku, pic_files, t, key1, key2, key3, you...
function deal_sku (line 112) | def deal_sku(folder, sku, key1, key2, key3, youtube_num):
function deal_sku_list (line 120) | def deal_sku_list():
function convert_2_webp (line 125) | def convert_2_webp(src, dst):
function convert_folder (line 137) | def convert_folder(src_folder, dst_folder):
FILE: 006-TikTok/post_autotk.py
function get_real_files (line 16) | def get_real_files(folder):
function format_num (line 33) | def format_num(num):
function split_en_title (line 42) | def split_en_title(spl, content):
function chose_cover_title (line 56) | def chose_cover_title(transfer: str): # 截取合适的标题和封面内容
function match_info (line 77) | def match_info(pre, files, transfers):
function format_time_by_str (line 99) | def format_time_by_str(date): # str转换为时间戳
function format_time_by_stamp (line 104) | def format_time_by_stamp(stamp): # 时间戳转换为时间
function random_tag (line 108) | def random_tag(tags):
function write_title_one (line 125) | def write_title_one(file_name, folder):
function write_title (line 134) | def write_title():
function read_title_one (line 139) | def read_title_one(file_name):
function del_content (line 151) | def del_content(content, dels):
function is_number (line 157) | def is_number(s):
function rename_files (line 166) | def rename_files(src_folder): # 重新生成序列
function start_post (line 187) | def start_post(userId, src_folder, date_start, title_tags, transfers, mu...
function all_step (line 244) | def all_step():
FILE: 006-TikTok/tikstar.py
function parse_tags (line 11) | def parse_tags(page):
FILE: 006-TikTok/tt_review.py
function start_vpn (line 70) | def start_vpn(): # 启动代理app
function review_forYou (line 85) | def review_forYou():
function review_tiktok (line 102) | def review_tiktok(): # 评论
function comment (line 139) | def comment(content):
function comment_foryou (line 167) | def comment_foryou(content):
function print_t (line 194) | def print_t(content):
FILE: 007-CutVideoAudio/editors.py
class Editors (line 21) | class Editors(object):
method __init__ (line 33) | def __init__(self):
method print_hint (line 38) | def print_hint():
method start (line 53) | def start(self, ffmpeg, video, music, dst):
method print_ui (line 58) | def print_ui(txt):
method print_all_ui (line 63) | def print_all_ui(txt, print_type: PrintType = PrintType.log):
method get_beijing_time (line 69) | def get_beijing_time():
method is_expired (line 93) | def is_expired():
method add_total_count (line 102) | def add_total_count(count=1):
method add_bgm_count (line 110) | def add_bgm_count(count=1):
method get_total_count (line 118) | def get_total_count():
method get_bgm_count (line 123) | def get_bgm_count():
method add_success_count (line 128) | def add_success_count():
method get_success_count (line 137) | def get_success_count():
method add_failed_count (line 142) | def add_failed_count():
method get_failed_count (line 151) | def get_failed_count():
FILE: 007-CutVideoAudio/ff_cut.py
class ListCut (line 19) | class ListCut(Editors):
method __init__ (line 21) | def __init__(self):
method start (line 27) | def start(self, ffmpeg, video, music, dst):
FILE: 007-CutVideoAudio/ff_util.py
function get_duration (line 12) | def get_duration(ff_file, src):
function format_h_m_s (line 30) | def format_h_m_s(t):
function format_ms (line 37) | def format_ms(t):
function format_duration_by_ms (line 46) | def format_duration_by_ms(ms):
function cut_audio (line 59) | def cut_audio(ff_file, src, start, dur, dst):
function cut_video (line 75) | def cut_video(ff_file, src, start, dur, fps, bit, dst):
function muxer_va (line 94) | def muxer_va(ff_file, src_v, src_a, dst):
FILE: 007-CutVideoAudio/ff_util_v2.py
function get_va_infos (line 15) | def get_va_infos(ff_file, src):
function get_duration (line 44) | def get_duration(ff_file, src):
function format_h_m_s (line 63) | def format_h_m_s(t):
function format_ms (line 67) | def format_ms(t):
function format_to_time (line 73) | def format_to_time(ms):
function format_to_ms (line 88) | def format_to_ms(duration: str):
function srt_to_ass (line 108) | def srt_to_ass(ff_file, src, dst):
function cut_with_subtitle (line 112) | def cut_with_subtitle(ff_file, src, dst, srt, width, height, margin_v, f...
function cut_va_full (line 153) | def cut_va_full(ff_file, src, dst, dur: str = None, start='00:00:00.000'...
function cut_va_tail (line 188) | def cut_va_tail(ff_file, src, dst, dur_full: str = None, start='00:00:00...
function cut_audio (line 198) | def cut_audio(ff_file, src, start, dur, dst):
function cut_video (line 214) | def cut_video(ff_file, src, start, dur, fps, bit, dst):
function cut_va_dur (line 233) | def cut_va_dur(ff_file, src, dst, start=0, dur=0, fps=None, bit=None):
function cut_va_start_end (line 264) | def cut_va_start_end(ff_file, src, dst, start='00:00:00', end='00:00:00'...
function cut_va_end (line 272) | def cut_va_end(ff_file, src, dst, start=0, end=0, fps=None, bit=None):
function muxer_va (line 292) | def muxer_va(ff_file, src_v, src_a, dst):
FILE: 007-CutVideoAudio/type_enum.py
class PrintType (line 13) | class PrintType(Enum):
FILE: 007-CutVideoAudio/ui.py
class Ui (line 21) | class Ui(Frame):
method __init__ (line 22) | def __init__(self, master=None):
method window_init (line 31) | def window_init(self):
method createWidgets (line 40) | def createWidgets(self):
method save_dir (line 132) | def save_dir(self):
method save_file (line 136) | def save_file(self):
method set_dir (line 140) | def set_dir(self, ffmpeg=None, video=None, music=None, dst=None):
method download_url (line 154) | def download_url(self):
method output (line 159) | def output(self, txt):
method func_ui_print (line 162) | def func_ui_print(self, txt, print_type: PrintType = None):
method start_download (line 174) | def start_download(self):
FILE: 007-CutVideoAudio/utils.py
function get_domain (line 15) | def get_domain(url: str = None):
function get_real_files (line 25) | def get_real_files(folder):
function format_num (line 40) | def format_num(num):
class Config (line 49) | class Config(object):
method __init__ (line 55) | def __init__(self):
method instance (line 62) | def instance(cls, *args, **kwargs):
method get_expired_time (line 68) | def get_expired_time(self):
method get_version_name (line 71) | def get_version_name(self):
method get_version_code (line 74) | def get_version_code(self):
FILE: 008-ChatGPT-UI/gpt.py
class Gpt (line 16) | class Gpt(object):
method __init__ (line 19) | def __init__(self, config: Config):
method update_config (line 30) | def update_config(self, config: Config):
method start (line 43) | def start(self):
method print (line 55) | def print(self, content):
method query_openai_stream (line 58) | def query_openai_stream(self, data: dict) -> str:
method content_change (line 84) | def content_change(self, content: str):
method handle_input (line 91) | def handle_input(self, content: str):
method query_openai (line 124) | def query_openai(self, data: dict) -> str:
FILE: 008-ChatGPT-UI/main.py
class EntryWithPlaceholder (line 17) | class EntryWithPlaceholder(tk.Entry):
method __init__ (line 18) | def __init__(self, master=None, placeholder='', **kwargs):
method put_placeholder (line 27) | def put_placeholder(self):
method remove_placeholder (line 31) | def remove_placeholder(self):
method on_focus_in (line 37) | def on_focus_in(self, event):
method on_focus_out (line 40) | def on_focus_out(self, event):
class Application (line 45) | class Application(tk.Frame):
method __init__ (line 46) | def __init__(self, config: Config, master=None):
method create_config (line 56) | def create_config(self):
method create_folder (line 66) | def create_folder(self):
method create_key (line 76) | def create_key(self):
method create_model (line 86) | def create_model(self):
method create_proxy (line 96) | def create_proxy(self):
method create_send (line 106) | def create_send(self):
method create_widgets (line 116) | def create_widgets(self):
method refresh (line 150) | def refresh(self):
method func_ui_print (line 157) | def func_ui_print(self, txt):
method click_config (line 160) | def click_config(self):
method click_create (line 168) | def click_create(self):
method click_folder (line 172) | def click_folder(self):
method set_entry (line 176) | def set_entry(self, entry: tk.Entry, content):
method on_return_key (line 180) | def on_return_key(self, event):
method click_send (line 183) | def click_send(self):
method show_text (line 195) | def show_text(self, content):
method clear (line 199) | def clear(self):
method copy (line 202) | def copy(self):
FILE: 008-ChatGPT-UI/utils.py
function get_domain (line 19) | def get_domain(url: str = None):
class ConfigIni (line 24) | class ConfigIni(object):
method __init__ (line 27) | def __init__(self):
method instance (line 34) | def instance(cls, *args, **kwargs):
method get_expired_time (line 40) | def get_expired_time(self):
method get_version_name (line 43) | def get_version_name(self):
method get_version_code (line 46) | def get_version_code(self):
method get_title (line 49) | def get_title(self):
method get_email (line 52) | def get_email(self):
class Config (line 56) | class Config:
method __init__ (line 75) | def __init__(self, dir: str) -> None:
method load (line 85) | def load(self, file):
method get (line 100) | def get(self, key, default=None):
method click_create (line 103) | def click_create(self):
method write_json (line 117) | def write_json(self, content, file_path):
method update (line 125) | def update(self, path: str):
method update_by_content (line 133) | def update_by_content(self, key: str = None, model: str = None, folder...
FILE: 009-Translate/config.py
function is_zh_language (line 22) | def is_zh_language():
class IniConfig (line 31) | class IniConfig(object):
method __init__ (line 32) | def __init__(self):
method full (line 47) | def full(self, tag, name):
method main (line 50) | def main(self, name):
method settings (line 53) | def settings(self, name):
method loading (line 56) | def loading(self, name):
method language (line 59) | def language(self, name):
method config (line 62) | def config(self, name):
method translate (line 65) | def translate(self):
FILE: 009-Translate/core.py
class Language (line 14) | class Language(object):
method __init__ (line 16) | def __init__(self, id_name, full_name, zh_name=None, google=True, yand...
method is_enable (line 33) | def is_enable(self, translator: str):
class Translation (line 37) | class Translation(object):
method __init__ (line 38) | def __init__(self):
method set_to_lang (line 59) | def set_to_lang(self, lang):
method set_from_lang (line 63) | def set_from_lang(self, lang):
method check_select_language (line 67) | def check_select_language(self):
method get_translators (line 81) | def get_translators(self):
method get_languages (line 93) | def get_languages(self):
method choose_translator (line 104) | def choose_translator(self, tl):
method _search_id_name (line 107) | def _search_id_name(self, is_from=True):
method get_id_name (line 128) | def get_id_name(self, is_from=True):
method translate (line 139) | def translate(self, window, content, is_html=False, file_path: str = N...
FILE: 009-Translate/load_srt.py
class Tag (line 13) | class Tag:
method __init__ (line 14) | def __init__(self):
class Translator (line 20) | class Translator(object):
method __init__ (line 24) | def __init__(self):
method __callback (line 44) | def __callback(self, status, src, dst, msg=None):
method __support_file (line 55) | def __support_file(self, src: str):
method __deal_file (line 66) | def __deal_file(self):
method __parse_srt (line 98) | def __parse_srt(self, src: str):
method __parse_file (line 125) | def __parse_file(self, src: str):
method __merge_content (line 135) | def __merge_content(self, tags):
method __request_item (line 155) | def __request_item(self, content):
method __request_list (line 175) | def __request_list(self, contents):
method __merge_file (line 190) | def __merge_file(self, tags, items, dst):
method __translate (line 198) | def __translate(self):
method add_callback (line 223) | def add_callback(self, listener):
method translate_file (line 231) | def translate_file(self, src, dst=None, from_lang='zh', to_lang='en'):
FILE: 009-Translate/main.py
class MainWin (line 16) | class MainWin(object):
method __init__ (line 17) | def __init__(self):
method advanced_ui (line 21) | def advanced_ui(self, window):
method make_window (line 26) | def make_window(self):
method show (line 101) | def show(self):
FILE: 009-Translate/tran_test.py
function t_test (line 17) | def t_test(srt_source):
function translate_callback (line 22) | def translate_callback(status, **kwargs):
FILE: 009-Translate/ui.py
function loading_show (line 11) | def loading_show():
class LoadingWin (line 26) | class LoadingWin(object):
function settings_show (line 33) | def settings_show():
FILE: 009-Translate/utils.py
function get_cache (line 11) | def get_cache(key: str, default=None):
function save_cache (line 24) | def save_cache(key: str, value):
function read (line 34) | def read(file_path) -> str:
function write (line 46) | def write(content, file_path):
function get_theme (line 55) | def get_theme():
class Key (line 68) | class Key:
FILE: 010-YouTubeUpload/youtube.py
class YtbConst (line 30) | class YtbConst:
class YtbUploader (line 81) | class YtbUploader:
method __init__ (line 86) | def __init__(self) -> None:
method __login (line 113) | def __login(self):
method __clear_field (line 117) | def __clear_field(self, field):
method __write_in_field (line 127) | def __write_in_field(self, field, string, select_all=False):
method __get_video_id (line 136) | def __get_video_id(self) -> Optional[str]:
method upload (line 147) | def upload(self, browser, path, meta, thumbnail=None):
method exit (line 155) | def exit(self):
method __wait_loading (line 158) | def __wait_loading(self):
method __upload (line 173) | def __upload(self) -> Tuple[bool, Optional[str]]:
Condensed preview — 158 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (635K chars).
[
{
"path": ".gitignore",
"chars": 1846,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "001-Downloader/.gitignore",
"chars": 1792,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "001-Downloader/README.md",
"chars": 1088,
"preview": "# 资源下载器\n本项目主要通过网络上开源的项目聚合成了一个跨平台的下载工具，可批量下载抖音、快手视音频资源。下载地址：\n\nMacOS：[Downloader1.0.3.app](https://github.com/xhunmon/Pyth"
},
{
"path": "001-Downloader/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "001-Downloader/config.ini",
"chars": 118,
"preview": "# 常用配置模块\n[common]\n#软件使用截止日期\nexpired_time=2025/12/15 23:59:59\n\n#app的版本名称\nversion_name=1.0.4\n\n#app的版本号\nversion_code=1040"
},
{
"path": "001-Downloader/doc/mac-sh/main.spec",
"chars": 1263,
"preview": "# -*- mode: python ; coding: utf-8 -*-\n\n\nblock_cipher = None\n\n\na = Analysis(['main.py','type_enum.py','ui.py','utils.py'"
},
{
"path": "001-Downloader/doc/mac-sh/pyinstaller.sh",
"chars": 183,
"preview": "#!/bin/bash\n\npyinstaller -F -i res/logo.ico main.spec main.py -w \\\n-p type_enum.py \\\n-p ui.py \\\n-p utils.py \\\n-p downlo"
},
{
"path": "001-Downloader/doc/win-sh/main.spec",
"chars": 1268,
"preview": "# -*- mode: python ; coding: utf-8 -*-\n\n\nblock_cipher = None\n\n\na = Analysis(['main.py','type_enum.py','ui.py','utils.py'"
},
{
"path": "001-Downloader/doc/win-sh/pyinstaller.sh",
"chars": 175,
"preview": "#!/bin/bash\n\npyinstaller -F -i res\\\\logo.ico -w main.spec main.py\n-p type_enum.py\n-p ui.py\n-p utils.py\n-p downloader.p"
},
{
"path": "001-Downloader/douyin/dy_download.py",
"chars": 12020,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 抖音视频下载\n@Date :2021/08/14\n@Author :xhunmon\n@M"
},
{
"path": "001-Downloader/downloader.py",
"chars": 5319,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description:downloader.py 所有下载类的基类，负责与UI界面的绑定\n@Date :2021/08/"
},
{
"path": "001-Downloader/kuaishou/ks_download.py",
"chars": 14381,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 快手视频下载\n@Date :2021/09/102\n@Author :qincji\n@M"
},
{
"path": "001-Downloader/main.py",
"chars": 415,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 程序主入口\n@Date :2021/08/14\n@Author :xhunmon\n@Ma"
},
{
"path": "001-Downloader/test/bilibili_video_download_v1.py",
"chars": 8439,
"preview": "# !/usr/bin/python\n# -*- coding:utf-8 -*-\n# time: 2019/04/17--08:12\n__author__ = 'Henry'\n\n'''\n项目: B站视频下载\n\n版本1: 加密API版,不需"
},
{
"path": "001-Downloader/test/ff_video.py",
"chars": 1447,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: ffmpeg去掉最后一帧，改变md5\n@Date :2022/02/17\n@Author "
},
{
"path": "001-Downloader/test/test_pyinstaller.py",
"chars": 926,
"preview": "#!/usr/bin/env python\r\n# -*- coding:utf-8 -*-\r\nimport os\r\n# 测试打包\r\n# import pyppeteer\r\n# import sys\r\n# import asyncio\r\n# "
},
{
"path": "001-Downloader/test/urls.txt",
"chars": 0,
"preview": ""
},
{
"path": "001-Downloader/test/xhs_download.py",
"chars": 2648,
"preview": "import os\nimport random\nimport time\n\nimport requests\nfrom my_fake_useragent import UserAgent\n\nua = UserAgent(family='chr"
},
{
"path": "001-Downloader/type_enum.py",
"chars": 282,
"preview": "#!/usr/bin/env python\n# -*- coding:utf-8 -*-\n\"\"\"\n@Description:dy_download.py\n@Date :2021/08/14\n@Author :xhunmo"
},
{
"path": "001-Downloader/ui.py",
"chars": 6726,
"preview": "#!/usr/bin/env python\r\n# -*- coding:utf-8 -*-\r\n\"\"\"\r\n@Description: 用于GUI界面显示\r\n@Date :2021/08/14\r\n@Author :xhunm"
},
{
"path": "001-Downloader/utils.py",
"chars": 1241,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description:工具类\n@Date :2021/08/16\n@Author :xhunmon\n@Mail "
},
{
"path": "002-V2rayPool/.gitignore",
"chars": 1799,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "002-V2rayPool/002-V2rayPool.iml",
"chars": 338,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<module type=\"PYTHON_MODULE\" version=\"4\">\n <component name=\"NewModuleRootManager"
},
{
"path": "002-V2rayPool/README.md",
"chars": 2354,
"preview": "# v2ray节点代理池\n学习python爬虫过程中，我们需要一些代理。而本项目则是通过收集网上公开的节点，来实现自己的代理请求的过程！\n\n## v2ray是什么？如何使用？\n- [v2ray官方指导](https://www.v2ray."
},
{
"path": "002-V2rayPool/base/net_proxy.py",
"chars": 2728,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 网络代理请求的基类\n@Date :2021/09/15\n@Author :xhunmon"
},
{
"path": "002-V2rayPool/core/client.py",
"chars": 10981,
"preview": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\nimport base64\nimport json\nimport os\nimport urllib\n\nfrom core.conf import"
},
{
"path": "002-V2rayPool/core/conf.py",
"chars": 1306,
"preview": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\nimport configparser\n\nimport os\n\n\nclass Config:\n __v2ray_core_path = No"
},
{
"path": "002-V2rayPool/core/group.py",
"chars": 8235,
"preview": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\nimport base64\nimport json\nfrom urllib.parse import quote\n\n__author__ = 'q"
},
{
"path": "002-V2rayPool/core/json_template/client.json",
"chars": 1792,
"preview": "{\n \"log\": {\n \"access\": \"\",\n \"error\": \"\",\n \"loglevel\": \"info\"\n },\n \"inbounds\": [\n {\n \"port\": 1080,\n "
},
{
"path": "002-V2rayPool/core/json_template/client_socks.json",
"chars": 1724,
"preview": "{\n \"log\": {\n \"access\": \"\",\n \"error\": \"\",\n \"loglevel\": \"info\"\n },\n \"inbounds\": [\n {\n \"port\": 1080,\n "
},
{
"path": "002-V2rayPool/core/json_template/client_ss.json",
"chars": 1300,
"preview": "{\n \"log\": {\n \"access\": \"\",\n \"error\": \"\",\n \"loglevel\": \"info\"\n },\n \"inbounds\": [\n {\n \"port\": 1080,\n "
},
{
"path": "002-V2rayPool/core/json_template/client_trojan.json",
"chars": 1328,
"preview": "{\n \"log\": {\n \"access\": \"\",\n \"error\": \"\",\n \"loglevel\": \"info\"\n },\n \"inbounds\": [\n {\n \"port\": 1080,\n "
},
{
"path": "002-V2rayPool/core/json_template/dyn_port.json",
"chars": 231,
"preview": "{\n \"protocol\": \"vmess\",\n \"port\": \"10000-20000\",\n \"tag\": \"dynamicPort\", \n \"settings\": {\n \"default\": {\n "
},
{
"path": "002-V2rayPool/core/json_template/http.json",
"chars": 1312,
"preview": "{\n \"network\": \"tcp\",\n \"security\": \"none\",\n \"tlsSettings\": {},\n \"httpSettings\": {},\n \"tcpSettings\": {\n \"header\": "
},
{
"path": "002-V2rayPool/core/json_template/http2.json",
"chars": 193,
"preview": "{\n \"network\": \"h2\",\n \"security\": \"tls\",\n \"tlsSettings\": {},\n \"tcpSettings\": {},\n \"httpSettings\": { \n \"path\": \"/r"
},
{
"path": "002-V2rayPool/core/json_template/kcp.json",
"chars": 381,
"preview": "{\n \"network\": \"kcp\",\n \"security\": \"none\",\n \"tlsSettings\": {},\n \"tcpSettings\": {},\n \"httpSettings\": {},\n \"kcpSettin"
},
{
"path": "002-V2rayPool/core/json_template/mtproto.json",
"chars": 373,
"preview": "{\n \"mtproto-in\": {\n \"tag\": \"tg-in\",\n \"port\": 5829,\n \"protocol\": \"mtproto\",\n \"settings\": {\n \"users\": [{"
},
{
"path": "002-V2rayPool/core/json_template/quic.json",
"chars": 257,
"preview": "{\n \"network\": \"quic\",\n \"security\": \"none\",\n \"tlsSettings\": {},\n \"tcpSettings\": {},\n \"kcpSettings\": {},\n \"httpSetti"
},
{
"path": "002-V2rayPool/core/json_template/server.json",
"chars": 1236,
"preview": "{\n \"log\": {\n \"access\": \"/var/log/v2ray/access.log\",\n \"error\": \"/var/log/v2ray/error.log\",\n \"loglevel\": \"info\"\n"
},
{
"path": "002-V2rayPool/core/json_template/socks.json",
"chars": 117,
"preview": "{\n \"auth\": \"password\",\n \"accounts\": [\n {\n \"user\": \"hello\",\n \"pass\": \"socks\"\n }\n ],\n \"udp\": true\n}"
},
{
"path": "002-V2rayPool/core/json_template/ss.json",
"chars": 149,
"preview": "{\n \"port\": 1024,\n \"protocol\": \"shadowsocks\",\n \"settings\": {\n \"method\": \"aes-128-gcm\",\n \"password\": \"password\",\n"
},
{
"path": "002-V2rayPool/core/json_template/stats_settings.json",
"chars": 602,
"preview": "{\n \"stats\": {},\n \"api\": {\n \"services\": [\n \"StatsService\"\n ],\n \"tag\": \"api\"\n },\n \"policy\": {\n \"level"
},
{
"path": "002-V2rayPool/core/json_template/tcp.json",
"chars": 193,
"preview": "{\n \"network\": \"tcp\",\n \"security\": \"none\",\n \"tlsSettings\": {},\n \"tcpSettings\": {},\n \"kcpSettings\": {},\n \"wsSettings"
},
{
"path": "002-V2rayPool/core/json_template/vless.json",
"chars": 160,
"preview": "{\n \"clients\": [\n {\n \"id\": \"d4e321ea-e118-11ea-a265-42010a8c0002\"\n }\n ],\n \"decryption\": \"none\",\n \"fallback"
},
{
"path": "002-V2rayPool/core/json_template/ws.json",
"chars": 229,
"preview": "{\n \"network\": \"ws\",\n \"security\": \"none\",\n \"tlsSettings\": {},\n \"tcpSettings\": {},\n \"kcpSettings\": {},\n \"httpSetting"
},
{
"path": "002-V2rayPool/core/profile.py",
"chars": 7020,
"preview": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\nimport json\nimport os\n\nfrom conf import Config\nfrom group import SS, Sock"
},
{
"path": "002-V2rayPool/core/utils.py",
"chars": 5794,
"preview": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\nimport random\nimport re\nimport os\nimport socket\nimport string\nimport sys\n"
},
{
"path": "002-V2rayPool/db/db_main.py",
"chars": 9249,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 数据相关接口封装\n@Date :2021/08/30\n@Author :xhunmon\n"
},
{
"path": "002-V2rayPool/db/local.py",
"chars": 10884,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 数据库相关类\n@Date :2021/08/25\n@Author :xhunmon\n@M"
},
{
"path": "002-V2rayPool/db/net.py",
"chars": 9421,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 从网络获取\n@Date :2021/08/30\n@Author :xhunmon\n@Ma"
},
{
"path": "002-V2rayPool/doc/(参考用)config.json",
"chars": 1136,
"preview": "{\n \"log\": {\n \"access\": \"\",\n \"error\": \"\",\n \"loglevel\": \"info\"\n },\n \"inbounds\": [\n {\n \"port\": \"1080\",\n \"listen\": \""
},
{
"path": "002-V2rayPool/test_main.py",
"chars": 1363,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 主要入口\n@Date :2021/08/25\n@Author :xhunmon\n@Mai"
},
{
"path": "003-Keywords/.gitignore",
"chars": 1799,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "003-Keywords/README.md",
"chars": 499,
"preview": "# 通过google trends查找相关关键词，并且生成趋势\n\n## 效果\n\n例子：通过`women ring`关键词查找出有1千个相关关键词：[women ring.csv](women-ring/women ring.csv)\n\n!["
},
{
"path": "003-Keywords/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "003-Keywords/amazon/items.py",
"chars": 262,
"preview": "# Define here the models for your scraped items\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics/ite"
},
{
"path": "003-Keywords/amazon/middlewares.py",
"chars": 4899,
"preview": "# Define here the models for your spider middleware\n#\n# See documentation in:\n# https://docs.scrapy.org/en/latest/topics"
},
{
"path": "003-Keywords/amazon/pipelines.py",
"chars": 360,
"preview": "# Define your item pipelines here\n#\n# Don't forget to add your pipeline to the ITEM_PIPELINES setting\n# See: https://doc"
},
{
"path": "003-Keywords/amazon/settings.py",
"chars": 3114,
"preview": "# Scrapy settings for amazon project\n#\n# For simplicity, this file contains only settings considered important or\n# comm"
},
{
"path": "003-Keywords/amazon/spiders/__init__.py",
"chars": 161,
"preview": "# This package will contain the spiders of your Scrapy project\n#\n# Please refer to the documentation for information on "
},
{
"path": "003-Keywords/amazon/spiders/alibaba.py",
"chars": 1165,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 亚马逊相关关键词获取\n@Date :2021/09/24\n@Author :xhunmo"
},
{
"path": "003-Keywords/amazon/spiders/amazon.py",
"chars": 1254,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 亚马逊相关关键词获取\n@Date :2021/09/24\n@Author :xhunmo"
},
{
"path": "003-Keywords/amazon/spiders/checkip.py",
"chars": 1005,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 检查当前代理是否起作用\n@Date :2021/09/24\n@Author :xhunm"
},
{
"path": "003-Keywords/amazon/spiders/spiders.py",
"chars": 5602,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 爬虫页面集合\n@Date :2021/09/26\n@Author :xhunmon\n@M"
},
{
"path": "003-Keywords/google.py",
"chars": 8588,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: google相关获取\n@Date :2021/10/08\n@Author :xhunmo"
},
{
"path": "003-Keywords/main.py",
"chars": 1287,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 关键词获取\n@Date :2021/09/22\n@Author :xhunmon\n@Ma"
},
{
"path": "003-Keywords/mypytrends/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "003-Keywords/mypytrends/dailydata.py",
"chars": 5312,
"preview": "from datetime import date, timedelta\nfrom functools import partial\nfrom time import sleep\nfrom calendar import monthrang"
},
{
"path": "003-Keywords/mypytrends/exceptions.py",
"chars": 273,
"preview": "class ResponseError(Exception):\n \"\"\"Something was wrong with the response from Google\"\"\"\n\n def __init__(self, mess"
},
{
"path": "003-Keywords/mypytrends/request.py",
"chars": 23248,
"preview": "import json\nimport sys\nimport time\nfrom datetime import datetime, timedelta\n\nimport pandas as pd\nimport requests\n\nfrom p"
},
{
"path": "003-Keywords/mypytrends/test_trendReq.py",
"chars": 3649,
"preview": "from unittest import TestCase\nimport pandas.api.types as ptypes\n\nfrom mypytrends.request import TrendReq\n\n\nclass TestTre"
},
{
"path": "003-Keywords/run_api.py",
"chars": 964,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 通过框架自带命令 启动命令脚本\n@Date :2021/09/20\n@Author :x"
},
{
"path": "003-Keywords/scrapy.cfg",
"chars": 255,
"preview": "# Automatically created by: scrapy startproject\n#\n# For more information about the [deploy] section see:\n# https://scrap"
},
{
"path": "003-Keywords/v2ray_pool/__init__.py",
"chars": 283,
"preview": "# 运行时路径。并非__init__.py的路径\nimport os\nimport sys\n\nBASE_DIR = \"../002-V2rayPool\"\nif os.path.exists(BASE_DIR):\n sys.path.a"
},
{
"path": "003-Keywords/v2ray_pool/_db-checked.txt",
"chars": 7773,
"preview": "ss://YWVzLTI1Ni1nY206ZzVNZUQ2RnQzQ1dsSklkQDE0Mi4yMDIuNDguMTA4OjUwMDM#%E7%BE%8E%E5%9B%BD-2.48MB/s,142.202.48.108,加拿大 魁北克"
},
{
"path": "003-Keywords/v2ray_pool/_db-uncheck.txt",
"chars": 3718,
"preview": "vmess://eyJ2IjogIjIiLCAicHMiOiAiZ2l0aHViLmNvbS9mcmVlZnEgLSBcdTViODlcdTVmYmRcdTc3MDFcdTgwNTRcdTkwMWEgMSIsICJhZGQiOiAiNDIu"
},
{
"path": "003-Keywords/v2ray_util.py",
"chars": 1625,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 管理v2ray_pool的工具\n@Date :2022/1/14\n@Author :xh"
},
{
"path": "003-Keywords/women-ring/women ring.csv",
"chars": 52040,
"preview": "keyword,no,top keyword,top range,rising keyword,rising range\r\nwomen ring,0,ring for women,100,are men and women ring siz"
},
{
"path": "004-EmailNotify/.gitignore",
"chars": 1799,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "004-EmailNotify/README.md",
"chars": 146,
"preview": "# 监听玩客家平台币价变化，使用邮件通知\n\n本小项目主要体现邮箱的作用，如果利用微信绑定收信邮箱，便能实时知道自己想要知道的信息。 注意：发送邮箱的登录密码一般是在设置smtp中生成的授权码。\n\n- 在服务器上启动脚本，与连接无关：\n```"
},
{
"path": "004-EmailNotify/main.py",
"chars": 5669,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 玩客家 币价变化，邮件通知 api\n@Date :2020/12/17\n@Author "
},
{
"path": "005-PaidSource/.gitignore",
"chars": 1799,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "005-PaidSource/005-PaidSource.iml",
"chars": 338,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<module type=\"PYTHON_MODULE\" version=\"4\">\n <component name=\"NewModuleRootManager"
},
{
"path": "005-PaidSource/README.md",
"chars": 749,
"preview": "# 这些脚本你肯定会有用到的\n\n### 操作已打开的chrome浏览器\n\n场景：某些情况我们获取怎么都获取不到cookie，但我们可以使用先在浏览器上登录，然后进行自动化操作。\n\n操作指南：\n\n```shell\n需要以该方式启动的浏览器：\n"
},
{
"path": "005-PaidSource/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "005-PaidSource/chrome.py",
"chars": 4918,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 用已经打开的chrome浏览器进行自动化操作。\n在某些应用场景我们获取怎么都获取不到cookie，但我们可以"
},
{
"path": "005-PaidSource/ff_video.py",
"chars": 1457,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 使用ffmpeg去掉最后一帧，改变md5。短视频搬运专用\n@Date :2022/02/17\n@"
},
{
"path": "005-PaidSource/file_util.py",
"chars": 4807,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 文件相关处理\n@Date :2022/01/22\n@Author :xhunmon\n@M"
},
{
"path": "005-PaidSource/gsearch.py",
"chars": 8462,
"preview": "import re\nimport time\nimport os\nfrom urllib.parse import quote_plus\n\nimport chardet\nimport requests_html\nimport pypandoc"
},
{
"path": "005-PaidSource/gtransfer.py",
"chars": 3842,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 实现从excel文件获取关键词进行翻译后写入新文件\n@Date :2021/10/22\n@Aut"
},
{
"path": "005-PaidSource/kaoqin.py",
"chars": 1975,
"preview": "\"\"\"\n@Description: excel表的常规操作，这里实现统计考勤\n@Date :2022/02/21\n@Author :xhunmon\n@Mail :xhunmon@gmail.com\n\"\"\"\n\n"
},
{
"path": "005-PaidSource/keywords.py",
"chars": 3000,
"preview": "import json\nimport random\nimport re\nimport time\nfrom urllib.parse import quote_plus\n\nfrom v2ray_pool import Net\n\nimport "
},
{
"path": "005-PaidSource/main.py",
"chars": 5565,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 关键词获取\n@Date :2021/09/22\n@Author :xhunmon\n@Ma"
},
{
"path": "005-PaidSource/other_site.py",
"chars": 21764,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 获取其他站点信息爬虫\n@Date :2022/1/14\n@Author :xhunmon"
},
{
"path": "005-PaidSource/v2ray_pool/__init__.py",
"chars": 283,
"preview": "# 运行时路径。并非__init__.py的路径\nimport os\nimport sys\n\nBASE_DIR = \"../002-V2rayPool\"\nif os.path.exists(BASE_DIR):\n sys.path.a"
},
{
"path": "005-PaidSource/v2ray_pool/_db-checked.txt",
"chars": 926,
"preview": "ss://YWVzLTI1Ni1nY206cEtFVzhKUEJ5VFZUTHRN@134.195.196.68:443#github.com/freefq%20-%20%E5%8C%97%E7%BE%8E%E5%9C%B0%E5%8C%B"
},
{
"path": "005-PaidSource/v2ray_pool/_db-uncheck.txt",
"chars": 0,
"preview": ""
},
{
"path": "005-PaidSource/v2ray_util.py",
"chars": 1638,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 管理v2ray_pool的工具\n@Date :2022/1/14\n@Author :xh"
},
{
"path": "006-TikTok/.gitignore",
"chars": 1799,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "006-TikTok/006-TikTok.iml",
"chars": 371,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<module type=\"PYTHON_MODULE\" version=\"4\">\n <component name=\"NewModuleRootManager"
},
{
"path": "006-TikTok/README.md",
"chars": 608,
"preview": "# App自动化\n\n## 效果\n\n抖音和tiktok能自动刷App评论。\n\n## 实现\n1. 使用android手机，能与电脑正常使用adb连接\n2. 使用[uiautomator2](https://github.com/openatx/"
},
{
"path": "006-TikTok/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "006-TikTok/dy_review.py",
"chars": 6445,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 抖音app刷评论\n@Date :2021/12/22\n@Author :xhunmon\n"
},
{
"path": "006-TikTok/file_util.py",
"chars": 4807,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 文件相关处理\n@Date :2022/01/22\n@Author :xhunmon\n@M"
},
{
"path": "006-TikTok/google_transfer_by_excel.py",
"chars": 3605,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 实现从excel文件获取关键词进行翻译后写入新文件\n@Date :2021/10/22\n@Aut"
},
{
"path": "006-TikTok/img_2_webp.py",
"chars": 8092,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: WordPress 站点 将图片转 webp 格式，实现内容重组\n@Date :2022/10/"
},
{
"path": "006-TikTok/main.py",
"chars": 294,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: tiktok 相关开源库\n@Date :2021/12/22\n@Author :xhun"
},
{
"path": "006-TikTok/post_autotk.py",
"chars": 7754,
"preview": "\"\"\"\n@Description: 生成autotk.app(https://github.com/xhunmon/autotk)中post发送内容解析，\n@Date :2022/3/20\n@Author :xhunmo"
},
{
"path": "006-TikTok/tikstar.py",
"chars": 866,
"preview": "\"\"\"\n@Description: 解析www.tikstar.com网站相关内容，获取tags\n@Date :2021/12/22\n@Author :xhunmon\n@Mail :xhunmon@gmail"
},
{
"path": "006-TikTok/tt_review.py",
"chars": 9276,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: tiktok app（版本：22.8.2）刷评论脚本\n@Date :2021/12/22\n@Au"
},
{
"path": "006-TikTok/v2ray_pool/__init__.py",
"chars": 220,
"preview": "# 运行时路径。并非__init__.py的路径\nimport os\nimport sys\n\nBASE_DIR = \"../002-V2rayPool\"\nif os.path.exists(BASE_DIR):\n sys.path.a"
},
{
"path": "007-CutVideoAudio/.gitignore",
"chars": 1792,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "007-CutVideoAudio/007-CutVideoAudio.iml",
"chars": 338,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<module type=\"PYTHON_MODULE\" version=\"4\">\n <component name=\"NewModuleRootManager"
},
{
"path": "007-CutVideoAudio/README.md",
"chars": 1059,
"preview": "# FFmpeg批量剪切音视频\n\n[V2版本脚本](ff_util_v2.py)更加完善，但是未接入GUI\n\n本项目主要通过ffmpeg工具进行批量视频剪辑，随机剪辑，从而躲过自媒体平台的检查，从而达到一份视频多个账号运营。\n\n使用前提：*"
},
{
"path": "007-CutVideoAudio/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "007-CutVideoAudio/config.ini",
"chars": 118,
"preview": "# 常用配置模块\n[common]\n#软件使用截止日期\nexpired_time=2025/12/15 23:59:59\n\n#app的版本名称\nversion_name=1.0.0\n\n#app的版本号\nversion_code=1000"
},
{
"path": "007-CutVideoAudio/doc/mac-sh/main.spec",
"chars": 1260,
"preview": "# -*- mode: python ; coding: utf-8 -*-\n\n\nblock_cipher = None\n\n\na = Analysis(['main.py','type_enum.py','ui.py','utils.py'"
},
{
"path": "007-CutVideoAudio/doc/mac-sh/pyinstaller.sh",
"chars": 155,
"preview": "#!/bin/bash\n\npyinstaller -F -i res/logo.ico main.spec main.py -w \\\n-p type_enum.py \\\n-p ui.py \\\n-p utils.py \\\n-p ff_uti"
},
{
"path": "007-CutVideoAudio/editors.py",
"chars": 4544,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description:editing.py 所有视频类的基类，负责与UI界面的绑定\n@Date :2022/03/14\n"
},
{
"path": "007-CutVideoAudio/ff_cut.py",
"chars": 3058,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description:editing.py 所有视频类的基类，负责与UI界面的绑定\n@Date :2022/03/14\n"
},
{
"path": "007-CutVideoAudio/ff_util.py",
"chars": 2568,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description:ff_util.py ffmpeg截切命令工具\n@Date :2022/03/14\n@Author"
},
{
"path": "007-CutVideoAudio/ff_util_v2.py",
"chars": 9938,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description:ff_util.py ffmpeg截切命令工具\n@Date :2022/03/14\n@Author"
},
{
"path": "007-CutVideoAudio/main.py",
"chars": 683,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: 程序主入口\n@Date :2022/03/14\n@Author :xhunmon\n@Ma"
},
{
"path": "007-CutVideoAudio/type_enum.py",
"chars": 274,
"preview": "#!/usr/bin/env python\n# -*- coding:utf-8 -*-\n\"\"\"\n@Description:dy_download.py\n@Date :2022/03/14\n@Author :xhunmo"
},
{
"path": "007-CutVideoAudio/ui.py",
"chars": 8296,
"preview": "#!/usr/bin/env python\r\n# -*- coding:utf-8 -*-\r\n\"\"\"\r\n@Description: 用于GUI界面显示\r\n@Date :2021/08/14\r\n@Author :xhunm"
},
{
"path": "007-CutVideoAudio/utils.py",
"chars": 1715,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description:工具类\n@Date :2021/08/16\n@Author :xhunmon\n@Mail "
},
{
"path": "008-ChatGPT-UI/.gitignore",
"chars": 1799,
"preview": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\n\n# C extensions\n*.so\n\n# Distribution / packagi"
},
{
"path": "008-ChatGPT-UI/README.md",
"chars": 188,
"preview": "\n# GPT-UI 2.0 移动☞: [iMedia](https://github.com/xhunmon/iMedia)\n\n- UI大优化\n- 支持图片生成\n\n演示：\n\nhttps://user-images.githubusercon"
},
{
"path": "008-ChatGPT-UI/config.ini",
"chars": 134,
"preview": "[common]\nexpired_time=2025/12/15 23:59:59\n\ntitle=GPT-UI\n\nversion_name=v1.0.1--github/xhunmon\n\nversion_code=1010\n\nemail=x"
},
{
"path": "008-ChatGPT-UI/config.json",
"chars": 222,
"preview": "{\n \"key\": \"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\",\n \"api_base\": \"\",\n \"model\": \"gpt-3.5-turbo\",\n \"strea"
},
{
"path": "008-ChatGPT-UI/doc/config.json",
"chars": 274,
"preview": "{\n \"api_base\": \"\",\n \"key\": \"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\",\n \"model\": \"gpt-3.5-turbo\",\n \"strea"
},
{
"path": "008-ChatGPT-UI/doc/pyinstaller.sh",
"chars": 298,
"preview": "#!/bin/bash\n\n\npyinstaller --windowed --name GPT-UI --add-data \"config.ini:.\" --icon logo.ico main.py gpt.py utils.py\n\n#"
},
{
"path": "008-ChatGPT-UI/gpt.py",
"chars": 4626,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description:gpt.py\n@Date :2023/03/31\n@Author :xhunmon\n@Ma"
},
{
"path": "008-ChatGPT-UI/main.py",
"chars": 8100,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: main\n@Date :2023/03/31\n@Author :xhunmon\n@Mai"
},
{
"path": "008-ChatGPT-UI/requirements.txt",
"chars": 30,
"preview": "openai\nrequests[socks]\ntkinter"
},
{
"path": "008-ChatGPT-UI/utils.py",
"chars": 4559,
"preview": "#!/usr/bin/env python\n# -*- encoding: utf-8 -*-\n\"\"\"\n@Description: tool\n@Date :2023/03/31\n@Author :xhunmon\n@Mai"
},
{
"path": "009-Translate/README.md",
"chars": 299,
"preview": "# 多平台免费翻译神器\n\n## 1. 输入框翻译\n\n直接在输入框输入内容，选择目标语言，翻译平台即可。\n\n\n## 2.文件翻译\n\n- 1.支持txt翻译，但是文本内容不宜过大\n\n- 2.支持html翻译\n\n- 3.定制版srt字幕文件翻译，"
},
{
"path": "009-Translate/asset/ch.ini",
"chars": 2112,
"preview": "[Main]\nTitle = Super86翻译\nDescription = 免费多平台多语言翻译...\nEnableProxy = 使用代理\nRun = 翻译\nClear = 清空\nCopy = 复制\nFile = 选择文件\nSettin"
},
{
"path": "009-Translate/asset/config.ini",
"chars": 28142,
"preview": "[Config]\nLoading = R0lGODlhoAAYAKEAALy+vOTm5P7+/gAAACH/C05FVFNDQVBFMi4wAwEAAAAh+QQJCQACACwAAAAAoAAYAAAC55SPqcvtD6OctNqLs"
},
{
"path": "009-Translate/asset/en.ini",
"chars": 2605,
"preview": "[Main]\nTitle = Super86 Translator\nDescription = Free multi-platform and multi-language translation software...\nEnablePro"
},
{
"path": "009-Translate/asset/language.json",
"chars": 21747,
"preview": "[\n {\n \"en_name\": \"Chinese(简体)\",\n \"id_name\": \"zh-CHS\",\n \"zh_name\": \"简体\",\n \"google\": \"zh-CN\",\n \"yandex\": \""
},
{
"path": "009-Translate/config.py",
"chars": 1704,
"preview": "# -*- coding: utf-8 -*-\n\"\"\"\n@Author : Xhunmon \n@Time : 2023/4/23 08:51\n@FileName: config.py\n@desc: \n\"\"\"\nimport confi"
},
{
"path": "009-Translate/core.py",
"chars": 8760,
"preview": "# -*- coding: utf-8 -*-\n\"\"\"\n@Author : Xhunmon \n@Time : 2023/4/22 22:29\n@FileName: core.py\n@desc: \n\"\"\"\nimport transla"
},
{
"path": "009-Translate/doc/pyinstaller.sh",
"chars": 969,
"preview": "#!/bin/bash\n\n\n#pyinstaller --windowed --name GPT-UI --add-data \"config.ini:.\" --icon logo.ico main.py gpt.py utils.py\n\n"
},
{
"path": "009-Translate/load_srt.py",
"chars": 7385,
"preview": "# -*- coding: utf-8 -*-\n\"\"\"\n@Author : Xhunmon \n@Time : 2023/4/18 14:47\n@FileName: load_srt.py\n@desc: 从本地加载srt字幕文件\n\"\""
},
{
"path": "009-Translate/main.py",
"chars": 8395,
"preview": "# -*- coding: utf-8 -*-\n\"\"\"\n@Author : Xhunmon \n@Time : 2023/4/21 23:03\n@FileName: guiv2.py\n@desc: \n\"\"\"\nimport PySimp"
},
{
"path": "009-Translate/tran_test.py",
"chars": 664,
"preview": "#!/usr/bin/python3.9\n# -*- coding: utf-8 -*-\n\"\"\"\n@Time : 2023/5/18 14:38\n@Author : xhunmon\n@Email : xhunmon@126.co"
},
{
"path": "009-Translate/ui.py",
"chars": 4305,
"preview": "# -*- coding: utf-8 -*-\n\"\"\"\n@Author : Xhunmon \n@Time : 2023/4/22 22:31\n@FileName: ui.py\n@desc: \n\"\"\"\nfrom config impo"
},
{
"path": "009-Translate/utils.py",
"chars": 1865,
"preview": "# -*- coding: utf-8 -*-\n\"\"\"\n@Author : Xhunmon \n@Time : 2023/4/22 21:54\n@FileName: utils.py\n@desc: \n\"\"\"\nimport PySimp"
},
{
"path": "010-YouTubeUpload/README.md",
"chars": 368,
"preview": "# YouTube uploader 自动上传\n\n因为使用API上传是有限制的，每天只能上传6条。因此得使用浏览器操作\n\n## 前提\n\n1. Chome浏览器，同时需要配置内核，可参考以下文章\n\n> selenium复用已打开浏览器：htt"
},
{
"path": "010-YouTubeUpload/main.py",
"chars": 1232,
"preview": "#!/usr/bin/python3.9\n# -*- coding: utf-8 -*-\n\"\"\"\n@Time : 2023/5/22 11:44\n@Author : xhunmon\n@Email : xhunmon@126.co"
},
{
"path": "010-YouTubeUpload/tst.py",
"chars": 156,
"preview": "#!/usr/bin/python3.9\n# -*- coding: utf-8 -*-\n\"\"\"\n@Time : 2023/5/22 11:44\n@Author : xhunmon\n@Email : xhunmon@126.co"
},
{
"path": "010-YouTubeUpload/youtube.py",
"chars": 14786,
"preview": "#!/usr/bin/python3.9\n# -*- coding: utf-8 -*-\n\"\"\"\n@Time : 2023/5/9 09:10\n@Author : xhunmon\n@Email : xhunmon@126.com"
},
{
"path": "LICENSE",
"chars": 18092,
"preview": " GNU GENERAL PUBLIC LICENSE\n Version 2, June 1991\n\n Copyright (C) 1989, 1991 Fr"
},
{
"path": "README.md",
"chars": 835,
"preview": "学习python最好的方式就是通过不断的实践！\n项目地址：[https://github.com/xhunmon/PythonIsTools](https://github.com/xhunmon/PythonIsTools) \n\n\n# 实"
}
]
// ... and 3 more files (download for full content)
About this extraction
This page contains the full source code of the xhunmon/PythonIsTools GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 158 files (555.3 KB), approximately 178.4k tokens, and a symbol index with 647 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.