[ { "path": "README.md", "content": "# magical_spider\n神奇的蜘蛛🕷，一个几乎适用于所有web端站点的采集方案。\n\n\n### 诞生背景\n纯属瞎扯：2022年全球变暖，各行业内卷严重，爬虫届更是入门抖音起步瑞数，为了减缓人才流失，推出magical_spider。\n\n真实原因：一时兴起，吾辈当自强，重铸selenium荣光！ \n\n博客地址： [lxspider](http://www.lxspider.com) 爬虫逆向工具站：[lxtools](http://www.cnlans.com/lx/tools)\n\n\n### 项目简介\n- 非常规derver.pageSource。\n- 通过Flask远程调用chromederver实现xmlHttpRequest。\n- 通过sqlit记录任务状态。\n- 通过undetected_selenium+stealth.min.js绕过一些校验。\n- 目前适用于瑞数、加速乐等cookie加密，以及头条系的请求过程加密。\n\n\n### 项目声明\n- 项目仅供学习参考。\n- 如有风控校验需自行解决，滑块可参考middlerware.py。\n- 方案适用于应急场景或数据量要求不高时，若时间充裕建议通过逆向处理。推荐阅读：[《爬虫逆向进阶实战》](https://github.com/lixi5338619/lxBook)\n\n\n\n### 部署\n[linux部署文档](static/docs/部署.txt)\n\n---\n\n## 使用说明\n\n1、配置settings.py，启动 flask 服务\n\n2、运行方法参考demo文件内容,主要借助runflow.py。\n\n3、测试代码\n\nGET请求\n```python\nfrom demo.runflow import magical_start,magical_request,magical_close\n\nproject_name = 'cnipa'\nbase_url = 'https://www.cnipa.gov.cn'\n\nsession_id,process_url = magical_start(project_name,base_url)\n\nprint(len(magical_request(session_id, process_url,'https://www.cnipa.gov.cn/col/col57/index.html')))\n\nmagical_close(session_id,process_url,project_name)\n```\n\nPOST请求\n```python\nfrom demo.runflow import magical_start,magical_request,magical_close\nimport json\n\nproject_name = 'chinadrugtrials'\nbase_url = 'http://www.chinadrugtrials.org.cn'\n\nsession_id,process_url = magical_start(project_name,base_url)\n\ndata = {\"id\": \"\",\"ckm_index\": \"\",\"sort\": \"desc\",\"sort2\": \"\",\"rule\": \"CTR\",\"secondLevel\": \"0\",\"currentpage\": \"2\",\"keywords\": \"\",\"reg_no\": \"\",\"indication\": \"\",\"case_no\": \"\",\"drugs_name\": \"\",\"drugs_type\": \"\",\"appliers\": \"\",\"communities\": \"\",\"researchers\": \"\",\"agencies\": \"\",\"state\": \"\"}\nformdata = json.dumps(data)\n\nprint(magical_request(session_id=session_id, process_url=process_url,\n request_url='http://www.chinadrugtrials.org.cn/clinicaltrials.searchlist.dhtml',\n request_type='post',formdata=formdata\n ))\n\nmagical_close(session_id,process_url,project_name)\n```\n\n4、index页可以查看和管理当前运行中的任务，也能查看系统内存和磁盘使用情况。\n\n5、demo文件夹中有任务流程汇总runflow.py，以及抖音、药监局案例，单任务和多任务示例。\n\n![Alt](./static/image/index.png)\n" }, { "path": "browserapi.py", "content": "# -*- coding: utf-8 -*-\nimport undetected_chromedriver as webdriver\nfrom undetected_chromedriver.options import ChromeOptions\nfrom settings import *\nfrom selenium.webdriver import ActionChains\nfrom middlerware import Slide\nimport time\nimport platform\n\n\nclass Browser():\n \"\"\"Browser Env : undetected_chromedriver + stealth.js\n headless_enable: 无头模式\n images_enable: 图像开关\n incognito_enable: 无痕模式\n logging_enable: 开启日志\n stealth_enable: stealth 伪装模式\n proxy: 启用代理, 格式：http://127.0.0.1:8888\n \"\"\"\n def __init__(self):\n options = ChromeOptions()\n options.add_argument(\"--lang=en-us\")\n\n if headless_enable:\n options.add_argument(\"--headless\")\n\n if plugin_enable:\n options.add_argument('--disable-images')\n options.add_argument('--disable-plugins')\n options.add_argument('disable-audio')\n options.add_argument('disable-translate')\n\n if proxy:\n options.add_argument('--proxy-server=' + proxy)\n\n if logging_enable:\n options.add_argument('log-level=3')\n\n if incognito_enable:\n options.add_argument(\"--incognito\")\n\n if detach_enable:\n options.add_experimental_option(\"detach\",True)\n\n if platform.system().lower()=='linux':\n options.add_argument(\"--headless\")\n options.add_argument('--no-sandbox')\n options.add_argument('--disable-gpu')\n options.add_argument('--disable-dev-shm-usage')\n\n self.browser = webdriver.Chrome(driver_executable_path=driverpath, options=options)\n if stealth_enable:\n self.stealth_enable()\n\n def stealth_enable(self):\n with open(stealth_path,'r',encoding='utf-8') as file:\n stealth_min_js = file.read()\n self.browser.execute_cdp_cmd(\"Page.addScriptToEvaluateOnNewDocument\", {\n \"source\": stealth_min_js\n })\n\n\n def start_request(self,url):\n self.browser.get(url)\n return self.browser\n\n\n def close(self):\n self.browser.close()\n self.browser.quit()\n\n\nclass BrowserApi():\n def __init__(self,browser):\n self.browser = browser\n\n def browser_ps(self,url):\n self.browser.get(url)\n return self.browser.page_source\n\n\n def browser_get(self, url):\n doc = self.browser.execute_script('''\n function queryData(url) {\n var p = new Promise(function(resolve,reject) {\n var e={\n \"url\":\"%s\",\n \"method\":\"GET\"\n };\n var h = new XMLHttpRequest;\n h.open(e.method, e.url, true);\n h.setRequestHeader(\"salute-by\",\"lx\");\n h.onreadystatechange =function() {\n if(h.readyState === 4 && h.status === 200) {\n resolve(h.responseText);\n } else {}\n };\n h.send(null);\n });\n return p;\n }\n var p1 = queryData('lx');\n res = Promise.all([p1]).then(function(result){\n return result\n })\n return res\n ''' % (url))\n return doc[0]\n\n\n def browser_post(self, url, formdata=\"\"):\n doc = self.browser.execute_script('''\n function queryData(url) {\n var p = new Promise(function(resolve,reject) {\n var e={\"url\":\"%s\",\n \"method\":\"POST\",\n \"data\" : '%s'\n };\n var h = new XMLHttpRequest;\n h.open(e.method, e.url, true);\n h.setRequestHeader(\"accept\",\"application/json, text/plain, */*\"); \n h.setRequestHeader(\"content-type\",\"application/json;charset=UTF-8\");\n h.setRequestHeader(\"salute-by\",\"lx\");\n h.onreadystatechange =function() {\n if(h.readyState != 4) return;\n if(h.readyState === 4 && h.status ===200) {\n resolve(h.responseText);\n } else {\n }\n };\n h.send(e.data);\n });\n return p;\n }\n var p1 = queryData('lx');\n res = Promise.all([p1]).then(function(result){\n return result\n })\n return res;\n ''' % (url, formdata))\n return doc[0]\n\n\n def check_slide(self,bg_xpath,gap_xpath,slider_xpath,domain=None):\n \"\"\"params:\n bg_xpath : 带缺口的背景图片的 xpath\n gap_xpath: 缺口滑块图片的 xpath\n slider_xpath: 待拖动滑块的 xpath\n domain: 图片doamin，非 http开头需补全链接\n \"\"\"\n while 1:\n try:\n bg = self.browser.find_element_by_id(bg_xpath).get_attribute('src')\n gap = self.browser.find_element_by_xpath(gap_xpath).get_attribute('src')\n if not bg.startswith('http'):bg = domain+bg\n if not gap.startswith('http'):gap = domain+gap\n slide_app = Slide(gap=gap, bg=bg)\n distance = slide_app.discern()\n except:\n break\n try:\n slider = self.browser.find_element_by_xpath(slider_xpath)\n ActionChains(self.browser).click_and_hold(slider).perform()\n _tracks = slide_app.get_tracks(distance)\n new_1 = _tracks[-1] - (sum(_tracks) - distance)\n _tracks.pop()\n _tracks.append(new_1)\n for mouse_x in _tracks:\n ActionChains(self.browser).move_by_offset(mouse_x, 0).perform()\n ActionChains(self.browser).release().perform()\n time.sleep(1)\n except:\n break\n\n\n" }, { "path": "config/system_info.py", "content": "# -*- coding: utf-8 -*-\nimport psutil\nclass SystemInfoUtil(object):\n @classmethod\n def get_format_byte(cls, value):\n \"\"\"字节\"\"\"\n kb, b = divmod(value, 1024)\n mb, kb = divmod(kb, 1024)\n gb, mb = divmod(mb, 1024)\n\n if gb > 0:\n return f'{round(gb + mb * 0.001)}GB'\n elif mb > 0:\n return f'{round(mb + kb * 0.001)}MB'\n elif kb > 0:\n return f'{round(kb + b * 0.001)}KB'\n else:\n return f'{round(b)}B'\n\n @classmethod\n def get_virtual_memory(cls):\n \"\"\"\n 内存使用情况\n\n total: 总内存\n available: 可用内存\n percent: 内存使用率\n used: 已使用的内存\n :return:\n \"\"\"\n virtual_memory = psutil.virtual_memory()\n return {\n 'total': virtual_memory.total,\n 'total_format': cls.get_format_byte(virtual_memory.total),\n 'available': virtual_memory.available,\n 'available_format': cls.get_format_byte(virtual_memory.available),\n 'percent': round(virtual_memory.percent),\n 'used': virtual_memory.used,\n 'used_format': cls.get_format_byte(virtual_memory.used),\n }\n\n @classmethod\n def get_disk_usage(cls):\n \"\"\"磁盘使用情况\"\"\"\n disk_usage = psutil.disk_usage('/')\n return {\n 'total': disk_usage.total,\n 'total_format': cls.get_format_byte(disk_usage.total),\n 'used': disk_usage.used,\n 'used_format': cls.get_format_byte(disk_usage.used),\n 'free': disk_usage.free,\n 'free_format': cls.get_format_byte(disk_usage.free),\n 'percent': round(disk_usage.percent),\n }" }, { "path": "db.py", "content": "import os.path\nimport sqlite3\nfrom models import *\nfrom settings import magicalpath\n\n\ndef create_connection():\n db = sqlite3.connect(magicalpath)\n return db\n\n\ndef select_process():\n db = create_connection()\n con = db.cursor()\n con.execute(\"select * from process\")\n res = con.fetchall()\n con.close()\n db.close()\n return res\n\n\ndef select_process_name(processName:str)->Process:\n db = create_connection()\n con = db.cursor()\n con.execute(\"select * from process where processName=?\",(processName,))\n res = con.fetchone()\n con.close()\n db.close()\n return res\n\ndef select_process_id(processId:str)->Process:\n db = create_connection()\n con = db.cursor()\n con.execute(\"select * from process where processId=?\",(processId,))\n res = con.fetchone()\n con.close()\n db.close()\n return res\n\ndef insert_process(process:Process)->Process:\n db = create_connection()\n cursor = db.cursor()\n try:\n cursor.execute(\"insert into process(processId, processName, processUrl, createTime) values ('%s','%s','%s','%s')\" % (process.processId, process.processName, process.processUrl,datetime.datetime.now()))\n db.commit()\n cursor.close()\n db.close()\n return process\n except Exception as e:\n print(e)\n db.rollback()\n cursor.close()\n db.close()\n\n\ndef delete_process(process_name):\n db = create_connection()\n cursor = db.cursor()\n try:\n cursor.execute(\"delete FROM process where processName ='%s'\" % (process_name))\n db.commit()\n cursor.close()\n db.close()\n except Exception as e:\n db.rollback()\n cursor.close()\n db.close()\n\ndef delete_process_id(processId):\n db = create_connection()\n cursor = db.cursor()\n try:\n cursor.execute(\"delete FROM process where processId ='%s'\" % (processId))\n db.commit()\n cursor.close()\n db.close()\n except Exception as e:\n db.rollback()\n cursor.close()\n db.close()\n\n\n\nif __name__ == '__main__':\n if not os.path.exists(magicalpath):\n con = sqlite3.connect(magicalpath)\n cursor = con.cursor()\n cursor.execute(\"CREATE TABLE IF NOT EXISTS `process`(`processId` VARCHAR(90),`processName` VARCHAR(90) UNIQUE,`processUrl` VARCHAR(256),`createTime` DATA,`baseUrl` VARCHAR(256));\")\n con.commit()" }, { "path": "demo/runflow.py", "content": "import requests\nsess = requests.session()\nhost = 'http://127.0.0.1:5000'\n\ndef magical_start(project_name,base_url = 'http://www.lxspider.com'):\n # 1、create browser and select session_id\n result = sess.post(f'{host}/create',data={'name':project_name,'url':base_url}).json()\n session_id,process_url = result['session_id'],result['process_url']\n return session_id,process_url\n\n\ndef magical_request(session_id,process_url,request_url,request_type='get',formdata=''):\n # 2、request browser_xhr\n data = {'session_id': session_id, 'process_url': process_url,\n 'request_url': request_url, 'request_type': request_type}\n\n if request_type.lower()=='post':\n data.update({'request_type':'post','formdata':formdata})\n\n result = sess.post(f'{host}/xhr',data=data).json()\n return result['result']\n\n\ndef magical_close(session_id,process_url,process_name):\n # 4、close browser\n close_data = {'session_id':session_id,'process_url':process_url,'process_name':process_name}\n sess.post(f'{host}/close',data=close_data).json()\n" }, { "path": "demo/单任务GET-demo.py", "content": "from demo.runflow import magical_start,magical_request,magical_close\n\nproject_name = 'cnipa'\nbase_url = 'https://www.cnipa.gov.cn'\n\nsession_id,process_url = magical_start(project_name,base_url)\n\nfor i in range(200):\n print(len(magical_request(session_id, process_url,'https://www.cnipa.gov.cn/col/col2486/index.html')))\n\n\nmagical_close(session_id,process_url,project_name)\n" }, { "path": "demo/单任务POST-demo.py", "content": "from demo.runflow import magical_start,magical_request,magical_close\nimport json\n\n# POST案例昨天忘记加了，感谢 [尘川] 的提醒 by:2022/08/10\n\nproject_name = 'chinadrugtrials'\nbase_url = 'http://www.chinadrugtrials.org.cn'\n\nsession_id,process_url = magical_start(project_name,base_url)\n\ndata = {\"id\": \"\",\"ckm_index\": \"\",\"sort\": \"desc\",\"sort2\": \"\",\"rule\": \"CTR\",\"secondLevel\": \"0\",\"currentpage\": \"2\",\"keywords\": \"\",\"reg_no\": \"\",\"indication\": \"\",\"case_no\": \"\",\"drugs_name\": \"\",\"drugs_type\": \"\",\"appliers\": \"\",\"communities\": \"\",\"researchers\": \"\",\"agencies\": \"\",\"state\": \"\"}\nformdata = json.dumps(data)\nfor i in range(100):\n print(len(magical_request(session_id=session_id, process_url=process_url,\n request_url='http://www.chinadrugtrials.org.cn/clinicaltrials.searchlist.dhtml',\n request_type='post',formdata=formdata\n )))\n\nmagical_close(session_id,process_url,project_name)\n" }, { "path": "demo/多任务demo.py", "content": "from demo.runflow import magical_start,magical_request,magical_close\nimport time\n\n# 各任务间互不影响，可选择使用多线程或多进程，大家自由发挥\n\ndef r1():\n project_name1 = '药监局新闻任务1'\n s1,p1 = magical_start(project_name1,'https://www.nmpa.gov.cn')\n request_list = [\n 'https://www.nmpa.gov.cn/xxgk/ggtg/index.html',\n 'https://www.nmpa.gov.cn/xxgk/fgwj/index.html',\n 'https://www.nmpa.gov.cn/xxgk/fgwj/index.html'\n ]\n for request_url in request_list:\n print(\"r1:\", len(magical_request(s1, p1, request_url)))\n time.sleep(5)\n magical_close(s1,p1,project_name1)\n\ndef r2():\n project_name2 = '药监局新闻任务2'\n s2,p2 = magical_start(project_name2,'https://www.nmpa.gov.cn')\n request_list = ['https://www.nmpa.gov.cn/zwgk/rshxx/index.html',\n 'https://www.nmpa.gov.cn/zwgk/xwfb/index.html',\n 'https://www.nmpa.gov.cn/zwgk/xwfb/index.html'\n ]\n for request_url in request_list:\n print(\"r2:\", len(magical_request(s2, p2, request_url)))\n time.sleep(5)\n magical_close(s2,p2,project_name2)\n\n\nimport threading\nthread1 = threading.Thread(target=r1)\nthread2 = threading.Thread(target=r2)\nthread1.start()\nthread2.start()" }, { "path": "demo/抖音-步骤拆解demo.py", "content": "import requests\n\n# 步骤拆解，简化版查看 药监局.py\n\nproject_name = '抖音任务2' # project_name不可重复，勿创建重复任务名\nbase_url = 'https://www.douyin.com'\n\n# 1、browser init and select browser_session\nresult = requests.post('http://127.0.0.1:5000/create',data={'name':project_name,'url':base_url}).json()\nsession_id = result['session_id']\nprocess_url = result['process_url']\n\n# 2、request browser_xhr\n# URL需要更换为你浏览器中的\nrequest_list = [\n 'https://www.douyin.com/aweme/v1/web/search/item/?device_platform=webapp&aid=6383&channel=channel_pc_web&search_channel=aweme_video_web&sort_type=0&publish_time=0&keyword=%E9%9E%A0%E5%A9%A7%E7%A5%8E&search_source=normal_search&query_correct_type=1&is_filter_search=0&from_group_id=&offset=0&count=10&version_code=170400&version_name=17.4.0&cookie_enabled=true&screen_width=1920&screen_height=1080&browser_language=zh-CN&browser_platform=Win32&browser_name=Chrome&browser_version=104.0.0.0&browser_online=true&engine_name=Blink&engine_version=104.0.0.0&os_name=Windows&os_version=10&cpu_core_num=20&device_memory=8&platform=PC&downlink=10&effective_type=4g&round_trip_time=50&webid=7122050458177701414',\n 'https://www.douyin.com/aweme/v1/web/search/item/?device_platform=webapp&aid=6383&channel=channel_pc_web&search_channel=aweme_video_web&sort_type=0&publish_time=0&keyword=lx&search_source=normal_search&query_correct_type=1&is_filter_search=0&from_group_id=&offset=0&count=10&version_code=170400&version_name=17.4.0&cookie_enabled=true&screen_width=1920&screen_height=1080&browser_language=zh-CN&browser_platform=Win32&browser_name=Chrome&browser_version=104.0.0.0&browser_online=true&engine_name=Blink&engine_version=104.0.0.0&os_name=Windows&os_version=10&cpu_core_num=20&device_memory=8&platform=PC&downlink=10&effective_type=4g&round_trip_time=50&webid=7122050458177701414',\n 'https://www.douyin.com/aweme/v1/web/search/item/?device_platform=webapp&aid=6383&channel=channel_pc_web&search_channel=aweme_video_web&sort_type=0&publish_time=0&keyword=pythonlx&search_source=normal_search&query_correct_type=1&is_filter_search=0&from_group_id=&offset=0&count=10&version_code=170400&version_name=17.4.0&cookie_enabled=true&screen_width=1920&screen_height=1080&browser_language=zh-CN&browser_platform=Win32&browser_name=Chrome&browser_version=104.0.0.0&browser_online=true&engine_name=Blink&engine_version=104.0.0.0&os_name=Windows&os_version=10&cpu_core_num=20&device_memory=8&platform=PC&downlink=10&effective_type=4g&round_trip_time=50&webid=7122050458177701414',\n 'https://www.douyin.com/aweme/v1/web/search/item/?device_platform=webapp&aid=6383&channel=channel_pc_web&search_channel=aweme_video_web&sort_type=0&publish_time=0&keyword=lx666&search_source=normal_search&query_correct_type=1&is_filter_search=0&from_group_id=&offset=0&count=10&version_code=170400&version_name=17.4.0&cookie_enabled=true&screen_width=1920&screen_height=1080&browser_language=zh-CN&browser_platform=Win32&browser_name=Chrome&browser_version=104.0.0.0&browser_online=true&engine_name=Blink&engine_version=104.0.0.0&os_name=Windows&os_version=10&cpu_core_num=20&device_memory=8&platform=PC&downlink=10&effective_type=4g&round_trip_time=50&webid=7122050458177701414',\n]\nfor request_url in request_list:\n data = {'session_id':session_id,'process_url':process_url,\n 'request_url':request_url,'request_type':'get'}\n result = requests.post('http://127.0.0.1:5000/xhr',data=data).json()\n print(len(result['result']))\n\n# 3、close browser\nclose_data = {'session_id':session_id,'process_url':process_url,'process_name':project_name}\nrequests.post('http://127.0.0.1:5000/close',data=close_data).json()\n" }, { "path": "demo/药监局.py", "content": "from demo.runflow import magical_start,magical_request,magical_close\n\nproject_name = '药监局1'\nbase_url = 'https://www.nmpa.gov.cn'\nrequest_list = [\n 'https://www.nmpa.gov.cn/yaopin/ypjgdt/index.html',\n 'https://www.nmpa.gov.cn/yaopin/ypjgdt/20220705190551125.html'\n]\n\nsession_id,process_url = magical_start(project_name,base_url)\n\nfor request_url in request_list:\n print(magical_request(session_id, process_url, request_url))\n\nmagical_close(session_id,process_url,project_name)" }, { "path": "engine.py", "content": "# -*- coding: utf-8 -*-\nfrom browserapi import Browser,BrowserApi\nfrom db import *\nfrom models import Process\nfrom selenium import webdriver\nfrom selenium.webdriver.remote.webdriver import WebDriver\n\n\ndef create_browser(url,name):\n bro = Browser()\n browser = bro.start_request(url)\n session_id = browser.session_id\n process_url = browser.command_executor._url\n insert_process(Process(session_id,name,process_url,url))\n return browser\n\ndef attachToSession(session_id,url):\n original_execute = WebDriver.execute\n def new_command_execute(self, command, params=None):\n if command == \"newSession\":\n return {'success': 0, 'value': None, 'sessionId': session_id}\n else:\n return original_execute(self, command, params)\n WebDriver.execute = new_command_execute\n driver = webdriver.Remote(command_executor=url, desired_capabilities={})\n driver.session_id = session_id\n WebDriver.execute = original_execute\n return driver\n\n\ndef carry_browser(session_id,process_url,request_url,request_type,formdata):\n try:\n browser = attachToSession(session_id,process_url)\n except:\n # 防止窗口崩溃 -> 增加的重建操作\n print(\"防止窗口崩溃 -> 增加的重建操作\")\n browser_info = select_process_id(session_id)\n base_url = browser_info[4]\n process_name = browser_info[1]\n delete_process(process_name)\n browser = create_browser(base_url,process_name)\n print(\"browser 重建成功\")\n\n broapi = BrowserApi(browser)\n if request_type=='get':\n result = broapi.browser_get(request_url)\n else:\n result = broapi.browser_post(request_url,formdata)\n return result\n\n\ndef close_browser(session_id,process_url,process_name):\n delete_process(process_name)\n browser = attachToSession(session_id,process_url)\n browser.close()\n browser.quit()\n\n\n\ndef select_all_process():\n return select_process()\n\n" }, { "path": "middlerware.py", "content": "# -*- coding: utf-8 -*-\nimport os,requests\nfrom urllib.parse import urlparse\ntry:\n import cv2, numpy as np\nexcept:\n ...\n\nclass Slide(object):\n def __init__(self, gap, bg, gap_size=None, bg_size=None, out=None):\n \"\"\"\n :param bg: 带缺口的图片链接或者url\n :param gap: 缺口图片链接或者url\n \"\"\"\n self.img_dir = os.path.join(os.getcwd(), 'img')\n if not os.path.exists(self.img_dir):\n os.makedirs(self.img_dir)\n\n bg_resize = bg_size if bg_size else (340, 212)\n gap_size = gap_size if gap_size else (68, 68)\n self.bg = self.check_is_img_path(bg, 'bg', resize=bg_resize)\n self.gap = self.check_is_img_path(gap, 'gap', resize=gap_size)\n self.out = out if out else os.path.join(self.img_dir, 'out.jpg')\n\n\n @staticmethod\n def check_is_img_path(img, img_type, resize):\n if img.startswith('http'):\n headers = {\n \"Accept\": \"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;\"\n \"q=0.8,application/signed-exchange;v=b3;q=0.9\",\n \"Accept-Encoding\": \"gzip, deflate, br\",\n \"Accept-Language\": \"zh-CN,zh;q=0.9,en-GB;q=0.8,en;q=0.7,ja;q=0.6\",\n \"Cache-Control\": \"max-age=0\",\n \"Connection\": \"keep-alive\",\n \"Host\": urlparse(img).hostname,\n \"Upgrade-Insecure-Requests\": \"1\",\n \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \"\n \"Chrome/91.0.4472.164 Safari/537.36\",\n }\n img_res = requests.get(img, headers=headers)\n if img_res.status_code == 200:\n img_path = f'./img/{img_type}.jpg'\n image = np.asarray(bytearray(img_res.content), dtype=\"uint8\")\n image = cv2.imdecode(image, cv2.IMREAD_COLOR)\n if resize:\n image = cv2.resize(image, dsize=resize)\n cv2.imwrite(img_path, image)\n return img_path\n else:\n raise Exception(f\"保存{img_type}图片失败\")\n else:\n return img\n\n\n @staticmethod\n def clear_white(img):\n \"\"\"清除图片的空白区域，这里主要清除滑块的空白\"\"\"\n img = cv2.imread(img)\n rows, cols, channel = img.shape\n min_x = 255\n min_y = 255\n max_x = 0\n max_y = 0\n for x in range(1, rows):\n for y in range(1, cols):\n t = set(img[x, y])\n if len(t) >= 2:\n if x <= min_x:\n min_x = x\n elif x >= max_x:\n max_x = x\n\n if y <= min_y:\n min_y = y\n elif y >= max_y:\n max_y = y\n img1 = img[min_x:max_x, min_y: max_y]\n return img1\n\n\n def template_match(self, tpl, target):\n th, tw = tpl.shape[:2]\n result = cv2.matchTemplate(target, tpl, cv2.TM_CCOEFF_NORMED)\n # 寻找矩阵(一维数组当作向量,用Mat定义) 中最小值和最大值的位置\n min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)\n tl = max_loc\n br = (tl[0] + tw, tl[1] + th)\n # 绘制矩形边框，将匹配区域标注出来\n # target：目标图像\n # tl：矩形定点\n # br：矩形的宽高\n # (0,0,255)：矩形边框颜色\n # 1：矩形边框大小\n cv2.rectangle(target, tl, br, (0, 0, 255), 2)\n cv2.imwrite(self.out, target)\n return tl[0]\n\n\n @staticmethod\n def image_edge_detection(img):\n edges = cv2.Canny(img, 100, 200)\n return edges\n\n\n def discern(self):\n img1 = self.clear_white(self.gap)\n img1 = cv2.cvtColor(img1, cv2.COLOR_RGB2GRAY)\n slide = self.image_edge_detection(img1)\n\n back = cv2.imread(self.bg, 0)\n back = self.image_edge_detection(back)\n\n slide_pic = cv2.cvtColor(slide, cv2.COLOR_GRAY2RGB)\n back_pic = cv2.cvtColor(back, cv2.COLOR_GRAY2RGB)\n x = self.template_match(slide_pic, back_pic)\n # 输出横坐标, 即 滑块在图片上的位置\n return x\n\n\n @staticmethod\n def get_tracks(distance, rate=0.6, t=0.2, v=0):\n \"\"\"\n 将distance分割成小段的距离\n :param distance: 总距离\n :param rate: 加速减速的临界比例\n :param a1: 加速度\n :param a2: 减速度\n :param t: 单位时间\n :param t: 初始速度\n :return: 小段的距离集合\n \"\"\"\n tracks = []\n # 加速减速的临界值\n mid = rate * distance\n # 当前位移\n s = 0\n # 循环\n while s < distance:\n # 初始速度\n v0 = v\n if s < mid:\n a = 20\n else:\n a = -3\n # 计算当前t时间段走的距离\n s0 = v0 * t + 0.5 * a * t * t\n # 计算当前速度\n v = v0 + a * t\n # 四舍五入距离，因为像素没有小数\n tracks.append(round(s0))\n # 计算当前距离\n s += s0\n return tracks\n" }, { "path": "models.py", "content": "import datetime\n\nclass Process:\n def __init__(self, processId, processName,processUrl,baseUrl,createTime = datetime.datetime.now()) -> None:\n super().__init__()\n self.processId = processId\n self.processName = processName\n self.processUrl = processUrl\n self.createTime = createTime\n self.baseUrl = baseUrl\n" }, { "path": "server.py", "content": "# -*- coding: utf-8 -*-\nfrom datetime import timedelta\nfrom flask import Flask,session\nfrom flask import render_template,request,redirect,url_for,jsonify\nimport os\nfrom engine import *\nfrom config.system_info import SystemInfoUtil\nfrom settings import host,port\n\n\napp = Flask(__name__)\napp.config['SECRET_KEY'] = os.urandom(24)\napp.config['SEND_FILE_MAX_AGE_DEFAULT'] = timedelta(days=7)\n\n\n@app.route('/')\ndef index_info():\n process = select_all_process()\n if not process:process=[[\"\",\"没有在运行的任务\",\"\",\"\"]]\n disk_usage = SystemInfoUtil.get_disk_usage()\n virtual_memory = SystemInfoUtil.get_virtual_memory()\n return render_template('index.html',process=process,disk_usage=disk_usage,virtual_memory=virtual_memory)\n\n\n@app.route('/create',methods=['POST'])\ndef browser_start():\n url = request.form.get(\"url\")\n name = request.form.get(\"name\")\n try:\n create_browser(url,name)\n session_id, process_name, process_url, datetime,base_url = select_process_name(name)\n result = {'session_id': session_id, 'process_name': process_name,\n 'process_url': process_url, 'datetime': datetime}\n return jsonify(result)\n except:\n return jsonify({\"result\":0,\"detail\":\"驱动配置错误或任务名已存在\"})\n\n\n@app.route('/xhr',methods=['POST'])\ndef browser_xhr():\n session_id = request.form.get(\"session_id\")\n process_url = request.form.get(\"process_url\")\n request_url = request.form.get(\"request_url\")\n request_type = request.form.get(\"request_type\")\n formdata = request.form.get(\"formdata\")\n result = carry_browser(session_id,process_url,request_url,request_type,formdata)\n return jsonify({\"result\":result})\n\n\n@app.route('/close',methods=['POST'])\ndef browser_close():\n session_id = request.form.get(\"session_id\")\n process_url = request.form.get(\"process_url\")\n process_name = request.form.get(\"process_name\")\n try:\n close_browser(session_id,process_url,process_name)\n return jsonify({\"result\":1})\n except:\n return jsonify({\"result\":0,\"detail\":\"驱动窗口已自动关闭\"})\n\n\n@app.route('/delete/',methods=['GET'])\ndef delete_process_name(process_name):\n try:\n process = select_process_name(process_name)\n close_browser(session_id=process[0],process_url=process[2],process_name=process_name)\n except:\n delete_process(process_name)\n print(\"delete except: line 70\")\n return redirect('/')\n\n\nif __name__ == '__main__':\n app.run(host=host,port=port,use_reloader=False,debug=True)" }, { "path": "settings.py", "content": "# MagicalSpider Settings\n\n# 隐藏界面\nheadless_enable = True\n\n# 高匿模式、可能影响创建时间\nstealth_enable = True\n\n# 代理设置\nproxy = None\n\n# 无痕访问\nincognito_enable = False\n\n# 分离模式\ndetach_enable = False\n\nplugin_enable = False\n\nlogging_enable = False\n\n\ndriverpath = './config/chromedriver.exe'\n\nmagicalpath = './config/magical.db'\n\nstealth_path = './config/stealth.min.js'\n\nhost = '0.0.0.0'\n\nport = 5000\n\n# 让 Selenium 在 Linux 中以有头模式运行\n# xvfb-run python3 test.py -s -screen 0 1920x1080x16\n" }, { "path": "static/css/index.css", "content": "body{\n background: url(/static/image/bg.png);\n background-size: 100% 100%;\n background-repeat:no-repeat;\n}\na{\n text-decoration:none;\n}\np{\n font-size: 18px;\n color: white;\n}\nimg{\n width: 207.99px;\n height: 207.99px;\n}\n\n\ntable{\n border-collapse: collapse;\n margin-left: 6%;\n text-align: center;\n}\ntable td, table th\n{\n border: 1px solid #cad9ea;\n color: #666;\n height: 30px;\n}\ntable thead th\n{\n background-color: #CCE8EB;\n width: 260px;\n}\ntable tr:nth-child(odd)\n{\n background: #fff;\n}\ntable tr:nth-child(even)\n{\n background: #F5FAFA;\n}\n\n.lx{\n display: inline-block;\n margin-left: 5%;\n}\n.bt{\n font-weight:bold;\n color:#5b91d6;\n}\n.blog{\n border:solid;\n border-color:#5784d0;\n width: 10%;\n margin-left: 65.6%;\n position: fixed;\n top: 0;\n}" }, { "path": "static/docs/program.txt", "content": "magical_spider，一个几乎适用于所有web端站点的采集方案。\n\n## 项目简介\n\n1、主要使用谷歌驱动，但非常规derver.page_source。\n\n2、通过 flask 远程调用 chromederver 实现 xmlHttpRequest 传输数据。\n\n3、通过sqlit 记录和管理任务状态。\n\n4、通过undetected_selenium+stealth.min.js绕过一些校验。\n\n5、测试通过瑞数、加速乐等cookie加密，以及头条系的请求过程加密。\n\n6、支持 linux 部署，支持多任务。\n\n## 项目原理\n\n打造一个近乎真实的浏览器环境，去完成网站内部环境的请求加载，直接返回响应内容供本地调用。\n\n\n## 项目声明\n\n1、整合了一些其他开源项目，仅供学习参考。\n\n2、适用于应急场景或小量任务，方便便捷，若时间充裕建议通过逆向处理。\n\n3、如有风控校验需自行解决，滑块可参考middlerware.py。\n\n\n## 备注\n\n1、index页可以查看和管理当前运行中的任务，也能查看系统内存和磁盘使用情况。\n\n2、demo文件夹中有任务流程汇总runflow.py，以及抖音、药监局案例，单任务和多任务示例。\n\n3、运行前配置服务信息和驱动路径，启动flask服务后再执行任务。" }, { "path": "static/docs/部署.txt", "content": "Linux部署\n\n1.安装chrome (自行选择安装位置)\nyum install https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm\n\n2.检查chrome的版本\ngoogle-chrome --version\n\n3.安装对应版本的 chromedriver_linux64\n比如我的chrome版本是104.0.5112.79\nwget https://npm.taobao.org/mirrors/chromedriver/104.0.5112.79/chromedriver_linux64.zip\n\n4.解压\nunzip chromedriver_linux64\n\n5.授权\nchmod 777 chromedriver\n\n6.修改项目代码settings.py中的chromedriver路径\n\n7.安装python依赖后启动flask项目\n- Python依赖 ：flask、sqlite3、selenium、websockets、opencv-python、numpy\n- flask启动方式：python3 sever.py\n\n8.开启服务器端口访问权限\n\n9.运行项目测试" }, { "path": "templates/index.html", "content": "\n\n\n \n MagicalSpider\n\n\n\n\n\n
\n
\n
\n
\n
\n
\n

\n 【MagicalSpider】 神奇的蜘蛛🕷，一个几乎适用于所有web端站点的采集方案。(比如瑞数、加速乐、头条系、五秒盾等)\n

\n
\n

\n 诞生背景：\n 2022年全球变暖，各行业内卷严重，爬虫届更是入门抖音+瑞数，导致发生 [从入门到放弃] 。\n 所以吾辈当自强，重铸selenium荣光 (本段纯属瞎扯) 。\n

\n
\n

\n magical_spider：项目说明\n 、部署文档\n

\n\n\n
\n
\n\n\n

运行中的任务:

\n
\n \n \n \n \n \n \n \n \n \n {% for p in process %}\n \n \n \n \n \n {% if p[0] %}\n \n {% else %}\n \n {% endif %}\n\n \n {% endfor %}\n \n
任务名任务ID任务地址创建时间任务管理
{{ p[1] }}{{ p[0] }}{{ p[2] }}{{ p[3][:19] }}删除任务
\n
\n
\n\n
\n
\n \n

个人博客：lxspider

\n \n
\n\n
\n

公众号《Pythonlx》

\n

\n \"\"\n

\n
\n\n
\n

内存使用情况

\n

总内存: {{ virtual_memory.total_format }}

\n

可用内存: {{ virtual_memory.available_format }}

\n

已使用内存: {{ virtual_memory.used_format }}

\n

内存使用率: {{ virtual_memory.percent }}%

\n
\n
\n
\n\n\n
\n

磁盘使用情况

\n

总内存: {{ disk_usage.total_format }}

\n

可用内存: {{ disk_usage.free_format }}

\n

已使用内存: {{ disk_usage.used_format }}

\n

内存使用率: {{ disk_usage.percent }}%

\n
\n
\n
\n\n{#
#}\n{#

微信赞助

#}\n{#

#}\n{# \"\"#}\n{#

#}\n{#
#}\n\n
\n\n\n" }, { "path": "undetected_chromedriver/__init__.py", "content": "#!/usr/bin/env python3\n#from __future__ import annotations\n\nimport subprocess\n\n\"\"\"\n\n 888 888 d8b\n 888 888 Y8P\n 888 888\n .d8888b 88888b. 888d888 .d88b. 88888b.d88b. .d88b. .d88888 888d888 888 888 888 .d88b. 888d888\nd88P\" 888 \"88b 888P\" d88\"\"88b 888 \"888 \"88b d8P Y8b d88\" 888 888P\" 888 888 888 d8P Y8b 888P\"\n888 888 888 888 888 888 888 888 888 88888888 888 888 888 888 Y88 88P 88888888 888\nY88b. 888 888 888 Y88..88P 888 888 888 Y8b. Y88b 888 888 888 Y8bd8P Y8b. 888\n \"Y8888P 888 888 888 \"Y88P\" 888 888 888 \"Y8888 \"Y88888 888 888 Y88P \"Y8888 888 88888888\n\nby UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)\n\n\"\"\"\n\n\n\"\"\"\n user_data_dir、language、webdriver、Webelement、no-first-run、window-size、log-level、start-maximized、no-sandbox\n\"\"\"\n\n__version__ = \"3.1.5r4\"\n\n\nimport json\nimport logging\nimport os\nimport re\nimport shutil\nimport sys\nimport tempfile\nimport time\nimport inspect\nimport threading\n\nimport selenium.webdriver.chrome.service\nimport selenium.webdriver.chrome.webdriver\nimport selenium.webdriver.common.service\nimport selenium.webdriver.remote.webdriver\n\nfrom .cdp import CDP\nfrom .options import ChromeOptions\nfrom .patcher import IS_POSIX\nfrom .patcher import Patcher\nfrom .reactor import Reactor\nfrom .dprocess import start_detached\n\n__all__ = (\n \"Chrome\",\n \"ChromeOptions\",\n \"Patcher\",\n \"Reactor\",\n \"CDP\",\n \"find_chrome_executable\",\n)\n\nlogger = logging.getLogger(\"uc\")\nlogger.setLevel(logging.getLogger().getEffectiveLevel())\n\n\nclass Chrome(selenium.webdriver.chrome.webdriver.WebDriver):\n \"\"\"\n\n Controls the ChromeDriver and allows you to drive the browser.\n\n The webdriver file will be downloaded by this module automatically,\n you do not need to specify this. however, you may if you wish.\n\n Attributes\n ----------\n\n Methods\n -------\n\n reconnect()\n\n this can be useful in case of heavy detection methods\n -stops the chromedriver service which runs in the background\n -starts the chromedriver service which runs in the background\n -recreate session\n\n\n start_session(capabilities=None, browser_profile=None)\n\n differentiates from the regular method in that it does not\n require a capabilities argument. The capabilities are automatically\n recreated from the options at creation time.\n\n --------------------------------------------------------------------------\n NOTE:\n Chrome has everything included to work out of the box.\n it does not `need` customizations.\n any customizations MAY lead to trigger bot migitation systems.\n\n --------------------------------------------------------------------------\n \"\"\"\n\n _instances = set()\n session_id = None\n debug = False\n\n def __init__(\n self,\n options=None,\n user_data_dir=None,\n driver_executable_path=None,\n browser_executable_path=None,\n port=0,\n enable_cdp_events=False,\n service_args=None,\n desired_capabilities=None,\n advanced_elements=False,\n service_log_path=None,\n keep_alive=True,\n log_level=0,\n headless=False,\n version_main=None,\n patcher_force_close=False,\n suppress_welcome=True,\n use_subprocess=False,\n debug=False,\n **kw\n ):\n \"\"\"\n Creates a new instance of the chrome driver.\n\n Starts the service and then creates new instance of chrome driver.\n\n Parameters\n ----------\n\n options: ChromeOptions, optional, default: None - automatic useful defaults\n this takes an instance of ChromeOptions, mainly to customize browser behavior.\n anything other dan the default, for example extensions or startup options\n are not supported in case of failure, and can probably lowers your undetectability.\n\n\n user_data_dir: str , optional, default: None (creates temp profile)\n if user_data_dir is a path to a valid chrome profile directory, use it,\n and turn off automatic removal mechanism at exit.\n\n driver_executable_path: str, optional, default: None(=downloads and patches new binary)\n\n browser_executable_path: str, optional, default: None - use find_chrome_executable\n Path to the browser executable.\n If not specified, make sure the executable's folder is in $PATH\n\n port: int, optional, default: 0\n port you would like the service to run, if left as 0, a free port will be found.\n\n enable_cdp_events: bool, default: False\n :: currently for chrome only\n this enables the handling of wire messages\n when enabled, you can subscribe to CDP events by using:\n\n driver.add_cdp_listener(\"Network.dataReceived\", yourcallback)\n # yourcallback is an callable which accepts exactly 1 dict as parameter\n\n\n service_args: list of str, optional, default: None\n arguments to pass to the driver service\n\n desired_capabilities: dict, optional, default: None - auto from config\n Dictionary object with non-browser specific capabilities only, such as \"item\" or \"loggingPref\".\n\n advanced_elements: bool, optional, default: False\n makes it easier to recognize elements like you know them from html/browser inspection, especially when working\n in an interactive environment\n\n default webelement repr:\n \n\n advanced webelement repr\n )>\n\n note: when retrieving large amounts of elements ( example: find_elements_by_tag(\"*\") ) and print them, it does take a little more time.\n\n\n service_log_path: str, optional, default: None\n path to log information from the driver.\n\n keep_alive: bool, optional, default: True\n Whether to configure ChromeRemoteConnection to use HTTP keep-alive.\n\n log_level: int, optional, default: adapts to python global log level\n\n headless: bool, optional, default: False\n can also be specified in the options instance.\n Specify whether you want to use the browser in headless mode.\n warning: this lowers undetectability and not fully supported.\n\n version_main: int, optional, default: None (=auto)\n if you, for god knows whatever reason, use\n an older version of Chrome. You can specify it's full rounded version number\n here. Example: 87 for all versions of 87\n\n patcher_force_close: bool, optional, default: False\n instructs the patcher to do whatever it can to access the chromedriver binary\n if the file is locked, it will force shutdown all instances.\n setting it is not recommended, unless you know the implications and think\n you might need it.\n\n suppress_welcome: bool, optional , default: True\n a \"welcome\" alert might show up on *nix-like systems asking whether you want to set\n chrome as your default browser, and if you want to send even more data to google.\n now, in case you are nag-fetishist, or a diagnostics data feeder to google, you can set this to False.\n Note: if you don't handle the nag screen in time, the browser loses it's connection and throws an Exception.\n\n use_subprocess: bool, optional , default: False,\n\n False (the default) makes sure Chrome will get it's own process (so no subprocess of chromedriver.exe or python\n This fixes a LOT of issues, like multithreaded run, but mst importantly. shutting corectly after\n program exits or using .quit()\n\n unfortunately, there is always an edge case in which one would like to write an single script with the only contents being:\n --start script--\n import undetected_chromedriver as uc\n d = uc.Chrome()\n d.get('https://somesite/')\n ---end script --\n\n and will be greeted with an error, since the program exists before chrome has a change to launch.\n in that case you can set this to `True`. The browser will start via subprocess, and will keep running most of times.\n ! setting it to True comes with NO support when being detected. !\n\n \"\"\"\n self.debug = debug\n patcher = Patcher(\n executable_path=driver_executable_path,\n force=patcher_force_close,\n version_main=version_main,\n )\n patcher.auto()\n self.patcher = patcher\n if not options:\n options = ChromeOptions()\n\n\n try:\n if hasattr(options, \"_session\") and options._session is not None:\n # prevent reuse of options,\n # as it just appends arguments, not replace them\n # you'll get conflicts starting chrome\n raise RuntimeError(\"you cannot reuse the ChromeOptions object\")\n except AttributeError:\n pass\n\n options._session = self\n\n debug_port = selenium.webdriver.common.service.utils.free_port()\n debug_host = \"127.0.0.1\"\n\n if not options.debugger_address:\n options.debugger_address = \"%s:%d\" % (debug_host, debug_port)\n\n if enable_cdp_events:\n options.set_capability(\n \"goog:loggingPrefs\", {\"performance\": \"ALL\", \"browser\": \"ALL\"}\n )\n\n options.add_argument(\"--remote-debugging-host=%s\" % debug_host)\n options.add_argument(\"--remote-debugging-port=%s\" % debug_port)\n\n if user_data_dir:\n options.add_argument('--user-data-dir=%s' % user_data_dir)\n\n language, keep_user_data_dir = None, bool(user_data_dir)\n\n # see if a custom user profile is specified in options\n for arg in options.arguments:\n\n if \"lang\" in arg:\n m = re.search(\"(?:--)?lang(?:[ =])?(.*)\", arg)\n try:\n language = m[1]\n except IndexError:\n logger.debug(\"will set the language to en-US,en;q=0.9\")\n language = \"en-US,en;q=0.9\"\n\n if \"user-data-dir\" in arg:\n m = re.search(\"(?:--)?user-data-dir(?:[ =])?(.*)\", arg)\n try:\n user_data_dir = m[1]\n logger.debug(\n \"user-data-dir found in user argument %s => %s\" % (arg, m[1])\n )\n keep_user_data_dir = True\n\n except IndexError:\n logger.debug(\n \"no user data dir could be extracted from supplied argument %s \"\n % arg\n )\n\n if not user_data_dir:\n\n # backward compatiblity\n # check if an old uc.ChromeOptions is used, and extract the user data dir\n\n if hasattr(options, \"user_data_dir\") and getattr(\n options, \"user_data_dir\", None\n ):\n import warnings\n\n warnings.warn(\n \"using ChromeOptions.user_data_dir might stop working in future versions.\"\n \"use uc.Chrome(user_data_dir='/xyz/some/data') in case you need existing profile folder\"\n )\n options.add_argument(\"--user-data-dir=%s\" % options.user_data_dir)\n keep_user_data_dir = True\n logger.debug(\n \"user_data_dir property found in options object: %s\" % user_data_dir\n )\n\n else:\n user_data_dir = os.path.normpath(tempfile.mkdtemp())\n keep_user_data_dir = False\n arg = \"--user-data-dir=%s\" % user_data_dir\n options.add_argument(arg)\n logger.debug(\n \"created a temporary folder in which the user-data (profile) will be stored during this\\n\"\n \"session, and added it to chrome startup arguments: %s\" % arg\n )\n\n if not language:\n try:\n import locale\n language = locale.getdefaultlocale()[0].replace(\"_\", \"-\")\n except Exception:\n pass\n if not language:\n language = \"en-US\"\n\n options.add_argument(\"--lang=%s\" % language)\n\n if not options.binary_location:\n options.binary_location = (\n browser_executable_path or find_chrome_executable()\n )\n\n self._delay = 3\n\n self.user_data_dir = user_data_dir\n self.keep_user_data_dir = keep_user_data_dir\n\n if suppress_welcome:\n options.arguments.extend([\"--no-default-browser-check\", \"--no-first-run\"])\n if headless or options.headless:\n options.headless = True\n options.add_argument(\"--window-size=1920,1080\")\n options.add_argument(\"--start-maximized\")\n options.add_argument(\"--no-sandbox\")\n # fixes \"could not connect to chrome\" error when running\n # on linux using privileged user like root (which i don't recommend)\n\n options.add_argument(\n \"--log-level=%d\" % log_level\n or divmod(logging.getLogger().getEffectiveLevel(), 10)[0]\n )\n\n if hasattr(options, 'handle_prefs'):\n options.handle_prefs(user_data_dir)\n\n # fix exit_type flag to prevent tab-restore nag\n try:\n with open(\n os.path.join(user_data_dir, \"Default/Preferences\"),\n encoding=\"latin1\",\n mode=\"r+\",\n ) as fs:\n config = json.load(fs)\n if config[\"profile\"][\"exit_type\"] is not None:\n # fixing the restore-tabs-nag\n config[\"profile\"][\"exit_type\"] = None\n fs.seek(0, 0)\n json.dump(config, fs)\n logger.debug(\"fixed exit_type flag\")\n except Exception as e:\n logger.debug(\"did not find a bad exit_type flag \")\n\n self.options = options\n\n if not desired_capabilities:\n desired_capabilities = options.to_capabilities()\n\n if not use_subprocess:\n self.browser_pid = start_detached(\n options.binary_location, *options.arguments\n )\n else:\n browser = subprocess.Popen(\n [options.binary_location, *options.arguments],\n stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n close_fds=IS_POSIX,\n )\n self.browser_pid = browser.pid\n\n super(Chrome, self).__init__(\n executable_path=patcher.executable_path,\n port=port,\n options=options,\n service_args=service_args,\n desired_capabilities=desired_capabilities,\n service_log_path=service_log_path,\n keep_alive=keep_alive,\n )\n\n self.reactor = None\n\n if enable_cdp_events:\n if logging.getLogger().getEffectiveLevel() == logging.DEBUG:\n logging.getLogger(\n \"selenium.webdriver.remote.remote_connection\"\n ).setLevel(20)\n reactor = Reactor(self)\n reactor.start()\n self.reactor = reactor\n\n if advanced_elements:\n from .webelement import WebElement\n\n self._web_element_cls = WebElement\n\n if options.headless:\n self._configure_headless()\n\n def __getattribute__(self, item):\n\n if not super().__getattribute__(\"debug\"):\n return super().__getattribute__(item)\n else:\n import inspect\n\n original = super().__getattribute__(item)\n if inspect.ismethod(original) and not inspect.isclass(original):\n\n def newfunc(*args, **kwargs):\n logger.debug(\n \"calling %s with args %s and kwargs %s\\n\"\n % (original.__qualname__, args, kwargs)\n )\n return original(*args, **kwargs)\n\n return newfunc\n return original\n\n def _configure_headless(self):\n\n orig_get = self.get\n logger.info(\"setting properties for headless\")\n\n def get_wrapped(*args, **kwargs):\n if self.execute_script(\"return navigator.webdriver\"):\n logger.info(\"patch navigator.webdriver\")\n self.execute_cdp_cmd(\n \"Page.addScriptToEvaluateOnNewDocument\",\n {\n \"source\": \"\"\"\n Object.defineProperty(window, 'navigator', {\n value: new Proxy(navigator, {\n has: (target, key) => (key === 'webdriver' ? false : key in target),\n get: (target, key) =>\n key === 'webdriver' ?\n false :\n typeof target[key] === 'function' ?\n target[key].bind(target) :\n target[key]\n })\n });\n\n \"\"\"\n },\n )\n\n logger.info(\"patch user-agent string\")\n self.execute_cdp_cmd(\n \"Network.setUserAgentOverride\",\n {\n \"userAgent\": self.execute_script(\n \"return navigator.userAgent\"\n ).replace(\"Headless\", \"\")\n },\n )\n self.execute_cdp_cmd(\n \"Page.addScriptToEvaluateOnNewDocument\",\n {\n \"source\": \"\"\"\n Object.defineProperty(navigator, 'maxTouchPoints', {\n get: () => 1\n })\"\"\"\n },\n )\n return orig_get(*args, **kwargs)\n\n self.get = get_wrapped\n\n def __dir__(self):\n return object.__dir__(self)\n\n def _get_cdc_props(self):\n return self.execute_script(\n \"\"\"\n let objectToInspect = window,\n result = [];\n while(objectToInspect !== null)\n { result = result.concat(Object.getOwnPropertyNames(objectToInspect));\n objectToInspect = Object.getPrototypeOf(objectToInspect); }\n return result.filter(i => i.match(/.+_.+_(Array|Promise|Symbol)/ig))\n \"\"\"\n )\n\n def _hook_remove_cdc_props(self):\n # 它可以让当前标签页打开的所有网页，在网页内容加载之前执行一段 JavaScript 代码\n self.execute_cdp_cmd(\n \"Page.addScriptToEvaluateOnNewDocument\",\n {\n \"source\": \"\"\"\n let objectToInspect = window,\n result = [];\n while(objectToInspect !== null) \n { result = result.concat(Object.getOwnPropertyNames(objectToInspect));\n objectToInspect = Object.getPrototypeOf(objectToInspect); }\n result.forEach(p => p.match(/.+_.+_(Array|Promise|Symbol)/ig)\n &&delete window[p]&&console.log('removed',p))\n \"\"\"\n },\n )\n\n def get(self, url):\n if self._get_cdc_props():\n self._hook_remove_cdc_props()\n return super().get(url)\n\n def add_cdp_listener(self, event_name, callback):\n if (\n self.reactor\n and self.reactor is not None\n and isinstance(self.reactor, Reactor)\n ):\n self.reactor.add_event_handler(event_name, callback)\n return self.reactor.handlers\n return False\n\n def clear_cdp_listeners(self):\n if self.reactor and isinstance(self.reactor, Reactor):\n self.reactor.handlers.clear()\n\n def tab_new(self, url: str):\n \"\"\"\n this opens a url in a new tab.\n apparently, that passes all tests directly!\n\n Parameters\n ----------\n url\n\n Returns\n -------\n\n \"\"\"\n if not hasattr(self, \"cdp\"):\n from .cdp import CDP\n\n cdp = CDP(self.options)\n cdp.tab_new(url)\n\n def reconnect(self, timeout=0.1):\n try:\n self.service.stop()\n except Exception as e:\n logger.debug(e)\n time.sleep(timeout)\n try:\n self.service.start()\n except Exception as e:\n logger.debug(e)\n\n try:\n self.start_session()\n except Exception as e:\n logger.debug(e)\n\n def start_session(self, capabilities=None, browser_profile=None):\n if not capabilities:\n capabilities = self.options.to_capabilities()\n super(selenium.webdriver.chrome.webdriver.WebDriver, self).start_session(\n capabilities, browser_profile\n )\n # super(Chrome, self).start_session(capabilities, browser_profile)\n\n def quit(self):\n logger.debug(\"closing webdriver\")\n if hasattr(self, \"service\") and getattr(self.service, \"process\", None):\n self.service.process.kill()\n try:\n if self.reactor and isinstance(self.reactor, Reactor):\n logger.debug(\"shutting down reactor\")\n self.reactor.event.set()\n except Exception: # noqa\n pass\n try:\n logger.debug(\"killing browser\")\n os.kill(self.browser_pid, 15)\n\n except TimeoutError as e:\n logger.debug(e, exc_info=True)\n except Exception: # noqa\n pass\n\n if (\n hasattr(self, \"keep_user_data_dir\")\n and hasattr(self, \"user_data_dir\")\n and not self.keep_user_data_dir\n ):\n for _ in range(5):\n try:\n\n shutil.rmtree(self.user_data_dir, ignore_errors=False)\n except FileNotFoundError:\n pass\n except (RuntimeError, OSError, PermissionError) as e:\n logger.debug(\n \"When removing the temp profile, a %s occured: %s\\nretrying...\"\n % (e.__class__.__name__, e)\n )\n else:\n logger.debug(\"successfully removed %s\" % self.user_data_dir)\n break\n time.sleep(0.1)\n\n # dereference patcher, so patcher can start cleaning up as well.\n # this must come last, otherwise it will throw 'in use' errors\n self.patcher = None\n\n def __del__(self):\n try:\n super().quit()\n # self.service.process.kill()\n except: # noqa\n pass\n self.quit()\n\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc_val, exc_tb):\n self.service.stop()\n time.sleep(self._delay)\n self.service.start()\n self.start_session()\n\n def __hash__(self):\n return hash(self.options.debugger_address)\n\n\ndef find_chrome_executable():\n \"\"\"\n Finds the chrome, chrome beta, chrome canary, chromium executable\n\n Returns\n -------\n executable_path : str\n the full file path to found executable\n\n \"\"\"\n candidates = set()\n if IS_POSIX:\n for item in os.environ.get(\"PATH\").split(os.pathsep):\n for subitem in (\n \"google-chrome\",\n \"chromium\",\n \"chromium-browser\",\n \"chrome\",\n \"google-chrome-stable\",\n ):\n candidates.add(os.sep.join((item, subitem)))\n if \"darwin\" in sys.platform:\n candidates.update(\n [\n \"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome\",\n \"/Applications/Chromium.app/Contents/MacOS/Chromium\",\n ]\n )\n else:\n for item in map(\n os.environ.get, (\"PROGRAMFILES\", \"PROGRAMFILES(X86)\", \"LOCALAPPDATA\")\n ):\n for subitem in (\n \"Google/Chrome/Application\",\n \"Google/Chrome Beta/Application\",\n \"Google/Chrome Canary/Application\",\n ):\n candidates.add(os.sep.join((item, subitem, \"chrome.exe\")))\n for candidate in candidates:\n if os.path.exists(candidate) and os.access(candidate, os.X_OK):\n return os.path.normpath(candidate)\n" }, { "path": "undetected_chromedriver/_compat.py", "content": "#!/usr/bin/env python3\n# this module is part of undetected_chromedriver\n\n\n\"\"\"\n\n 888 888 d8b\n 888 888 Y8P\n 888 888\n .d8888b 88888b. 888d888 .d88b. 88888b.d88b. .d88b. .d88888 888d888 888 888 888 .d88b. 888d888\nd88P\" 888 \"88b 888P\" d88\"\"88b 888 \"888 \"88b d8P Y8b d88\" 888 888P\" 888 888 888 d8P Y8b 888P\"\n888 888 888 888 888 888 888 888 888 88888888 888 888 888 888 Y88 88P 88888888 888\nY88b. 888 888 888 Y88..88P 888 888 888 Y8b. Y88b 888 888 888 Y8bd8P Y8b. 888\n \"Y8888P 888 888 888 \"Y88P\" 888 888 888 \"Y8888 \"Y88888 888 888 Y88P \"Y8888 888 88888888\n\nby UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)\n\n\"\"\"\n\nimport io\nimport logging\nimport os\nimport random\nimport re\nimport string\nimport sys\nimport zipfile\nfrom distutils.version import LooseVersion\nfrom urllib.request import urlopen, urlretrieve\n\nfrom selenium.webdriver import Chrome as _Chrome, ChromeOptions as _ChromeOptions\n\nTARGET_VERSION = 0\nlogger = logging.getLogger(\"uc\")\n\n\nclass Chrome:\n def __new__(cls, *args, emulate_touch=False, **kwargs):\n\n if not ChromeDriverManager.installed:\n ChromeDriverManager(*args, **kwargs).install()\n if not ChromeDriverManager.selenium_patched:\n ChromeDriverManager(*args, **kwargs).patch_selenium_webdriver()\n if not kwargs.get(\"executable_path\"):\n kwargs[\"executable_path\"] = \"./{}\".format(\n ChromeDriverManager(*args, **kwargs).executable_path\n )\n if not kwargs.get(\"options\"):\n kwargs[\"options\"] = ChromeOptions()\n instance = object.__new__(_Chrome)\n instance.__init__(*args, **kwargs)\n\n instance._orig_get = instance.get\n\n def _get_wrapped(*args, **kwargs):\n if instance.execute_script(\"return navigator.webdriver\"):\n instance.execute_cdp_cmd(\n \"Page.addScriptToEvaluateOnNewDocument\",\n {\n \"source\": \"\"\"\n\n Object.defineProperty(window, 'navigator', {\n value: new Proxy(navigator, {\n has: (target, key) => (key === 'webdriver' ? false : key in target),\n get: (target, key) =>\n key === 'webdriver'\n ? undefined\n : typeof target[key] === 'function'\n ? target[key].bind(target)\n : target[key]\n })\n });\n \n \n \"\"\"\n },\n )\n return instance._orig_get(*args, **kwargs)\n\n instance.get = _get_wrapped\n instance.get = _get_wrapped\n instance.get = _get_wrapped\n\n original_user_agent_string = instance.execute_script(\n \"return navigator.userAgent\"\n )\n instance.execute_cdp_cmd(\n \"Network.setUserAgentOverride\",\n {\n \"userAgent\": original_user_agent_string.replace(\"Headless\", \"\"),\n },\n )\n if emulate_touch:\n instance.execute_cdp_cmd(\n \"Page.addScriptToEvaluateOnNewDocument\",\n {\n \"source\": \"\"\"\n Object.defineProperty(navigator, 'maxTouchPoints', {\n get: () => 1\n })\"\"\"\n },\n )\n logger.info(f\"starting undetected_chromedriver2.Chrome({args}, {kwargs})\")\n return instance\n\n\nclass ChromeOptions:\n def __new__(cls, *args, **kwargs):\n if not ChromeDriverManager.installed:\n ChromeDriverManager(*args, **kwargs).install()\n if not ChromeDriverManager.selenium_patched:\n ChromeDriverManager(*args, **kwargs).patch_selenium_webdriver()\n\n instance = object.__new__(_ChromeOptions)\n instance.__init__()\n instance.add_argument(\"start-maximized\")\n instance.add_experimental_option(\"excludeSwitches\", [\"enable-automation\"])\n instance.add_argument(\"--disable-blink-features=AutomationControlled\")\n return instance\n\n\nclass ChromeDriverManager(object):\n installed = False\n selenium_patched = False\n target_version = None\n\n DL_BASE = \"https://chromedriver.storage.googleapis.com/\"\n\n def __init__(self, executable_path=None, target_version=None, *args, **kwargs):\n\n _platform = sys.platform\n\n if TARGET_VERSION:\n # use global if set\n self.target_version = TARGET_VERSION\n\n if target_version:\n # use explicitly passed target\n self.target_version = target_version # user override\n\n if not self.target_version:\n # none of the above (default) and just get current version\n self.target_version = self.get_release_version_number().version[\n 0\n ] # only major version int\n\n self._base = base_ = \"chromedriver{}\"\n\n exe_name = self._base\n if _platform in (\"win32\",):\n exe_name = base_.format(\".exe\")\n if _platform in (\"linux\",):\n _platform += \"64\"\n exe_name = exe_name.format(\"\")\n if _platform in (\"darwin\",):\n _platform = \"mac64\"\n exe_name = exe_name.format(\"\")\n self.platform = _platform\n self.executable_path = executable_path or exe_name\n self._exe_name = exe_name\n\n def patch_selenium_webdriver(self_):\n \"\"\"\n Patches selenium package Chrome, ChromeOptions classes for current session\n\n :return:\n \"\"\"\n import selenium.webdriver.chrome.service\n import selenium.webdriver\n\n selenium.webdriver.Chrome = Chrome\n selenium.webdriver.ChromeOptions = ChromeOptions\n logger.info(\"Selenium patched. Safe to import Chrome / ChromeOptions\")\n self_.__class__.selenium_patched = True\n\n def install(self, patch_selenium=True):\n \"\"\"\n Initialize the patch\n\n This will:\n download chromedriver if not present\n patch the downloaded chromedriver\n patch selenium package if is True (default)\n\n :param patch_selenium: patch selenium webdriver classes for Chrome and ChromeDriver (for current python session)\n :return:\n \"\"\"\n if not os.path.exists(self.executable_path):\n self.fetch_chromedriver()\n if not self.__class__.installed:\n if self.patch_binary():\n self.__class__.installed = True\n\n if patch_selenium:\n self.patch_selenium_webdriver()\n\n def get_release_version_number(self):\n \"\"\"\n Gets the latest major version available, or the latest major version of self.target_version if set explicitly.\n\n :return: version string\n \"\"\"\n path = (\n \"LATEST_RELEASE\"\n if not self.target_version\n else f\"LATEST_RELEASE_{self.target_version}\"\n )\n return LooseVersion(urlopen(self.__class__.DL_BASE + path).read().decode())\n\n def fetch_chromedriver(self):\n \"\"\"\n Downloads ChromeDriver from source and unpacks the executable\n\n :return: on success, name of the unpacked executable\n \"\"\"\n base_ = self._base\n zip_name = base_.format(\".zip\")\n ver = self.get_release_version_number().vstring\n if os.path.exists(self.executable_path):\n return self.executable_path\n urlretrieve(\n f\"{self.__class__.DL_BASE}{ver}/{base_.format(f'_{self.platform}')}.zip\",\n filename=zip_name,\n )\n with zipfile.ZipFile(zip_name) as zf:\n zf.extract(self._exe_name)\n os.remove(zip_name)\n if sys.platform != \"win32\":\n os.chmod(self._exe_name, 0o755)\n return self._exe_name\n\n @staticmethod\n def random_cdc():\n cdc = random.choices(string.ascii_lowercase, k=26)\n cdc[-6:-4] = map(str.upper, cdc[-6:-4])\n cdc[2] = cdc[0]\n cdc[3] = \"_\"\n return \"\".join(cdc).encode()\n\n def patch_binary(self):\n \"\"\"\n Patches the ChromeDriver binary\n\n :return: False on failure, binary name on success\n \"\"\"\n linect = 0\n replacement = self.random_cdc()\n with io.open(self.executable_path, \"r+b\") as fh:\n for line in iter(lambda: fh.readline(), b\"\"):\n if b\"cdc_\" in line:\n fh.seek(-len(line), 1)\n newline = re.sub(b\"cdc_.{22}\", replacement, line)\n fh.write(newline)\n linect += 1\n return linect\n\n\ndef install(executable_path=None, target_version=None, *args, **kwargs):\n ChromeDriverManager(executable_path, target_version, *args, **kwargs).install()\n" }, { "path": "undetected_chromedriver/cdp.py", "content": "#!/usr/bin/env python3\n# this module is part of undetected_chromedriver\n\nimport json\nimport logging\nfrom collections.abc import Mapping, Sequence\nimport requests\nimport websockets\n\nlog = logging.getLogger(__name__)\n\n\nclass CDPObject(dict):\n def __init__(self, *a, **k):\n super().__init__(*a, **k)\n self.__dict__ = self\n for k in self.__dict__:\n if isinstance(self.__dict__[k], dict):\n self.__dict__[k] = CDPObject(self.__dict__[k])\n elif isinstance(self.__dict__[k], list):\n for i in range(len(self.__dict__[k])):\n if isinstance(self.__dict__[k][i], dict):\n self.__dict__[k][i] = CDPObject(self)\n\n def __repr__(self):\n tpl = f\"{self.__class__.__name__}(\\n\\t{{}}\\n\\t)\"\n return tpl.format(\"\\n \".join(f\"{k} = {v}\" for k, v in self.items()))\n\n\nclass PageElement(CDPObject):\n pass\n\n\nclass CDP:\n log = logging.getLogger(\"CDP\")\n\n endpoints = CDPObject(\n {\n \"json\": \"/json\",\n \"protocol\": \"/json/protocol\",\n \"list\": \"/json/list\",\n \"new\": \"/json/new?{url}\",\n \"activate\": \"/json/activate/{id}\",\n \"close\": \"/json/close/{id}\",\n }\n )\n\n def __init__(self, options: \"ChromeOptions\"): # noqa\n self.server_addr = \"http://{0}:{1}\".format(*options.debugger_address.split(\":\"))\n\n self._reqid = 0\n self._session = requests.Session()\n self._last_resp = None\n self._last_json = None\n\n resp = self.get(self.endpoints.json) # noqa\n self.sessionId = resp[0][\"id\"]\n self.wsurl = resp[0][\"webSocketDebuggerUrl\"]\n\n def tab_activate(self, id=None):\n if not id:\n active_tab = self.tab_list()[0]\n id = active_tab.id # noqa\n self.wsurl = active_tab.webSocketDebuggerUrl # noqa\n return self.post(self.endpoints[\"activate\"].format(id=id))\n\n def tab_list(self):\n retval = self.get(self.endpoints[\"list\"])\n return [PageElement(o) for o in retval]\n\n def tab_new(self, url):\n return self.post(self.endpoints[\"new\"].format(url=url))\n\n def tab_close_last_opened(self):\n sessions = self.tab_list()\n opentabs = [s for s in sessions if s[\"type\"] == \"page\"]\n return self.post(self.endpoints[\"close\"].format(id=opentabs[-1][\"id\"]))\n\n async def send(self, method: str, params: dict):\n self._reqid += 1\n async with websockets.connect(self.wsurl) as ws:\n await ws.send(\n json.dumps({\"method\": method, \"params\": params, \"id\": self._reqid})\n )\n self._last_resp = await ws.recv()\n self._last_json = json.loads(self._last_resp)\n self.log.info(self._last_json)\n\n def get(self, uri):\n resp = self._session.get(self.server_addr + uri)\n try:\n self._last_resp = resp\n self._last_json = resp.json()\n except Exception:\n return\n else:\n return self._last_json\n\n def post(self, uri, data: dict = None):\n if not data:\n data = {}\n resp = self._session.post(self.server_addr + uri, json=data)\n try:\n self._last_resp = resp\n self._last_json = resp.json()\n except Exception:\n return self._last_resp\n\n @property\n def last_json(self):\n return self._last_json\n" }, { "path": "undetected_chromedriver/devtool.py", "content": "import asyncio\nimport logging\nimport time\nimport traceback\nfrom collections.abc import Mapping\nfrom collections.abc import Sequence\nfrom typing import Any\nfrom typing import Awaitable\nfrom typing import Callable\nfrom typing import List\nfrom typing import Optional\nfrom contextlib import ExitStack\nimport threading\nfrom functools import wraps, partial\n\n\nclass Structure(dict):\n \"\"\"\n This is a dict-like object structure, which you should subclass\n Only properties defined in the class context are used on initialization.\n\n See example\n \"\"\"\n\n _store = {}\n\n def __init__(self, *a, **kw):\n \"\"\"\n Instantiate a new instance.\n\n :param a:\n :param kw:\n \"\"\"\n\n super().__init__()\n\n # auxiliar dict\n d = dict(*a, **kw)\n for k, v in d.items():\n if isinstance(v, Mapping):\n self[k] = self.__class__(v)\n elif isinstance(v, Sequence) and not isinstance(v, (str, bytes)):\n self[k] = [self.__class__(i) for i in v]\n else:\n self[k] = v\n super().__setattr__(\"__dict__\", self)\n\n def __getattr__(self, item):\n return getattr(super(), item)\n\n def __getitem__(self, item):\n return super().__getitem__(item)\n\n def __setattr__(self, key, value):\n self.__setitem__(key, value)\n\n def __setitem__(self, key, value):\n super().__setitem__(key, value)\n\n def update(self, *a, **kw):\n super().update(*a, **kw)\n\n def __eq__(self, other):\n return frozenset(other.items()) == frozenset(self.items())\n\n def __hash__(self):\n return hash(frozenset(self.items()))\n\n @classmethod\n def __init_subclass__(cls, **kwargs):\n cls._store = {}\n\n def _normalize_strings(self):\n for k, v in self.copy().items():\n if isinstance(v, (str)):\n self[k] = v.strip()\n\n\ndef timeout(seconds=3, on_timeout: Optional[Callable[[callable], Any]] = None):\n def wrapper(func):\n @wraps(func)\n def wrapped(*args, **kwargs):\n def function_reached_timeout():\n if on_timeout:\n on_timeout(func)\n else:\n raise TimeoutError(\"function call timed out\")\n\n t = threading.Timer(interval=seconds, function=function_reached_timeout)\n t.start()\n try:\n return func(*args, **kwargs)\n except:\n t.cancel()\n raise\n finally:\n t.cancel()\n\n return wrapped\n\n return wrapper\n\n\ndef test():\n import sys, os\n\n sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))\n import undetected_chromedriver as uc\n import threading\n\n def collector(\n driver: uc.Chrome,\n stop_event: threading.Event,\n on_event_coro: Optional[Callable[[List[str]], Awaitable[Any]]] = None,\n listen_events: Sequence = (\"browser\", \"network\", \"performance\"),\n ):\n def threaded(driver, stop_event, on_event_coro):\n async def _ensure_service_started():\n while (\n getattr(driver, \"service\", False)\n and getattr(driver.service, \"process\", False)\n and driver.service.process.poll()\n ):\n print(\"waiting for driver service to come back on\")\n await asyncio.sleep(0.05)\n # await asyncio.sleep(driver._delay or .25)\n\n async def get_log_lines(typ):\n await _ensure_service_started()\n return driver.get_log(typ)\n\n async def looper():\n while not stop_event.is_set():\n log_lines = []\n try:\n for _ in listen_events:\n try:\n log_lines += await get_log_lines(_)\n except:\n if logging.getLogger().getEffectiveLevel() <= 10:\n traceback.print_exc()\n continue\n if log_lines and on_event_coro:\n await on_event_coro(log_lines)\n except Exception as e:\n if logging.getLogger().getEffectiveLevel() <= 10:\n traceback.print_exc()\n\n loop = asyncio.new_event_loop()\n asyncio.set_event_loop(loop)\n loop.run_until_complete(looper())\n\n t = threading.Thread(target=threaded, args=(driver, stop_event, on_event_coro))\n t.start()\n\n async def on_event(data):\n print(\"on_event\")\n print(\"data:\", data)\n\n def func_called(fn):\n def wrapped(*args, **kwargs):\n print(\n \"func called! %s (args: %s, kwargs: %s)\" % (fn.__name__, args, kwargs)\n )\n while driver.service.process and driver.service.process.poll() is not None:\n time.sleep(0.1)\n res = fn(*args, **kwargs)\n print(\"func completed! (result: %s)\" % res)\n return res\n\n return wrapped\n\n logging.basicConfig(level=10)\n\n options = uc.ChromeOptions()\n options.set_capability(\n \"goog:loggingPrefs\", {\"performance\": \"ALL\", \"browser\": \"ALL\", \"network\": \"ALL\"}\n )\n\n driver = uc.Chrome(version_main=96, options=options)\n\n # driver.command_executor._request = timeout(seconds=1)(driver.command_executor._request)\n driver.command_executor._request = func_called(driver.command_executor._request)\n collector_stop = threading.Event()\n collector(driver, collector_stop, on_event)\n\n driver.get(\"https://nowsecure.nl\")\n\n time.sleep(10)\n\n driver.quit()\n" }, { "path": "undetected_chromedriver/dprocess.py", "content": "import multiprocessing\nimport os\nimport platform\nimport sys\nfrom subprocess import PIPE\nfrom subprocess import Popen\nimport atexit\nimport traceback\nimport logging\nimport signal\n\nCREATE_NEW_PROCESS_GROUP = 0x00000200\nDETACHED_PROCESS = 0x00000008\n\nREGISTERED = []\n\n\ndef start_detached(executable, *args):\n \"\"\"\n Starts a fully independent subprocess (with no parent)\n :param executable: executable\n :param args: arguments to the executable, eg: ['--param1_key=param1_val', '-vvv' ...]\n :return: pid of the grandchild process\n 启动独立的子进程\n \"\"\"\n\n # create pipe\n reader, writer = multiprocessing.Pipe(False)\n\n # do not keep reference\n multiprocessing.Process(\n target=_start_detached,\n args=(executable, *args),\n kwargs={\"writer\": writer},\n daemon=True,\n ).start()\n # receive pid from pipe\n pid = reader.recv()\n REGISTERED.append(pid)\n # close pipes\n writer.close()\n reader.close()\n\n return pid\n\n\ndef _start_detached(executable, *args, writer: multiprocessing.Pipe = None):\n\n # configure launch\n kwargs = {}\n if platform.system() == \"Windows\":\n kwargs.update(creationflags=DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP)\n elif sys.version_info < (3, 2):\n # assume posix\n kwargs.update(preexec_fn=os.setsid)\n else: # Python 3.2+ and Unix\n kwargs.update(start_new_session=True)\n\n # run\n p = Popen([executable, *args], stdin=PIPE, stdout=PIPE, stderr=PIPE, **kwargs)\n\n # send pid to pipe\n writer.send(p.pid)\n sys.exit()\n\n\ndef _cleanup():\n for pid in REGISTERED:\n try:\n logging.getLogger(__name__).debug(\"cleaning up pid %d \" % pid)\n os.kill(pid, signal.SIGTERM)\n except: # noqa\n pass\n\n\natexit.register(_cleanup)\n" }, { "path": "undetected_chromedriver/options.py", "content": "#!/usr/bin/env python3\n# this module is part of undetected_chromedriver\n\n\nimport json\nimport os\ntry:\n from selenium.webdriver.chromium.options import ChromiumOptions as _ChromiumOptions\nexcept:\n from selenium.webdriver.chrome.options import Options as _ChromiumOptions\n\nclass ChromeOptions(_ChromiumOptions):\n _session = None\n _user_data_dir = None\n\n @property\n def user_data_dir(self):\n return self._user_data_dir\n\n @user_data_dir.setter\n def user_data_dir(self, path: str):\n \"\"\"\n Sets the browser profile folder to use, or creates a new profile\n at given .\n\n Parameters\n ----------\n path: str\n the path to a chrome profile folder\n if it does not exist, a new profile will be created at given location\n 设置要使用的浏览器配置文件文件夹，或创建新的配置文件\n \"\"\"\n apath = os.path.abspath(path)\n self._user_data_dir = os.path.normpath(apath)\n\n @staticmethod\n def _undot_key(key, value):\n \"\"\"turn a (dotted key, value) into a proper nested dict\"\"\"\n if \".\" in key:\n key, rest = key.split(\".\", 1)\n value = ChromeOptions._undot_key(rest, value)\n return {key: value}\n\n def handle_prefs(self, user_data_dir):\n prefs = self.experimental_options.get(\"prefs\")\n if prefs:\n user_data_dir = user_data_dir or self._user_data_dir\n default_path = os.path.join(user_data_dir, \"Default\")\n os.makedirs(default_path, exist_ok=True)\n\n # undot prefs dict keys\n undot_prefs = {}\n for key, value in prefs.items():\n undot_prefs.update(self._undot_key(key, value))\n\n prefs_file = os.path.join(default_path, \"Preferences\")\n if os.path.exists(prefs_file):\n with open(prefs_file, encoding=\"latin1\", mode=\"r\") as f:\n undot_prefs.update(json.load(f))\n\n with open(prefs_file, encoding=\"latin1\", mode=\"w\") as f:\n json.dump(undot_prefs, f)\n\n # remove the experimental_options to avoid an error\n del self._experimental_options[\"prefs\"]\n\n @classmethod\n def from_options(cls, options):\n o = cls()\n o.__dict__.update(options.__dict__)\n return o\n" }, { "path": "undetected_chromedriver/patcher.py", "content": "#!/usr/bin/env python3\n# this module is part of undetected_chromedriver\n\nimport io\nimport logging\nimport os\nimport random\nimport re\nimport string\nimport sys\nimport time\nimport zipfile\nfrom distutils.version import LooseVersion\nfrom urllib.request import urlopen, urlretrieve\nimport secrets\n\n\nlogger = logging.getLogger(__name__)\n\nIS_POSIX = sys.platform.startswith((\"darwin\", \"cygwin\", \"linux\"))\n\n\nclass Patcher(object):\n \"\"\"\n 获取webdriver最新版本\n \"\"\"\n url_repo = \"https://chromedriver.storage.googleapis.com\"\n zip_name = \"chromedriver_%s.zip\"\n exe_name = \"chromedriver%s\"\n # 判断当前系统\n platform = sys.platform\n if platform.endswith(\"win32\"):\n zip_name %= \"win32\"\n exe_name %= \".exe\"\n if platform.endswith(\"linux\"):\n zip_name %= \"linux64\"\n exe_name %= \"\"\n if platform.endswith(\"darwin\"):\n zip_name %= \"mac64\"\n exe_name %= \"\"\n\n if platform.endswith(\"win32\"):\n d = \"~/appdata/roaming/undetected_chromedriver\"\n elif platform.startswith(\"linux\"):\n d = \"~/.local/share/undetected_chromedriver\"\n elif platform.endswith(\"darwin\"):\n d = \"~/Library/Application Support/undetected_chromedriver\"\n else:\n d = \"~/.undetected_chromedriver\"\n data_path = os.path.abspath(os.path.expanduser(d))\n\n def __init__(self, executable_path=None, force=False, version_main: int = 0):\n \"\"\"\n \n Args:\n executable_path: None = automatic\n a full file path to the chromedriver executable\n force: False\n terminate processes which are holding lock\n version_main: 0 = auto\n specify main chrome version (rounded, ex: 82)\n \"\"\"\n\n self.force = force\n self.executable_path = None\n prefix = secrets.token_hex(8)\n\n if not os.path.exists(self.data_path):\n os.makedirs(self.data_path, exist_ok=True)\n\n if not executable_path:\n self.executable_path = os.path.join(\n self.data_path, \"_\".join([prefix, self.exe_name])\n )\n\n if not IS_POSIX:\n if executable_path:\n if not executable_path[-4:] == \".exe\":\n executable_path += \".exe\"\n\n self.zip_path = os.path.join(self.data_path, prefix)\n\n if not executable_path:\n self.executable_path = os.path.abspath(\n os.path.join(\".\", self.executable_path)\n )\n\n self._custom_exe_path = False\n\n if executable_path:\n self._custom_exe_path = True\n self.executable_path = executable_path\n self.version_main = version_main\n self.version_full = None\n\n def auto(self, executable_path=None, force=False, version_main=None):\n \"\"\"\"\"\"\n if executable_path:\n self.executable_path = executable_path\n self._custom_exe_path = True\n\n if self._custom_exe_path:\n ispatched = self.is_binary_patched(self.executable_path)\n if not ispatched:\n return self.patch_exe()\n else:\n return\n\n if version_main:\n self.version_main = version_main\n if force is True:\n self.force = force\n\n try:\n os.unlink(self.executable_path)\n except PermissionError:\n if self.force:\n self.force_kill_instances(self.executable_path)\n return self.auto(force=not self.force)\n try:\n if self.is_binary_patched():\n # assumes already running AND patched\n return True\n except PermissionError:\n pass\n # return False\n except FileNotFoundError:\n pass\n\n release = self.fetch_release_number()\n self.version_main = release.version[0]\n self.version_full = release\n self.unzip_package(self.fetch_package())\n return self.patch()\n\n def patch(self):\n self.patch_exe()\n return self.is_binary_patched()\n\n def fetch_release_number(self):\n \"\"\"\n Gets the latest major version available, or the latest major version of self.target_version if set explicitly.\n :return: version string\n :rtype: LooseVersion\n 获取可用的最新版\n \"\"\"\n path = \"/latest_release\"\n if self.version_main:\n path += f\"_{self.version_main}\"\n path = path.upper()\n logger.debug(\"getting release number from %s\" % path)\n return LooseVersion(urlopen(self.url_repo + path).read().decode())\n\n def parse_exe_version(self):\n with io.open(self.executable_path, \"rb\") as f:\n for line in iter(lambda: f.readline(), b\"\"):\n match = re.search(rb\"platform_handle\\x00content\\x00([0-9.]*)\", line)\n if match:\n return LooseVersion(match[1].decode())\n\n def fetch_package(self):\n \"\"\"\n Downloads ChromeDriver from source\n\n :return: path to downloaded file\n \"\"\"\n u = \"%s/%s/%s\" % (self.url_repo, self.version_full.vstring, self.zip_name)\n logger.debug(\"downloading from %s\" % u)\n # return urlretrieve(u, filename=self.data_path)[0]\n return urlretrieve(u)[0]\n\n def unzip_package(self, fp):\n \"\"\"\n Does what it says\n\n :return: path to unpacked executable\n 解压缩可执行文件\n \"\"\"\n logger.debug(\"unzipping %s\" % fp)\n try:\n os.unlink(self.zip_path)\n except (FileNotFoundError, OSError):\n pass\n \n os.makedirs(self.zip_path, mode=0o755, exist_ok=True)\n with zipfile.ZipFile(fp, mode=\"r\") as zf:\n zf.extract(self.exe_name, self.zip_path)\n os.rename(os.path.join(self.zip_path, self.exe_name), self.executable_path)\n os.remove(fp)\n os.rmdir(self.zip_path)\n os.chmod(self.executable_path, 0o755)\n return self.executable_path\n\n @staticmethod\n def force_kill_instances(exe_name):\n \"\"\"\n kills running instances.\n :param: executable name to kill, may be a path as well\n\n :return: True on success else False\n 通过进程号kill driver\n \"\"\"\n exe_name = os.path.basename(exe_name)\n if IS_POSIX:\n r = os.system(\"kill -f -9 $(pidof %s)\" % exe_name)\n else:\n r = os.system(\"taskkill /f /im %s\" % exe_name)\n return not r\n\n @staticmethod\n def gen_random_cdc():\n cdc = random.choices(string.ascii_lowercase, k=26)\n cdc[-6:-4] = map(str.upper, cdc[-6:-4])\n cdc[2] = cdc[0]\n cdc[3] = \"_\"\n return \"\".join(cdc).encode()\n\n def is_binary_patched(self, executable_path=None):\n \"\"\"simple check if executable is patched.\n\n :return: False if not patched, else True\n 检查可执行文件补丁\n \"\"\"\n executable_path = executable_path or self.executable_path\n with io.open(executable_path, \"rb\") as fh:\n for line in iter(lambda: fh.readline(), b\"\"):\n if b\"cdc_\" in line:\n return False\n else:\n return True\n\n def patch_exe(self):\n \"\"\"\n Patches the ChromeDriver binary\n\n :return: False on failure, binary name on success\n \"\"\"\n logger.info(\"patching driver executable %s\" % self.executable_path)\n\n linect = 0\n replacement = self.gen_random_cdc()\n with io.open(self.executable_path, \"r+b\") as fh:\n for line in iter(lambda: fh.readline(), b\"\"):\n if b\"cdc_\" in line:\n fh.seek(-len(line), 1)\n newline = re.sub(b\"cdc_.{22}\", replacement, line)\n fh.write(newline)\n linect += 1\n return linect\n\n def __repr__(self):\n return \"{0:s}({1:s})\".format(\n self.__class__.__name__,\n self.executable_path,\n )\n\n def __del__(self):\n\n if self._custom_exe_path:\n # if the driver binary is specified by user\n # we assume it is important enough to not delete it\n return\n else:\n timeout = 3 # stop trying after this many seconds\n t = time.monotonic()\n while True:\n now = time.monotonic()\n if now - t > timeout:\n # we don't want to wait until the end of time\n logger.debug(\n \"could not unlink %s in time (%d seconds)\"\n % (self.executable_path, timeout)\n )\n break\n try:\n os.unlink(self.executable_path)\n logger.debug(\"successfully unlinked %s\" % self.executable_path)\n break\n except (OSError, RuntimeError, PermissionError):\n time.sleep(0.1)\n continue\n except FileNotFoundError:\n break\n" }, { "path": "undetected_chromedriver/reactor.py", "content": "#!/usr/bin/env python3\n# this module is part of undetected_chromedriver\n\nimport asyncio\nimport json\nimport logging\nimport threading\n\nlogger = logging.getLogger(__name__)\n\n\nclass Reactor(threading.Thread):\n \"\"\"\n 异步事件处理\n \"\"\"\n def __init__(self, driver: \"Chrome\"):\n super().__init__()\n\n self.driver = driver\n self.loop = asyncio.new_event_loop()\n\n self.lock = threading.Lock()\n self.event = threading.Event()\n self.daemon = True\n self.handlers = {}\n\n def add_event_handler(self, method_name, callback: callable):\n \"\"\"\n\n Parameters\n ----------\n event_name: str\n example \"Network.responseReceived\"\n\n callback: callable\n callable which accepts 1 parameter: the message object dictionary\n\n Returns\n -------\n\n \"\"\"\n with self.lock:\n self.handlers[method_name.lower()] = callback\n\n @property\n def running(self):\n return not self.event.is_set()\n\n def run(self):\n try:\n asyncio.set_event_loop(self.loop)\n self.loop.run_until_complete(self.listen())\n except Exception as e:\n logger.warning(\"Reactor.run() => %s\", e)\n\n async def _wait_service_started(self):\n while True:\n with self.lock:\n if (\n getattr(self.driver, \"service\", None)\n and getattr(self.driver.service, \"process\", None)\n and self.driver.service.process.poll()\n ):\n await asyncio.sleep(self.driver._delay or 0.25)\n else:\n break\n\n async def listen(self):\n\n while self.running:\n\n await self._wait_service_started()\n await asyncio.sleep(1)\n\n try:\n with self.lock:\n log_entries = self.driver.get_log(\"performance\")\n\n for entry in log_entries:\n\n try:\n obj_serialized: str = entry.get(\"message\")\n obj = json.loads(obj_serialized)\n message = obj.get(\"message\")\n method = message.get(\"method\")\n\n if \"*\" in self.handlers:\n await self.loop.run_in_executor(\n None, self.handlers[\"*\"], message\n )\n elif method.lower() in self.handlers:\n await self.loop.run_in_executor(\n None, self.handlers[method.lower()], message\n )\n\n # print(type(message), message)\n except Exception as e:\n raise e from None\n\n except Exception as e:\n if \"invalid session id\" in str(e):\n pass\n else:\n logging.debug(\"exception ignored :\", e)\n" }, { "path": "undetected_chromedriver/v2.py", "content": "# for backward compatibility\nimport sys\n\nsys.modules[__name__] = sys.modules[__package__]\n" }, { "path": "undetected_chromedriver/webelement.py", "content": "import selenium.webdriver.remote.webelement\n\n\nclass WebElement(selenium.webdriver.remote.webelement.WebElement):\n \"\"\"\n Custom WebElement class which makes it easier to view elements when\n working in an interactive environment.\n\n standard webelement repr:\n \n\n using this WebElement class:\n )>\n \n selenium.webdriver.remote.webelement.WebElement\n\n 自定义的WebElement类，WebElement类可以代表任何 Web 对象，是selenium中所有元素的父类,也就是webelement对象拥有的方法,其它元素对象都会有。\n 如 div、a标签。\n \n \"\"\"\n\n @property\n def attrs(self):\n \"\"\"\n attr:div:
\n \"\"\"\n if not hasattr(self, \"_attrs\"):\n self._attrs = self._parent.execute_script(\n \"\"\"\n var items = {}; \n for (index = 0; index < arguments[0].attributes.length; ++index) \n {\n items[arguments[0].attributes[index].name] = arguments[0].attributes[index].value \n }; \n return items;\n \"\"\",\n self,\n )\n return self._attrs\n\n def __repr__(self):\n strattrs = \" \".join([f'{k}=\"{v}\"' for k, v in self.attrs.items()])\n if strattrs:\n strattrs = \" \" + strattrs\n return f\"{self.__class__.__name__} <{self.tag_name}{strattrs}>\"\n" } ]