Repository: Karmenzind/fp-server Branch: master Commit: 931fca8fab9d Files: 74 Total size: 236.0 KB Directory structure: gitextract_rvw5g_7c/ ├── .github/ │ └── ISSUE_TEMPLATE.md ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── README_CN.md ├── examples/ │ ├── middleware_for_scrapy.md │ └── use_with_requests.md ├── requirements.txt └── src/ ├── api/ │ ├── __init__.py │ ├── ping/ │ │ ├── __init__.py │ │ └── ping.py │ ├── proxy/ │ │ ├── __init__.py │ │ └── proxy.py │ ├── status/ │ │ ├── __init__.py │ │ └── status.py │ └── test/ │ ├── __init__.py │ └── test.py ├── config/ │ ├── __init__.py │ └── config.yml ├── core/ │ ├── __init__.py │ ├── context.py │ ├── crawler.py │ ├── db/ │ │ ├── __init__.py │ │ ├── initial_works.py │ │ ├── mongo.py │ │ ├── mysql.py │ │ └── redis.py │ ├── exceptions.py │ ├── heartbeat.py │ ├── middleware.py │ └── web.py ├── main.py ├── proxy_spider/ │ ├── __init__.py │ ├── const.py │ ├── items.py │ ├── middlewares.py │ ├── pipelines.py │ ├── settings.py │ ├── spiders/ │ │ ├── __init__.py │ │ ├── a3464.py │ │ ├── checker.py │ │ ├── coderbusy.py │ │ ├── coolproxy.py │ │ ├── data5u.py │ │ ├── ihuan.py │ │ ├── ip66.py │ │ ├── ip89.py │ │ ├── kuaidaili.py │ │ ├── mix.py │ │ ├── xicidaili.py │ │ └── yundaili.py │ └── utils.py ├── scrapy.cfg ├── service/ │ ├── __init__.py │ ├── proxy/ │ │ ├── __init__.py │ │ ├── functions.py │ │ ├── proxy.py │ │ └── serializers.py │ ├── spider/ │ │ ├── __init__.py │ │ ├── functions.py │ │ └── spider.py │ └── tasks/ │ ├── __init__.py │ └── spider.py └── utils/ ├── __init__.py ├── collections.py ├── docker.py ├── http_client.py ├── log.py ├── routes.py ├── send_email.py ├── time_ext.py ├── tools.py └── validators.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/ISSUE_TEMPLATE.md ================================================ Your operating system and Python version? Have you followed the manual and FAQ? Do you run fp-server with Docker? Paste your console output here. It'll be great if you can provide **a log file** (set `CONSOLE_OUTPUT = 0` and then run the server for at least 10 minutes). ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ # custom for_todo.md ./src/.scrapy ./src/launch_spider.py .idea/ config.yml.local build.sh ================================================ FILE: Dockerfile ================================================ FROM karmenzind/fp-server:base WORKDIR /fp_server ADD ./src/ /fp_server CMD redis-server /etc/redis.conf && python3 main.py ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) 2018 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================ # fp-server :exclamation: **This code really sucks. I'll rewrite it when I have free time. 这项目写的很烂，仅供参考，后续有时间会重构……** ------ [![Scrutinizer Build](https://img.shields.io/scrutinizer/build/g/filp/whoops.svg?style=flat-square)](https://github.com/Karmenzind/fp-server) [![Python Version](https://img.shields.io/badge/python-3.5%2B-yellow.svg?style=flat-square)](https://www.python.org/) [![CocoaPods](https://img.shields.io/cocoapods/l/AFNetworking.svg?style=flat-square)](https://github.com/Karmenzind/fp-server) A free proxy server based on [Tornado](http://www.tornadoweb.org/en/stable/#) and [Scrapy](https://scrapy.org/). Build your own proxy pool! Features: - continuously crawling and providing free proxies - asynchronous and high-perfermance - automatically check proxies in cycle and ditch unavailable ones - easy-to-use HTTP api 免费代理服务器，基于[Tornado](http://www.tornadoweb.org/en/stable/#)和[Scrapy](https://scrapy.org/)，在本地搭建自己的代理池 - 持续爬取新的免费代理，检测可用后存入本地数据库 - 完全异步，支持高并发 - 易用的HTTP API - 周期性检测代理可用性，自动更新 [**查看中文文档\_(:ι」∠)\_**](./README_CN.md) This project has been tested on: - Archlinux; Python-3.6.5 - Debian(WSL, Raspbian); Python-3.5.3 And it **cannot directly run on Windows**. Windows users may try [using Docker](#using-docker) or WSL to run this project. ## Contents ## * [Get started](#get-started) * [Using Docker](#using-docker) * [Manually install](#manually-install) * [web APIs](#web-apis) * [get proxies](#get-proxies) * [create new proxy manually](#create-new-proxy-manually) * [check status](#check-status) * [Config](#config) * [Introduction](#introduction) * [Customization](#customization) * [Source webs](#source-webs) * [FAQ](#faq) * [Examples](#examples) * [Use fp-server with Python requests module](#use-fp-server-with-python-requests-module) * [Use fp-server in Scrapy Project](#use-fp-server-in-scrapy-project) * [Bugs and feature requests](#bugs-and-feature-requests) * [TODOs and ideas](#todos-and-ideas) ## Get started ## Choose either one option as follows. After successful deployment, use the [APIs](#web-apis) to get proxies. ### Using Docker ### The easiest way to run this repo is using [Docker](https://www.docker.com/). Install Docker and then run: ```bash # download the image docker pull karmenzind/fp-server:stable # run the container # don't forget to modify `-p` if you prefer another port docker run -itd --name fpserver -p 12345:12345 karmenzind/fp-server:stable # check the output inside the container docker logs -f fpserver ``` For custom configuratiuon, see [this section](#config). ### Manually install ### 1. Install [Redis](https://redis.io/) and `python>=3.5`(I use Python-3.6.5). 2. Clone this repo. 3. Install python packages by: ```bash pip install -r requirements.txt ``` 4. Read the [config](#config) and modify it according to your need. 5. Start the server: ```bash python ./src/main.py ``` ## web APIs ## typical response: ```json { "code": 0, "msg": "ok", "data": {} } ``` - code: result of event (not http code), 0 for sucess - msg: message for event - data: detail for sucessful event ### get proxies ### ``` GET /api/proxy/ ``` params | Must/
Optional | detail | default ------------------------|-------------------|----------------------------------------------------------------------|---------| count | O | the number of proxies you need | 1 scheme | O | choices:`HTTP` `HTTPS` | both* anonymity | O | choices:`transparent` `anonymous` | both (TODO)
sort_by_speed | O | choices:
1: desending order
0: no order
-1: ascending order | 0 - both: include all type, not grouped **example** - To acquire 10 proxies in HTTP scheme with anonymity: ``` GET /api/proxy/?count=10&scheme=HTTP&anonymity=anonymous ``` The response: ```json { "code": 0, "msg": "ok", "data": { "count": 9, "items": [ { "port": 2000, "ip": "xxx.xxx.xx.xxx", "scheme": "HTTP", "url": "http://xxx.xxx.xxx.xx:xxxx", "anonymity": "transparent" } ] } } ``` **screenshot** ![](https://raw.githubusercontent.com/Karmenzind/i/master/fp-server/proxy_get.png) ### create new proxy manually ### ``` POST /api/proxy/ ``` params | Must/
Optional | detail | default ----------|-------------------|-----------------------------------|--------------------------------------| ip | M | e.g. 111.111.111.111 | port | M | e.g. 12345 | scheme | M | choices:`HTTP` `HTTPS` | anonymity | O | choices:`transparent` `anonymous` | `transparent` need_auth | O | choices: 0 1 | user | O | | password | O | | url | O | | generated by given
scheme+ip+port **screenshot** ![](https://raw.githubusercontent.com/Karmenzind/i/master/fp-server/proxy_post.png) ### check status ### Check server status. Include: - Running spiders - Stored proxies ``` GET /api/status/ ``` No params. **screenshot** ![](https://raw.githubusercontent.com/Karmenzind/i/master/fp-server/status.png) ## Config ## ### Introduction ### I choose YAML language for configuration file. The defination and default value for supported items are: ```yaml # server's http port HTTP_PORT: 12345 # redirect output to console other than log file CONSOLE_OUTPUT: 1 # Log # dir and filename requires `CONSOLE_OUTPUT: 0` LOG: level: 'debug' dir: './logs' filename: 'fp-server.log' # redis database REDIS: host: '127.0.0.1' port: 6379 db: 0 password: # stop crawling new proxies # after stored this many proxies PROXY_STORE_NUM: 500 # Check availability in cycle # It's for each single proxy, not the checker PROXY_STORE_CHECK_SEC: 3600 ``` ### Customization ### - If you use Docker: - Create a directory such as `/x/config_dir` and put your `config.yml` in it. Then modify the docker-run command like this: ``` docker run -itd --name fpserver -p 12345:12345 -v "/x/config_dir":"/fps-config" karmenzind/fp-server:stable ``` - External `config.yml` doesn't need to contain all config items. For example, it can be: ``` PROXY_STORE_NUM: 100 LOG: level: 'info' PROXY_STORE_CHECK_SEC: 7200 ``` And other items will be default values. - If you need to set a log file, **don't** modify `LOG-dir` in `config.yml`. Instead create a directory for log file such as `/x/log_dir` and change the docker-run command like: ``` docker run -itd --name fpserver -p 12345:12345 -v "/x/config_dir":"/fps_config" -v "/x/log_dir":"/fp_server/logs" karmenzind/fp-server:stable ``` - There's no need to modify the exposed port of the container. If you prefer publishing it to another port(say, 9999) on the host, change the `-p` parameter in docker-run command to `-p 9999:12345` - If you need to access the Redis from host, add a new publishing parameter like `-p 6379:6379` to docker-run command. - If you manually deploy the project: - Modify the internal config file: `src/config/common.py` ## Source webs ## Growing…… If you knew good free-proxy websites, please tell me and I will add them to this project. Supporting: - [x] [西刺代理](http://www.xicidaili.com) - [x] [快代理](http://www.kuaidaili.com) - [x] [云代理](http://www.ip3366.net) - [x] [66免费代理](http://www.66ip.cn/) - [x] [无忧代理](http://www.data5u.com/free/index.shtml) - [x] [3464](http://www.3464.com/data/Proxy/http/) - [x] [coderbusy](https://proxy.coderbusy.com/) - [x] [ip181](http://www.ip181.com/) - [x] [iphai](http://www.iphai.com/free/ng) - [x] [a2u](https://raw.githubusercontent.com/a2u/free-proxy-list/master/free-proxy-list.txt) - [x] [coolproxy](https://www.cool-proxy.net/proxies/http_proxy_list/country_code:/port:/anonymous:) - [ ] [万能代理](http://wndaili.cn) - [ ] [小幻代理](https://ip.ihuan.me) (figuring) - [ ] [89免费代理](http://www.89ip.cn/)(figuring) - [ ] [baizhongsou](http://ip.baizhongsou.com/) (stop providing free proxies) Thanks to: [Golmic](https://github.com/lujqme) [Eric_Chan](https://github.com/CL545740896/) ## FAQ ## - **_How about the availability and quality of the proxies?_** Before storing new proxy, fp-server will check its availability, anonymity and speed based on your local network. So, feel free to use the crawled proxies. - **_How many `PROXY_STORE_NUM` should I set? Is there any limitation?_** You should set it depends on your real requirement. If your project is a normal spider, then 300-500 will be fair enough. I haven't set any limitation for now. After stored 10000 available proxies, I stopped testing. The upper limit is relevant to source websites. I will add more websites if more people use this project. - **_How to use it in my project?_** See the next section. ## Examples These code can be directly copied to your project. **Remember** to modify the configuration and settings at first. I will write more snippets at leisure. Or you can tell me what example you want. ### Use fp-server with Python requests module [Here.](./examples/use_with_requests.md) ### Use fp-server in Scrapy Project Here is [a middleware for Scrapy](./examples/middleware_for_scrapy.md) to fetch and apply proxy for each request. Copy it to your `middlewares.py` and add the name to `DOWNLOADER_MIDDLEWARES` in your `settings.py`. If you want to keep a cookie pool for your proxies(an independent cookiejar for each IP), [this middleware](https://github.com/Karmenzind/IcrisCrawler/blob/master/IcrisCrawler/middlewares.py#L126) may help you. ## Bugs and feature requests ## I need your feedback to make it better.
Please [create an issue](https://github.com/Karmenzind/fp-server/issues/new) for any problems or advice. Known bugs: * Block while using Tornado-4.5.3 * Afer check, the redis key might change ## TODOs and ideas ## * Use ZSET * Add supervisor * Divide log module * More detailed api * Web frontend via bootstrap * Add user-agent pool * the checker's scheduler: - Periodically calculating the average speed of checking request, then reassign the checker based on this average and the quantity of stored proxies. * Provide region information. * use redis's HSET for calculation ================================================ FILE: README_CN.md ================================================ # fp-server 免费代理服务器，基于[Tornado](http://www.tornadoweb.org/en/stable/#)和[Scrapy](https://scrapy.org/)，在本地搭建属于自己的代理池 特性： - 持续爬取新的免费代理，检测可用后存入本地数据库 - 完全异步，支持高并发 - 易用的HTTP API - 周期性检测代理可用性，自动更新 > 走过路过给个star…… \_(:ι」∠)\_ 已经测试通过的环境： - Archlinux; Python-3.6.5 - Debian(WSL, Raspbian); Python-3.5.3 **代码暂不支持直接在Windows上运行，Windows用户请[选择Docker方式部署](#使用docker)或使用WSL……** ## 目录 ## * [安装使用](#安装使用) * [使用Docker](#使用docker) * [手动安装部署 ###](#手动安装部署-) * [Web接口](#web接口) * [获取代理](#获取代理) * [手动创建代理](#手动创建代理) * [查看状态](#查看状态) * [配置](#配置) * [介绍](#介绍) * [修改](#修改) * [代理来源](#代理来源) * [FAQ](#faq) * [代码示例](#代码示例) * [结合Python requests模块使用](#结合python-requests模块使用) * [在Scrapy项目中使用](#在scrapy项目中使用) * [问题和需求](#问题和需求) ## 安装使用 ## 选取下列方式之一安装部署本项目。 安装完成之后，通过[API](#web接口)获取代理 ### 使用Docker ### **最简单**的安装部署方式是使用[Docker](https://www.docker.com/)，安装Docker后执行如下命令： ```bash # 下载镜像 docker pull karmenzind/fp-server:stable # 启动容器 # 如果在配置文件修改了端口，此处的`-p`也需要修改 docker run -itd --name fpserver -p 12345:12345 karmenzind/fp-server:stable # 检查容器内部输出 docker logs -f fpserver ``` 更改配置请查看[下文](#配置) ### 手动安装部署 ### 1. 安装[Redis数据库](https://redis.io/)和`python>=3.5`(我用的是Python-3.6.5) 2. 克隆这个项目 3. 安装所需的Python包 ```bash pip install -r requirements.txt ``` 4. 阅读[配置介绍](#配置)，根据需要修改 5. 启动服务 ```bash python ./src/main.py ``` ## Web接口 ## 一般返回格式 ```json { "code": 0, "msg": "ok", "data": {} } ``` - code: 事件状态(并非HTTP状态码), 0代表成功 - msg: 事件相关信息 - data: 返回数据 ### 获取代理 ### ``` GET /api/proxy/ ``` 参数 | 必须(M)/
非必须(O) | 详情 | 默认 ------------------------|-----------------------|----------------------------------------------------|-------| count | O | 获取的数目 scheme | O | 协议。可选:`HTTP` `HTTPS` | both* anonymity | O | 匿名效果。可选:`transparent`透明， `anonymous`匿名 | both (TODO)
sort_by_speed | O | 排序:
1: 降序
0: 乱序
-1: 升序 | 0 - both: 包括所有类型，不分组 **举例** - 获取10个HTTP匿名代理: ``` GET /api/proxy/?count=10&scheme=HTTP&anonymity=anonymous ``` 返回： ```json { "code": 0, "msg": "ok", "data": { "count": 9, "items": [ { "port": 2000, "ip": "xxx.xxx.xx.xxx", "scheme": "HTTP", "url": "http://xxx.xxx.xxx.xx:xxxx", "anonymity": "transparent" } ] } } ``` **截图** ![](https://raw.githubusercontent.com/Karmenzind/i/master/fp-server/proxy_get.png) ### 手动创建代理 ### ``` POST /api/proxy/ ``` 参数 | 必须(M)/
非必须(O) | 详情 | 默认 ----------|-----------------------|--------------------------------|----------------------------| ip | M | 例如 111.111.111.111 | port | M | 例如 12345 | scheme | M | 可选:`HTTP` `HTTPS` | anonymity | O | 可选:`transparent` `anonymous` | `transparent` need_auth | O | 可选: 0 1 | user | O | | password | O | | url | O | | 根据给定scheme+ip+port声称 **screenshot** ![](https://raw.githubusercontent.com/Karmenzind/i/master/fp-server/proxy_post.png) ### 查看状态 ### 查看服务状态。包含： - 正在运行的爬虫 - Stored proxies ``` GET /api/status/ ``` 没有参数 **screenshot** ![](https://raw.githubusercontent.com/Karmenzind/i/master/fp-server/status.png) ## 配置 ## ### 介绍 配置文件采用YAML格式，定义和默认值如下： ```yaml # 服务运行端口 HTTP_PORT: 12345 # 在终端打印输出，不写入文件 CONSOLE_OUTPUT: 1 # 日志设置 # dir和filename生效需要先设置CONSOLE_OUTPUT为0 LOG: level: 'debug' dir: './logs' filename: 'fp-server.log' REDIS: host: '127.0.0.1' port: 6379 db: 0 password: # 本地要存储的代理总数 # 超过这个数目后，服务会停止爬取新的代理 # 根据你的需要来合理设置 PROXY_STORE_NUM: 500 # 设定周期，定时检查每个代理可用性 # 每个代理都会存储自己的最后检查时间，动态检查 PROXY_STORE_CHECK_SEC: 3600 ``` ### 修改 - 使用Docker部署: - 在本地新建目录，如`/x/config_dir`，在其中新建配置文件`config.yml`，然后将docker-run命令修改如下： ``` docker run -itd --name fpserver -p 12345:12345 -v "/x/config_dir":"/fps-config" karmenzind/fp-server:stable ``` - 外部`config.yml`的内容可以为上述配置项的子集，例如： ``` PROXY_STORE_NUM: 100 LOG: level: 'info' PROXY_STORE_CHECK_SEC: 7200 ``` 其他配置项会自动采用内部配置 - 如果要指定日志文件，**不要**修改`config.yml`中的`LOG-dir`。在本地新建日志目录，如`/x/log_dir`，结合上一步，修改docker-run命令为： ``` docker run -itd --name fpserver -p 12345:12345 -v "/x/config_dir":"/fps_config" -v "/x/log_dir":"/fp_server/logs" karmenzind/fp-server:stable ``` - 没有必要修改内部端口。如果需要将HTTP端口映射到其他宿主机端口（如9999），将docker-run命令的`-p`参数修改为`-p 9999:12345` - 如果需要在宿主机访问Redis数据库，可以给docker-run命令增加参数如`-p 6379:6379` - 手动方式部署： - 直接修改项目内文件: `src/config/config.yml` ## 代理来源 ## 这个列表还在增加，欢迎贡献新的代理网站给我，我会将它加进项目里 目前支持: - [x] [西刺代理](http://www.xicidaili.com) - [x] [快代理](http://www.kuaidaili.com) - [x] [云代理](http://www.ip3366.net) - [x] [66免费代理](http://www.66ip.cn/) - [x] [无忧代理](http://www.data5u.com/free/index.shtml) - [x] [3464](http://www.3464.com/data/Proxy/http/) - [x] [coderbusy](https://proxy.coderbusy.com/) - [x] [ip181](http://www.ip181.com/) - [x] [iphai](http://www.iphai.com/free/ng) - [x] [a2u](https://raw.githubusercontent.com/a2u/free-proxy-list/master/free-proxy-list.txt) - [x] [coolproxy](https://www.cool-proxy.net/proxies/http_proxy_list/country_code:/port:/anonymous:) - [ ] [万能代理](http://wndaili.cn) - [ ] [小幻代理](https://ip.ihuan.me) (figuring) - [ ] [89免费代理](http://www.89ip.cn/)(figuring) - [ ] [baizhongsou](http://ip.baizhongsou.com/) (stop providing free proxies) 感谢: [Golmic](https://github.com/lujqme) [Eric_Chan](https://github.com/CL545740896/) ## FAQ ## - 代理可用性如何？ fp-server在爬取代理时会先检测可用性（包括速度和匿名性），检测完毕后才会入库，不可用的直接抛弃，所以你获取到的都是相对可用的代理。 - 存储代理数目`PROXY_STORE_NUM`设置多少合适？有上限么？ 根据你自己的需求来合理设置。假如只是一个普通的爬虫项目，那么设置为300到500就可以了，因为fp-server会持续自检更新代理，数目过大没有意义。数目设置暂时没有上限，我在攒够一万个活动（可用的）代理后停止了测试，因为目前代理来源有限。根据项目关注度，我会持续增加代理来源。 - 怎么把它用到我的项目里？ 见下文。 ## 代码示例 以下代码可以直接复制粘贴到你的项目。记得先检查修改配置。 有空我会写更多例子，你也可以告诉我你需要什么。 ### 结合Python requests模块使用 [戳这里](./examples/use_with_requests.md)。 ### 在Scrapy项目中使用 这是个用于给每个请求获取、增加代理的[Scrapy中间件](./examples/middleware_for_scrapy.md)。把它复制到`middlewares.py`，然后修改`settings.py`里的`DOWNLOADER_MIDDLEWARES`。 如果你需要维护一个Cookie池，给每个代理IP分配不同的CookieJar，可以参考我写的[这个中间件](https://github.com/Karmenzind/IcrisCrawler/blob/master/IcrisCrawler/middlewares.py#L126)。 ## 问题和需求 ## 欢迎大家反馈，这样我才有动力维护 如果有Bug或者建议，请直接[创建issue](https://github.com/Karmenzind/fp-server/issues/new) ================================================ FILE: examples/middleware_for_scrapy.md ================================================ ## Intro This is a middleware for using fp-server in general scrapy project. It will continuously fetch random proxy and set it **at each request**. To use this in your project, you should: 1. keep the fp-server running 2. add the middleware code down below to your `middlewares.py` 2. modify your settings: ```python # don't use scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware # at the same time DOWNLOADER_MIDDLEWARES = { 'NameOfYourProject.middlewares.FPServerMiddleware': 745, } # follow your real settings # your fp-server's address FP_SERVER_URL = 'http://localhost:12345' # anonymity: `anonymous` or `transparent` (default: random) FP_SERVER_PROXY_ANONYMITY = 'anonymous' # override the proxy that is already set (except for explicit None) # before this middleware FP_SERVER_FORCE_OVERRIDE = 1 # coding for auth (default: latin-l) # HTTPPROXY_AUTH_ENCODING = 'latin-l' ``` ## Middleware Here is the code. ```python # coding: utf-8 from urllib.parse import urljoin import requests from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached from six.moves.urllib.request import proxy_bypass class FPServerMiddleware(HttpProxyMiddleware): """ A middleware, based on FPServer, continuesly fetch random proxy and set it for each request. FPServer required. """ def __init__(self, crawler, auth_encoding, fps_url, anonymity, force_override): if not fps_url: raise NotConfigured('FP_SERVER_URL not configured') self.fps_api = urljoin(fps_url, '/api/proxy/') self.anonymity = anonymity self.logger = crawler.spider.logger self.crawler = crawler self.auth_encoding = auth_encoding self.force_override = force_override def fetch_proxy(self, scheme): """ Get proxy from fpserver by given scheme. :scheme: `str` proxy protocol :return: url, scheme """ params = { "scheme": scheme, "anonymity": self.anonymity, } text = None try: req = requests.get(self.fps_api, params=params) text = req.text data = req.json() except: self.logger.exception("Failed to fetch proxy: %s" % text) else: _code = data.get('code') _proxies = data.get('data', {}).get('detail', []) if (_code is not 0) or (not _proxies): self.logger.warning( 'Response of fetch_proxy: %s' % data) return proxy_info = _proxies[0] proxy_url = proxy_info['url'] return self._get_proxy(proxy_url, scheme) @classmethod def from_crawler(cls, crawler): auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING', 'latin-l') fps_url = crawler.settings.get('FP_SERVER_URL') anonymity = crawler.settings.get('FP_SERVER_PROXY_ANONYMITY') force_override = crawler.settings.get('FP_SERVER_FORCE_OVERRIDE', 0) return cls(crawler, auth_encoding, fps_url, anonymity, force_override) def _set_proxy(self, request, scheme): _fetched = self.fetch_proxy(scheme) if not _fetched: self.logger.debug('No proxy fetched from fp-server.') raise AssertionError('Please check your fp-server.') creds, proxy = _fetched request.meta['proxy'] = proxy self.logger.debug('Applied proxy (%s) for %s' % (proxy, request.url)) if creds: request.headers['Proxy-Authorization'] = b'Basic' + creds def process_request(self, request, spider): # ignore if proxy is already set # except force_override is true and previous proxy is None if 'proxy' in request.meta: # take higher priority than force_override previous_proxy = request.meta['proxy'] if previous_proxy is None: return if self.force_override: self.logger.debug('Forcely overrode the old proxy (%s).' % previous_proxy) else: # extract credentials if present creds, proxy_url = self._get_proxy(previous_proxy, '') request.meta['proxy'] = proxy_url if creds and not request.headers.get('Proxy-Authorization'): request.headers['Proxy-Authorization'] = b'Basic ' + creds return parsed = urlparse_cached(request) scheme = parsed.scheme # 'no_proxy' is only supported by http schemes if scheme in ('http', 'https') and proxy_bypass(parsed.hostname): return self._set_proxy(request, scheme) ``` ================================================ FILE: examples/use_with_requests.md ================================================ This page shows how to use fp-server with `requests` in your python code. ```python #!/usr/bin/env python # -*- coding: utf-8 -*- import requests from urllib.parse import urljoin, urlparse FP_SERVER_ADDR = 'http://localhost:12345' FP_SERVER_API = urljoin(FP_SERVER_ADDR, '/api/proxy/') def fetch_proxy(url, anonymity='anonymous'): """ Get proxy from fpserver by given url. :url: the url you want to request :anonymity: `transparent` or `anonymous` :return: {scheme: url} """ _parse = urlparse(url) scheme = _parse[0] params = { "scheme": scheme, "anonymity": anonymity, } text = None try: req = requests.get(FP_SERVER_API, params=params) text = req.text data = req.json() except: print("Failed to fetch proxy: %s" % text) else: _code = data.get('code') _proxies = data.get('data', {}).get('detail', []) if (_code is not 0) or (not _proxies): print( 'Response of fetch_proxy: %s' % data) return proxy_info = _proxies[0] proxy_url = proxy_info['url'] result = {scheme: proxy_url} print("Got proxy: ", result) return result # say, the site is 'baidu' url = 'https://baidu.com' # get scheme and proxy _p = fetch_proxy(url) # build a request r = requests.get(url, proxies=_p, timeout=20, # and other parameters ) print(r.status_code) print(r.text) ``` ================================================ FILE: requirements.txt ================================================ aioredis==1.1.0 beautifulsoup4==4.6.0 bs4==0.0.1 lxml==4.6.5 motor==1.1 pymongo==3.6.1 python-redis==0.1.7 redis==2.10.6 requests==2.20.0 Scrapy==1.5.0 tornado==5.0.2 Tornado-MySQL==0.5.1 PyYAML==5.4 ================================================ FILE: src/api/__init__.py ================================================ # -*— coding:utf-8 -*- from api.ping import ping from api.proxy import proxy from api.status import status from api.test import test __all__ = (ping, proxy, test, status) ================================================ FILE: src/api/ping/__init__.py ================================================ # -*- coding:utf-8 -*- ================================================ FILE: src/api/ping/ping.py ================================================ # -*- coding:utf-8 -*- """ ping-pong """ from utils.routes import route from utils import log as logger from core.web import WebHandler @route(r'/api/ping/$') class Ping(WebHandler): async def get(self, *args, **kwargs): ret = { 'ping': 'pong' } logger.debug('return ret:', ret, caller=self) self.do_success(ret) ================================================ FILE: src/api/proxy/__init__.py ================================================ ================================================ FILE: src/api/proxy/proxy.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- """ API for proxy """ from core import exceptions from core.web import WebHandler from service.proxy.proxy import proxy_srv from service.proxy.serializers import ProxySerializer from utils import log as logger from utils.routes import route from utils.tools import subdict def return_developing(): raise exceptions.NotFound(msg=exceptions.ERR_MSG_IS_DEVELOPING) @route(r'/api/proxy/$') class GetProxyHandler(WebHandler): """ proxy api """ async def get(self, *args, **kwargs): """ get proxies """ count = int(self.get_param('count', 1)) scheme = self.get_param('scheme') if scheme: scheme = scheme.lower() anonymity = self.get_param('anonymity') spec = dict(count=count, scheme=scheme, anonymity=anonymity) _items = await proxy_srv.query(spec) items = [] for i in _items: s = ProxySerializer(i) items.append(s.to_representation()) data = { "count": len(items), "detail": items, } # sort_by_speed = self.get_param('sort_by_speed', 0) self.do_success(data) async def post(self, *args, **kwargs): """ create new proxies """ item = self.get_body() indispensibles = ('scheme', 'ip', 'port') print(type(item), item) for k in indispensibles: if k not in item: raise exceptions.ValidationError('%s cannot be empty.' % k) _f = subdict(item, ['ip', 'port', 'scheme']) existed = await proxy_srv.keys_by_dict(_f) if existed: self.do_success({'success': 0, 'key': existed}, msg='key already existed') return try: _key = await proxy_srv.new_proxy(item) self.do_success({'success': 1, 'key': _key}, msg='created successfully') return except Exception as e: logger.exception('Failed: %s Detail: %s' % (item, e)) self.do_failed(code=400, msg=str(e)) async def delete(self, *args, **kwargs): """ delete proxies """ self.do_success({'ok': 1}, 'todo') @route(r'/api/proxy/report/$') class ReportProxyHandler(WebHandler): async def post(self, *args, **kwargs): self.do_success({'ok': 1}, 'developing..') ================================================ FILE: src/api/status/__init__.py ================================================ ================================================ FILE: src/api/status/status.py ================================================ # -*- coding:utf-8 -*- from utils.routes import route from core.web import WebHandler from service.spider.spider import spider_srv from service.proxy.proxy import proxy_srv @route(r'/api/status/$') class Status(WebHandler): """ get current status """ async def get(self, *args, **kwargs): proxy_status = await proxy_srv.get_all_status() spider_data = await spider_srv.all_status() ret = { "spiders": spider_data, "proxies": proxy_status, } self.do_success(ret) ================================================ FILE: src/api/test/__init__.py ================================================ ================================================ FILE: src/api/test/test.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- """ API for proxy """ from utils.routes import route from core.web import WebHandler # @route(r'/api/spider/run_all/$') # class TestSpiderHandler(WebHandler): # async def get(self, *args, **kwargs): # from service.spider.spider import spider_srv # info = { # 'started': await spider_srv.start_crawling() # } # self.do_success(info, 'todo') ================================================ FILE: src/config/__init__.py ================================================ # -*- coding: utf-8 -*- def _init_config(): import os import sys from utils.docker import check_if_inside_docker from utils.tools import parse_yaml, merge_configure import pprint result = None conf_dir = os.path.dirname(__file__) inside_docker = check_if_inside_docker() internal_path = os.path.join(conf_dir, 'config.yml') if os.path.exists(internal_path): print("Found internal configure file", internal_path) result = parse_yaml(internal_path) if not result: print("Invalid config file.") sys.exit(255) else: print("Basic file %s not found." % internal_path) sys.exit(255) if inside_docker: print('** YOU ARE INSIDE A DOCKER CONTAINER **') ext_path = '/fps_config/config.yml' else: ext_path = os.path.join(conf_dir, 'config.yml.local') if os.path.exists(ext_path): print('Found external config file %s' % ext_path) ext_conf = parse_yaml(ext_path) merge_configure(result, ext_conf) print("Got user config:") pprint.pprint(result) return result locals().update(_init_config()) ####################################################################### # internal config # ####################################################################### # api module HANDLER_PATHES = ['api'] # whether support cross-domain # ALLOW_CORS = True # auto reload DEBUG = False # parallel limitations # None: no limit # 0 : turn off all MAX_RUNNING_NUM = { 'seeker': 7, 'checker': None, } # for checker (testing) CHECK_TIMEOUT = 20 ================================================ FILE: src/config/config.yml ================================================ # server's http port HTTP_PORT: 12345 # redirect output to console other than log file CONSOLE_OUTPUT: 1 # Log # dir and filename requires `CONSOLE_OUTPUT: 0` LOG: level: 'debug' dir: './logs' filename: 'fp-server.log' # redis database REDIS: host: '127.0.0.1' port: 6379 db: 0 password: # stop crawling new proxies # after stored this many proxies PROXY_STORE_NUM: 500 # Check availability in cycle # It's for each single proxy, not the checker PROXY_STORE_CHECK_SEC: 3600 # vim: ft=yaml ================================================ FILE: src/core/__init__.py ================================================ ================================================ FILE: src/core/context.py ================================================ # -*- coding:utf-8 -*- import asyncio import sys from tornado import options from tornado.ioloop import IOLoop from tornado.platform.asyncio import AnyThreadEventLoopPolicy from tornado.web import Application from core.middleware import Middleware from utils import log as logger from utils import tools from utils.routes import route welcome = """ _____ ____ ____ | ___| _ \ / ___| ___ _ ____ _____ _ __ | |_ | |_) |___\___ \ / _ \ '__\ \ / / _ \ '__| | _| | __/_____|__) | __/ | \ V / __/ | |_| |_| |____/ \___|_| \_/ \___|_| Github: https://github.com/Karmenzind/fp-server E-Mail: valesail7@gmail.com ================================================ """ class TornadoContext(object): def __init__(self, setting_module): """ @param setting_module the loaded setting module `DEBUG` debug mode，default is False `CONSOLE_OUTPUT` redirect log to console other than file if set to 1 `LOG` logging `level` DEBUG/INFO/... `dir` in which the log files be saved `filename` log file's name `HANDLER_PATHES` the uri handler module's relative path `HTTP_PORT` HTTP port `MIDDLEWARES` middlewares `ALLOW_CORS` whether support CORS，default: false `COOKIE_SECRET` cookie secret string `MYSQL` mysql `MONGODB` mongodb `REDIS` redis """ print(welcome) self.loop = None self.setting_module = setting_module self._install_twisted() self._init_event_loop_policy() self._get_event_loop() self._load_settings() self._init_logger() self._init_middlewares() self._init_crawler_runner() self._init_db_instance() self._init_uri_routes() self._init_application() self._db_initial_works() self._do_heartbeat() def start(self): """ start event loop """ logger.info('start io loop ...') # self.loop.start() # self.loop.start() IOLoop.current().start() # self.loop.run_forever() def _init_event_loop_policy(self): asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy()) def _get_event_loop(self): # FIXIT # if not self.loop: # # self.loop = IOLoop.current() # # tornado.ioloop.IOLoop.configure('tornado.platform.asyncio.AsyncIOMainLoop') # self.loop = asyncio.get_event_loop() # IOLoop().make_current() return self.loop def _load_settings(self): """ parse the settings """ settings = __import__(self.setting_module, {}, {}, ['models']) # debug mode self.debug = getattr(settings, 'DEBUG', False) # whether open console_output self.console_output = getattr(settings, 'CONSOLE_OUTPUT', 1) # logging self.log_config = getattr(settings, 'LOG', {}) # path for uri handlers self.handler_pathes = getattr(settings, 'HANDLER_PATHES', []) # server HTTP port if len(sys.argv) > 1: self.http_port = sys.argv[1] else: self.http_port = getattr(settings, 'HTTP_PORT', 10000) # middlewares self.middlewares = getattr(settings, 'MIDDLEWARES', []) # mysql self.mysql_config = getattr(settings, 'MYSQL', None) # mongodb self.mongo_config = getattr(settings, 'MONGODB', None) # redis self.redis_config = getattr(settings, 'REDIS', None) # support cors self.cors = getattr(settings, 'ALLOW_CORS', False) options.define( 'cors', self.cors, help='set http response header `Access-Control-Allow-Origin` to `*`' ) # cookie secret string self.cookie_secret = getattr( settings, 'COOKIE_SECRET', tools.get_uuid4()) def _init_logger(self): """ create logger """ level = self.log_config.get('level', 'debug') dirname = self.log_config.get('dir', './logs') filename = self.log_config.get('filename', 'fp-server.log') if self.console_output: logger.initLogger(level) else: log_filename = '%s_%s.log' % ( filename.split('.')[0], self.http_port) logger.initLogger(level, dirname, log_filename) options.parse_command_line() def _init_uri_routes(self): """ initialize uri routes """ logger.info('init uri routes start >>>', caller=self) handlers = route.make_routes(self.handler_pathes) self.handlers = handlers logger.info('init uri routes done <<<', caller=self) def _init_db_instance(self): """ load database staff """ logger.info('init db instance start >>>', caller=self) if self.mysql_config: from core.db.mysql import initMySQL logger.info('mysql config:', self.mysql_config, caller=self) initMySQL(**self.mysql_config) if self.mongo_config: from core.db.mongo import initMongodb logger.info('mongodb config:', self.mongo_config, caller=self) initMongodb(**self.mongo_config) if self.redis_config: from core.db.redis import init_aioredis_pool logger.info('redis config:', self.redis_config, caller=self) # self.loop.run_until_complete(initRedisPool(**self.redis_config)) # IOLoop.current().add_callback(initRedisPool, **self.redis_config) IOLoop.current().run_sync(init_aioredis_pool) logger.info('init db instance done <<<', caller=self) def _init_middlewares(self): """ load middlewares """ logger.info('load middleware start >>>', caller=self) middlewares = [] for m in self.middlewares: l = m.split('.') class_name = l[-1] model = '.'.join(l[:-1]) mo = __import__(model, {}, {}, ['classes']) middleware = getattr(mo, class_name) instance = middleware() if not isinstance(instance, Middleware): logger.warn( 'middleware must inherit from tbag.core.middleware.Middleware:', m, caller=self) continue middlewares.append(instance) logger.info('middleware:', middleware, caller=self) options.define('middlewares', middlewares, help='set web api middlewares') logger.info('load middleware done <<<', caller=self) def _init_application(self): """ Initialize tornado app """ settings = { 'debug': self.debug, 'cookie_secret': self.cookie_secret } app = Application(self.handlers, **settings) app.listen(self.http_port) logger.info('listen http port at:', self.http_port, caller=self) def _do_heartbeat(self): """ initialize heartbeat """ from core.heartbeat import heartbeat, initial_tasks logger.info('Heartbeat started...') IOLoop.current().add_callback(heartbeat.start) IOLoop.current().add_callback(initial_tasks) def _init_crawler_runner(self): """ initialize scrapy env and crawler runner """ from core.crawler import init_crawler_runner init_crawler_runner() def _db_initial_works(self): from core.db.initial_works import do_initial_works do_initial_works() def _install_twisted(self): import tornado.platform.twisted tornado.platform.twisted.install() ================================================ FILE: src/core/crawler.py ================================================ # coding: utf-8 # These work had to be done here for now # TODO: reconstruct import os import sys from utils import log as logger # XXX: Why must crochet? import crochet from scrapy.crawler import CrawlerRunner from scrapy.utils.project import get_project_settings # from proxy_spider.spiders.ihuan import IhuanSpider # from proxy_spider.spiders.ip89 import Ip89Spider from proxy_spider.spiders.a3464 import A3464Spider from proxy_spider.spiders.checker import CheckerSpider from proxy_spider.spiders.coderbusy import CoderbusySpider from proxy_spider.spiders.coolproxy import CoolproxySpider from proxy_spider.spiders.data5u import Data5uSpider from proxy_spider.spiders.ip66 import Ip66Spider from proxy_spider.spiders.kuaidaili import KuaidailiSpider from proxy_spider.spiders.mix import MixSpider from proxy_spider.spiders.xicidaili import XicidailiSpider from proxy_spider.spiders.yundaili import YundailiSpider from service.spider.functions import build_key CRAWLER_RUNNER = None def init_scrapy_env(): cur_dir = os.path.dirname(os.path.realpath(__file__)) settings_dir = os.path.join(cur_dir, 'proxy_spider') # settings_mod = os.path.join(settings_dir, 'settings.py') sys.path.append(settings_dir) sys.path.append(cur_dir) os.environ['SCRAPY_PROJECT'] = 'proxy_spider' os.environ['SCRAPY_SETTINGS_MODULE'] = 'proxy_spider.settings' def init_crawler_runner(): crochet.setup() init_scrapy_env() settings = get_project_settings() global CRAWLER_RUNNER CRAWLER_RUNNER = CrawlerRunner(settings) logger.info('Initialized crawler runner: %s' % CRAWLER_RUNNER) # TODO: move these to config file? enabled_spiders = { 'seeker': ( A3464Spider, CoderbusySpider, CoolproxySpider, Data5uSpider, Ip66Spider, KuaidailiSpider, MixSpider, XicidailiSpider, YundailiSpider, # Ip89Spider, # IhuanSpider, ), 'checker': ( CheckerSpider, ) } def get_keymap(): result = {} for _type, v in enabled_spiders.items(): keymap = {} for spider_cls in v: key = build_key(spider_cls, _type) keymap[key] = spider_cls result[_type] = keymap logger.info("Got spiders' keymap: %s" % enabled_spiders) return result spider_keymap = get_keymap() __all__ = [CRAWLER_RUNNER, enabled_spiders] if __name__ == '__main__': print(spider_keymap) ================================================ FILE: src/core/db/__init__.py ================================================ ================================================ FILE: src/core/db/initial_works.py ================================================ # coding: utf-8 from utils import log as logger from core.db.redis import pyredis_pool def do_initial_works(): """ works after db initialized :return: """ cli = pyredis_pool.acquire() from core.crawler import spider_keymap from service.spider.functions import key_prefix existed_keys = cli.keys('%s*' % key_prefix) for _type, _map in spider_keymap.items(): for k, v in _map.items(): if k in existed_keys: existed_keys.remove(k) cli.hmset(k, 'status', 'stopped', 'name', v.name) for k in existed_keys: cli.delete(k) logger.info('Finished database initial works.') ================================================ FILE: src/core/db/mongo.py ================================================ # -*- coding:utf-8 -*- import copy from urllib.parse import quote_plus from bson.objectid import ObjectId import motor from utils import log as logger from utils import time_ext MONGO_CONN = None DELETE_FLAG = 'delete' # True 已经删除，False 或者没有该字段表示没有删除 def initMongodb(host='127.0.0.1:27017', username='', password='', dbname='admin'): """ 初始化mongodb连接 """ if username and password: username = quote_plus(username) password = quote_plus(password) host = quote_plus(host) uri = 'mongodb://{username}:{password}@{host}/{dbname}'.format( username=username, password=password, host=host, dbname=dbname, ) else: uri = "mongodb://{host}/{dbname}".format(host=host, dbname=dbname) mongo_client = motor.motor_tornado.MotorClient(uri) global MONGO_CONN MONGO_CONN = mongo_client logger.info('create mongodb connection pool.') class MongoDBBase(object): """ mongodb 数据库操作接口 """ def __init__(self): """ 初始化 * self._db 初始化连接的数据库，子类指定 * self._table 初始化连接的colletion，子类指定 """ self.conn = MONGO_CONN self.dao = self.conn[self._db][self._table] async def get_list(self, spec={}, fields=None, sort=[], skip=0, limit=9999): """ 批量获取数据 @param spec 查询条件 @param fields 返回数据的字段 @param sort 排序规则 @param skip 查询游标 @param limit 返回数据条数 * NOTE: 必须传入limit，否则默认返回数据条数可能因为pymongo的默认值而改变 """ if '_id' in spec: spec['_id'] = self._convert_id_object(spec['_id']) spec[DELETE_FLAG] = {'$ne': True} datas = [] cursor = self.dao.find(spec, fields, sort=sort, skip=skip, limit=limit) async for item in cursor: item['_id'] = str(item['_id']) datas.append(item) return datas async def find_one(self, spec={}, fields=None, sort=[]): """ 查找单条数据 @param spec 查询条件 @param fields 返回数据的字段 @param sort 排序规则 """ data = await self.get_list(spec, fields, sort, limit=1) if data: return data[0] else: return None async def count(self, spec={}): """ 计算数据条数 @param spec 查询条件 @param n 返回查询的条数 """ spec[DELETE_FLAG] = {'$ne': True} n = await self.dao.count(spec) return n async def insert(self, docs_data): """ 插入数据 @param docs_data 插入数据 dict或list @param ret_ids 插入数据的id列表 """ docs = copy.deepcopy(docs_data) ret_ids = [] is_one = False create_time = time_ext.get_utc_time() if not isinstance(docs, list): docs = [docs] is_one = True for doc in docs: doc['_id'] = ObjectId() doc['create_time'] = create_time doc['modify_time'] = create_time ret_ids.append(str(doc['_id'])) self.dao.insert_many(docs) if is_one: return ret_ids[0] else: return ret_ids async def update(self, spec, update_fields, upsert=False, multi=False): """ 更新 @param spec 更新条件 @param update_fields 更新字段 @param upsert 如果不满足条件，是否插入新数据 @param multi 是否批量更新 @return modified_count 更新数据条数 """ spec[DELETE_FLAG] = {'$ne': True} if '_id' in spec: spec['_id'] = self._convert_id_object(spec['_id']) set_fields = update_fields.get('$set', {}) set_fields['modify_time'] = time_ext.get_utc_time() update_fields['$set'] = set_fields if not multi: result = await self.dao.update_one(spec, update_fields, upsert=upsert) return result.modified_count else: result = await self.dao.update_many(spec, update_fields, upsert=upsert) return result.modified_count async def delete(self, spec): """ 软删除 @param spec 删除条件 @return delete_count 删除数据的条数 """ spec[DELETE_FLAG] = {'$ne': True} if '_id' in spec: spec['_id'] = self._convert_id_object(spec['_id']) update_fields = {'$set': {DELETE_FLAG: True}} delete_count = await self.update(spec, update_fields, multi=True) return delete_count async def remove(self, spec, multi=False): """ 彻底删除数据 @param spec 删除条件 @param multi 是否全部删除 @return deleted_count 删除数据的条数 """ if not multi: result = await self.dao.delete_one(spec) return result.deleted_count else: result = await self.dao.delete_many(spec) return result.deleted_count async def distinct(self, key, spec={}): """ distinct查询 @param key 查询的key @param spec 查询条件 @return result 过滤结果list """ spec[DELETE_FLAG] = {'$ne': True} if '_id' in spec: spec['_id'] = self._convert_id_object(spec['_id']) result = await self.dao.distinct(key, spec) return result async def find_one_and_update(self, spec, update_fields, upsert=False, return_document=False, fields=None): """ 查询一条指定数据，并修改这条数据 @param spec 查询条件 @param update_fields 更新字段 @param upsert 如果不满足条件，是否插入新数据，默认False @param return_document 返回修改之前数据或修改之后数据，默认False为修改之前数据 @param fields 需要返回的字段，默认None为返回全部数据 @return result 修改之前或之后的数据 """ spec[DELETE_FLAG] = {'$ne': True} if '_id' in spec: spec['_id'] = self._convert_id_object(spec['_id']) set_fields = update_fields.get('$set', {}) set_fields['modify_time'] = time_ext.get_utc_time() update_fields['$set'] = set_fields result = await self.dao.find_one_and_update(spec, update_fields, projection=fields, upsert=upsert, return_document=return_document) if result and '_id' in result: result['_id'] = str(result['_id']) return result async def find_one_and_delete(self, spec={}, fields=None): """ 查询一条指定数据，并删除这条数据 @param spec 删除条件 @param fields 需要返回的字段，默认None为返回全部数据 @param result 删除之前的数据 """ spec[DELETE_FLAG] = {'$ne': True} if '_id' in spec: spec['_id'] = self._convert_id_object(spec['_id']) result = await self.dao.find_one_and_delete(spec, projection=fields) if result and '_id' in result: result['_id'] = str(result['_id']) return result def _convert_id_object(self, origin): """ 将字符串的_id转换成ObjectId类型 """ if isinstance(origin, str): return ObjectId(origin) elif isinstance(origin, (list, set)): return [ObjectId(item) for item in origin] elif isinstance(origin, dict): for key, value in origin.items(): origin[key] = self._convert_id_object(value) return origin __all__ = [initMongodb, MongoDBBase] ================================================ FILE: src/core/db/mysql.py ================================================ # -*- coding:utf-8 -*- from tornado_mysql import cursors, pools from utils import log as logger pools.DEBUG = True CONN_POOL = None def initMySQL(host='127.0.0.1', port=3306, username='', password='', db='mysql'): """ 初始化mysql连接池 @param host MySQL数据库ip @param port MySQL数据库端口 @param username MySQL数据库用户名 @param password MySQL数据库密码 @param db 需要连接的数据库名 """ mysql_config = { 'host': host, 'port': port, 'user': username, 'passwd': password, 'db': db, 'cursorclass': cursors.DictCursor, 'charset': 'utf8' } logger.info('mysql_config:', mysql_config) global CONN_POOL CONN_POOL = pools.Pool(mysql_config, max_idle_connections=1, max_recycle_sec=3) logger.info('create mysql connection pool.') async def exec_cmd(sql): """ 执行mysql命令 @param sql sql命令 @result result 返回的执行结果 """ sql = sql.replace('\t', ' ').replace('\n', ' ') logger.debug('sql:', sql) cursor = await CONN_POOL.execute(sql) result = cursor.fetchall() return result __all__ = [initMySQL, exec_cmd] ================================================ FILE: src/core/db/redis.py ================================================ # -*- coding:utf-8 -*- import aioredis from utils import log as logger aioredis_pool = None # TODO: build a common class for redis def get_redis_config(): """parse redis config :returns: dict """ import config redis_conf = getattr(config, 'REDIS', {}) host = redis_conf.get('host', 'localhost') port = redis_conf.get('port', 6379) url = 'redis://%s:%s' % (host, port) # FIXME result = dict( host=host, port=port, url=url, address=url, password=redis_conf.get('password') or None, db=redis_conf.get('db', 0), encoding=redis_conf.get('encoding', 'utf-8'), database=redis_conf.get('db', 0), ) return result redis_config = get_redis_config() async def init_aioredis_pool(): keys = ('address', 'db', 'password', 'encoding') conf = {k: v for k, v in redis_config.items() if k in keys} global aioredis_pool aioredis_pool = await aioredis.create_redis_pool(**conf) logger.info('Initialized aioredis pool: %s.' % aioredis_pool) def init_pyredis_pool(): from pyredis import Pool keys = ('host', 'port', 'database', 'password', 'encoding') conf = {k: v for k, v in redis_config.items() if k in keys} pool = Pool(**conf) logger.info('Initialized pyredis pool: %s' % pool) return pool pyredis_pool = init_pyredis_pool() class RedisDBBase: cli = aioredis_pool async def exec_cmd(self, *args, **kwargs): result = await aioredis_pool.execute(*args, **kwargs) logger.debug('cmd:', *args, 'result:', result, caller=self) return result __all__ = (init_aioredis_pool, RedisDBBase, pyredis_pool, aioredis_pool) ####################################################################### # snippits # ####################################################################### # import asyncio # from tornado.ioloop import IOLoop # # class RedisDBBase: # """ redis连接 # """ # # def __init__(self, host='redis://127.0.0.1:6379', channel='test'): # self.pool = None # 连接池 # self.pub_conn = None # publish连接 # self.host = host # self.channel = channel # # async def start(self): # await self._init_pool() # await self._init_publish() # await self._init_subscribe() # # async def _init_pool(self): # """ 初始化连接池 # """ # self.pool = await aioredis.create_redis_pool(self.host, encoding='utf-8') # logger.info('create redis pool success.', caller=self) # # async def _init_publish(self): # """ 初始化事件发布 # """ # self.pub_conn = await aioredis.create_redis(self.host) # logger.info('create redis publish channel success. channel:', self.channel, caller=self) # # async def _init_subscribe(self): # """ 初始化订阅连接 # """ # sub = await aioredis.create_redis(self.host) # channel, = await sub.subscribe(self.channel) # await asyncio.ensure_future(self.async_reader(channel)) # logger.info('subscribe channel success. channel:', self.channel, caller=self) # # async def exec_redis_cmd(self, *args): # logger.debug('cmd:', *args, caller=self) # result = await self.pool.execute(*args) # return result # # async def publish(self, content): # data = json.dumps(content) # await self.pub_conn.execute('PUBLISH', self.channel, data) # # logger.debug('content:', content, caller=self) # # async def async_reader(self, channel): # while await channel.wait_message(): # msg = await channel.get(encoding='utf-8') # data = json.loads(msg) # IOLoop.current().add_callback(WebsocketHandler.push_message, data) # # logger.debug('receive data:', data, caller=self) # # # redis_srv = RedisDBBase() # ioloop.run_until_complete(redis_srv.start()) # ================================================ FILE: src/core/exceptions.py ================================================ #!/usr/bin/env python # -*- coding: utf-8 -*- ####################################################################### # error const # ####################################################################### MSG_OK = {'code': 0, 'msg': 'OK'} # common errors ERR_MSG_INVALID = {'code': 400, 'msg': 'Failed'} ERR_MSG_WRONG_PARAMS = {'code': 400, 'msg': 'Check the params that you input'} ERR_MSG_PERMISSION_ERROR = {'code': 401, 'msg': 'Authentication failed'} ERR_MSG_BODY_ERROR = {'code': 411, 'msg': 'Invalid body format'} ERR_MSG_SYSTEM_ERROR = {'code': 500, 'msg': 'Internal error'} ERR_MSG_IS_DEVELOPING = {'code': 99999, 'msg': 'TODO...'} # paticular errors ####################################################################### # custom exceptions # ####################################################################### class CustomException(Exception): default_msg = 'A server error occurred.' default_data = None default_code = 500 def __init__(self, msg=None, code=None, data=None): self.msg = msg if msg is not None else self.default_msg self.code = code if code is not None else self.default_code self.data = data def __str__(self): str_msg = '[{code}] {msg}'.format(code=self.code, msg=self.msg) return str_msg class ValidationError(CustomException): default_msg = 'Bad Request' default_code = 400 class ParamError(CustomException): default_msg = 'Param Error' default_code = 400 class NotAuthenticated(CustomException): default_msg = 'Unauthorized' default_code = 401 class AuthenticationFailed(CustomException): default_msg = 'Forbidden' default_code = 403 class NotFound(CustomException): default_msg = 'Not found' default_code = 404 class SystemError(CustomException): default_msg = 'Internal Server Error' default_code = 500 class TimeoutException(CustomException): default_msg = 'Timeout' default_code = 504 ================================================ FILE: src/core/heartbeat.py ================================================ # -*- coding:utf-8 -*- import datetime from tornado.ioloop import IOLoop from utils import log as logger class HeartBeat(object): def __init__(self): self._count = 0 self._interval = 1 self.tasks = [] def start(self): self._count += 1 if self._count > 9999999: self._count = 1 if self._count % 60 == 0: logger.info('Server heartbeat count:', self._count, caller=self) IOLoop.current().add_timeout(datetime.timedelta(seconds=self._interval), self.start) for task in self.tasks: func = task['func'] args = task['args'] kwargs = task['kwargs'] kwargs['heart_beat_count'] = self._count IOLoop.current().add_callback(func, *args, **kwargs) def register(self, func, *args, **kwargs): """ register a task run it on each heartbeat @param func """ t = { 'func': func, 'args': args, 'kwargs': kwargs } self.tasks.append(t) heartbeat = HeartBeat() def initial_tasks(): from service.tasks.spider import SpiderTasks spider_task = SpiderTasks() heartbeat.register(spider_task.start) # __all__ = ('heartbeat',) ================================================ FILE: src/core/middleware.py ================================================ # -*- coding:utf-8 -*- # TODO: change name to process_* class Middleware: """ Base class of for middlewares """ async def prepare(self, request): """ 在HTTP请求的方法执行之前，执行此函数 @param request HTTP请求实例 """ pass async def finish(self, response): """ 在HTTP请求返回成功之后，执行此函数 @param response HTTP返回实例 """ pass ================================================ FILE: src/core/web.py ================================================ # -*- coding:utf-8 -*- import datetime import json from tornado.options import options from tornado.web import RequestHandler from utils import time_ext from core import exceptions class WebHandler(RequestHandler): """ web base handler """ @property def query_params(self): if not hasattr(self, '_query_params'): self._query_params = {} for key in self.request.arguments.keys(): value = self.get_argument(key) self._query_params[key] = value return self._query_params @property def data(self): if not hasattr(self, '_data'): self._data = None if self.request.body: try: self._data = json.loads(self.request.body.decode('utf-8')) except: raise exceptions.ValidationError('only support json') return self._data def _to_representation(self, instance): """ serialize datetime """ if isinstance(instance, datetime.datetime): return time_ext.get_time_str(instance, time_ext.UTC) if isinstance(instance, datetime.date): return instance.isoformat() if isinstance(instance, list): return [self._to_representation(item) for item in instance] if isinstance(instance, dict): for key in instance.keys(): instance[key] = self._to_representation(instance[key]) return instance else: return instance def get_param(self, key, defaut=None): """ get param of request * avoid straightly raising exception while using self.get_argument @param key parameter's name @param defaut return None if key doesn't exist @return value value of param """ value = self.get_argument(key, defaut) return value def get_params(self, *keys): """ get multiple params, see `get_param` @param keys list @return values list """ values = [] for key in keys: value = self.get_param(key) values.append(value) return values def get_body(self, parse_json=True): """ get body's data @param parse_json whether parse data with json @return body body's data """ body = self.request.body if not body: return None if parse_json: try: body = json.loads(body.decode('utf8')) except: raise exceptions.ValidationError(msg='请求body数据格式错误') return body def do_success(self, data={}, msg='success'): """ for successful event """ result = { 'code': 0, 'msg': msg, 'data': self._to_representation(data) } self.do_finish(result) def do_failed(self, code=400, msg='error', data={}): """ for failed event """ result = { 'code': code, 'msg': msg, 'data': self._to_representation(data) } self.set_status(200, 'OK') self.do_finish(result) def do_finish(self, result): """ add tail work """ # 跨域 cors = options.cors if cors: self.set_header('Access-Control-Allow-Origin', '*') self.set_header('Access-Control-Allow-Headers', '*') self.finish(result) def write_error(self, status_code, **kwargs): """ custom exception * overwrote """ exc_info = kwargs.get('exc_info') ex = exc_info[1] if isinstance(ex, exceptions.CustomException): self.do_failed(ex.code, ex.msg, ex.data) else: self.do_failed(500, '服务器内部错误') async def head(self, *args, **kwargs): await self.process('_head_', *args, **kwargs) async def get(self, *args, **kwargs): await self.process('_get_', *args, **kwargs) async def post(self, *args, **kwargs): await self.process('_post_', *args, **kwargs) async def put(self, *args, **kwargs): await self.process('_put_', *args, **kwargs) async def delete(self, *args, **kwargs): await self.process('_delete_', *args, **kwargs) async def patch(self, *args, **kwargs): await self.process('_patch_', *args, **kwargs) async def options(self, *args, **kwargs): await self.process('_options_', *args, **kwargs) async def process(self, func_name, *args, **kwargs): """ process the request @param func_name choices [_head_, _get_, _post_, _put_, _delete_, _patch_, _options_] @note preparation and finish work """ func = getattr(self, func_name, None) if not func: raise exceptions.NotFound() await self.do_prepare() await func(*args, **kwargs) await self.do_complete() async def do_prepare(self): """ preparation * may add calculation or authentication here """ middlewares = options.middlewares for m in middlewares: await m.prepare(self) async def do_complete(self): """ wind up * may add calculation or logging here """ middlewares = options.middlewares for m in middlewares: await m.finish(self) ================================================ FILE: src/main.py ================================================ # /usr/bin/env python3 # -*- coding:utf-8 -*- """ fp-server main entry """ import os import sys from core.context import TornadoContext # os.environ['PYTHONASYNCIODEBUG'] = '1' def main(): """ launch """ context = TornadoContext(setting_module='config') context.start() if __name__ == '__main__': src_dir = os.path.dirname(__file__) sys.path.insert(0, src_dir) main() ================================================ FILE: src/proxy_spider/__init__.py ================================================ # coding: utf-8 """ This module's structure is totally as-is basis of Scrapy project """ ================================================ FILE: src/proxy_spider/const.py ================================================ # coding: utf-8 """ many user_agents """ # http://www.useragentstring.com/pages/useragentstring.php user_agents = [ "Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16", "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14", "Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14", "Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1", "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0", "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246" 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36', 'Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36', 'Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_4) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1290.1 Safari/537.13', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.13 (KHTML, like Gecko) Chrome/24.0.1284.0 Safari/537.13', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.6 Safari/537.11', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/537.11', 'Mozilla/5.0 (Windows NT 6.0) yi; AppleWebKit/345667.12221 (KHTML, like Gecko) Chrome/23.0.1271.26 Safari/453667.1221', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.17 Safari/537.11', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.94 Safari/537.4', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_0) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.79 Safari/537.4', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1216.0 Safari/537.2', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1', 'Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/19.0.1047.0 Safari/535.22', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1042.0 Safari/535.21', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.21 (KHTML, like Gecko) Chrome/19.0.1041.0 Safari/535.21', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/18.6.872.0 Safari/535.2 UNTRUSTED/1.0 3gpp-gba UNTRUSTED/1.0', 'Mozilla/5.0 (Macintosh; AMD Mac OS X 10_8_2) AppleWebKit/535.22 (KHTML, like Gecko) Chrome/18.6.872', 'Mozilla/5.0 (X11; CrOS i686 1660.57.0) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.46 Safari/535.19', 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.45 Safari/535.19', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.151 Safari/535.19', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.19 (KHTML, like Gecko) Ubuntu/11.10 Chromium/18.0.1025.142 Chrome/18.0.1025.142 Safari/535.19', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.11 Safari/535.19', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.66 Safari/535.11', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Ubuntu/11.10 Chromium/17.0.963.65 Chrome/17.0.963.65 Safari/535.11', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Ubuntu/11.04 Chromium/17.0.963.65 Chrome/17.0.963.65 Safari/535.11', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Ubuntu/10.10 Chromium/17.0.963.65 Chrome/17.0.963.65 Safari/535.11', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.11 (KHTML, like Gecko) Ubuntu/11.10 Chromium/17.0.963.65 Chrome/17.0.963.65 Safari/535.11', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.65 Safari/535.11', 'Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.65 Safari/535.11', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.65 Safari/535.11', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.65 Safari/535.11', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.65 Safari/535.11', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_4) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.65 Safari/535.11', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Ubuntu/11.04 Chromium/17.0.963.56 Chrome/17.0.963.56 Safari/535.11', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11', 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.8 (KHTML, like Gecko) Chrome/17.0.940.0 Safari/535.8', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7ad-imcjapan-syosyaman-xkgi3lqg03!wgz', 'Mozilla/5.0 (X11; CrOS i686 1193.158.0) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7', 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.63 Safari/535.7xs5D9rRDFpg2g', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.8 (KHTML, like Gecko) Chrome/16.0.912.63 Safari/535.8', 'Mozilla/5.0 (Windows NT 5.2; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.63 Safari/535.7', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7', 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.6 (KHTML, like Gecko) Chrome/16.0.897.0 Safari/535.6', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.54 Safari/535.2', 'Mozilla/5.0 (X11; FreeBSD i386) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.2 (KHTML, like Gecko) Ubuntu/11.10 Chromium/15.0.874.120 Chrome/15.0.874.120 Safari/535.2', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.872.0 Safari/535.2', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.2 (KHTML, like Gecko) Ubuntu/11.04 Chromium/15.0.871.0 Chrome/15.0.871.0 Safari/535.2', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.864.0 Safari/535.2', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.861.0 Safari/535.2', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.861.0 Safari/535.2', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.861.0 Safari/535.2', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.860.0 Safari/535.2', 'Chrome/15.0.860.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/15.0.860.0', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/11.04 Chromium/14.0.825.0 Chrome/14.0.825.0 Safari/535.1', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.824.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.815.10913 Safari/535.1', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.815.0 Safari/535.1', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/11.04 Chromium/14.0.814.0 Chrome/14.0.814.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.814.0 Safari/535.1', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/10.04 Chromium/14.0.813.0 Chrome/14.0.813.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.813.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 5.2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.813.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.813.0 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.813.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.812.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.811.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.810.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.810.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.809.0 Safari/535.1', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/10.10 Chromium/14.0.808.0 Chrome/14.0.808.0 Safari/535.1', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/10.04 Chromium/14.0.808.0 Chrome/14.0.808.0 Safari/535.1', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/10.04 Chromium/14.0.804.0 Chrome/14.0.804.0 Safari/535.1', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.803.0 Safari/535.1', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/11.04 Chromium/14.0.803.0 Chrome/14.0.803.0 Safari/535.1', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.803.0 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.803.0 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.803.0 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.803.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.801.0 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.801.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 5.2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.794.0 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.794.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.792.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 5.2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.792.0 Safari/535.1', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.792.0 Safari/535.1', 'Mozilla/5.0 (Macintosh; PPC Mac OS X 10_6_7) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.790.0 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.790.0 Safari/535.1', 'Mozilla/5.0 (Windows; U; Windows NT 6.1) AppleWebKit/526.3 (KHTML, like Gecko) Chrome/14.0.564.21 Safari/526.3', 'Mozilla/5.0 (X11; CrOS i686 13.587.48) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.43 Safari/535.1', 'Mozilla/5.0 Slackware/13.37 (X11; U; Linux x86_64; en-US) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41', 'Mozilla/5.0 ArchLinux (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Ubuntu/11.04 Chromium/13.0.782.41 Chrome/13.0.782.41 Safari/535.1', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1', 'Mozilla/5.0 (Windows NT 5.2; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_3) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_3) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.32 Safari/535.1', 'Mozilla/5.0 (X11; Linux amd64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.220 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.220 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.220 Safari/535.1', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.215 Safari/535.1', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.215 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.215 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.215 Safari/535.1', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.20 Safari/535.1', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.20 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.20 Safari/535.1', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.20 Safari/535.1', 'Mozilla/5.0 (X11; CrOS i686 0.13.587) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.14 Safari/535.1', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.107 Safari/535.1', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_2) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.107 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.1 Safari/535.1', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.36 (KHTML, like Gecko) Chrome/13.0.766.0 Safari/534.36', 'Mozilla/5.0 (X11; Linux amd64) AppleWebKit/534.36 (KHTML, like Gecko) Chrome/13.0.766.0 Safari/534.36', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.35 (KHTML, like Gecko) Ubuntu/10.10 Chromium/13.0.764.0 Chrome/13.0.764.0 Safari/534.35', 'Mozilla/5.0 (X11; CrOS i686 0.13.507) AppleWebKit/534.35 (KHTML, like Gecko) Chrome/13.0.763.0 Safari/534.35', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.33 (KHTML, like Gecko) Ubuntu/9.10 Chromium/13.0.752.0 Chrome/13.0.752.0 Safari/534.33', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/534.31 (KHTML, like Gecko) Chrome/13.0.748.0 Safari/534.31', 'Mozilla/5.0 (Windows NT 6.1; en-US) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.750.0 Safari/534.30', 'Mozilla/5.0 (X11; CrOS i686 12.433.109) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.93 Safari/534.30', 'Mozilla/5.0 (X11; CrOS i686 12.0.742.91) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.93 Safari/534.30', 'Mozilla/5.0 Slackware/13.37 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/12.0.742.91', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Chromium/12.0.742.91 Safari/534.30', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.68 Safari/534.30', 'Mozilla/5.0 ArchLinux (X11; U; Linux x86_64; en-US) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.60 Safari/534.30', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.53 Safari/534.30', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.113 Safari/534.30', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.30 (KHTML, like Gecko) Ubuntu/11.04 Chromium/12.0.742.112 Chrome/12.0.742.112 Safari/534.30', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.30 (KHTML, like Gecko) Ubuntu/10.10 Chromium/12.0.742.112 Chrome/12.0.742.112 Safari/534.30', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.30 (KHTML, like Gecko) Ubuntu/10.04 Chromium/12.0.742.112 Chrome/12.0.742.112 Safari/534.30', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.30 (KHTML, like Gecko) Ubuntu/11.04 Chromium/12.0.742.112 Chrome/12.0.742.112 Safari/534.30', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.30 (KHTML, like Gecko) Ubuntu/10.10 Chromium/12.0.742.112 Chrome/12.0.742.112 Safari/534.30', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.30 (KHTML, like Gecko) Ubuntu/10.04 Chromium/12.0.742.112 Chrome/12.0.742.112 Safari/534.30', 'Mozilla/5.0 (Windows NT 7.1) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.112 Safari/534.30', 'Mozilla/5.0 (Windows NT 5.2) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.112 Safari/534.30', 'Mozilla/5.0 (Windows 8) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.112 Safari/534.30', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_6) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.112 Safari/534.30', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_4) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.112 Safari/534.30', 'Mozilla/5.0 (X11; CrOS i686 12.433.216) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.105 Safari/534.30', 'Mozilla/5.0 ArchLinux (X11; U; Linux x86_64; en-US) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.100 Safari/534.30', 'Mozilla/5.0 ArchLinux (X11; U; Linux x86_64; en-US) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.100', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.30 (KHTML, like Gecko) Slackware/Chrome/12.0.742.100 Safari/534.30', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.100 Safari/534.30', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.100 Safari/534.30', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.100 Safari/534.30', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_4) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.100 Safari/534.30', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.724.100 Safari/534.30', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.25 (KHTML, like Gecko) Chrome/12.0.706.0 Safari/534.25', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.25 (KHTML, like Gecko) Chrome/12.0.704.0 Safari/534.25', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.24 (KHTML, like Gecko) Ubuntu/10.10 Chromium/12.0.703.0 Chrome/12.0.703.0 Safari/534.24', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Ubuntu/10.10 Chromium/12.0.702.0 Chrome/12.0.702.0 Safari/534.24', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/12.0.702.0 Safari/534.24', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/12.0.702.0 Safari/534.24', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.700.3 Safari/534.24', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.699.0 Safari/534.24', 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.699.0 Safari/534.24', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_6) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.698.0 Safari/534.24', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.697.0 Safari/534.24', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.71 Safari/534.24', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.68 Safari/534.24', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.68 Safari/534.24', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.68 Safari/534.24', 'Mozilla/5.0 Slackware/13.37 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/11.0.696.50', 'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.43 Safari/534.24', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.34 Safari/534.24', 'Mozilla/5.0 (Windows NT 6.0; WOW64) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.34 Safari/534.24', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.3 Safari/534.24', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.3 Safari/534.24', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.3 Safari/534.24', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.14 Safari/534.24', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.12 Safari/534.24', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_6) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.12 Safari/534.24', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/534.24 (KHTML, like Gecko) Ubuntu/10.04 Chromium/11.0.696.0 Chrome/11.0.696.0 Safari/534.24', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.0 Safari/534.24', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.694.0 Safari/534.24', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.23 (KHTML, like Gecko) Chrome/11.0.686.3 Safari/534.23', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.21 (KHTML, like Gecko) Chrome/11.0.682.0 Safari/534.21', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.21 (KHTML, like Gecko) Chrome/11.0.678.0 Safari/534.21', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_7_0; en-US) AppleWebKit/534.21 (KHTML, like Gecko) Chrome/11.0.678.0 Safari/534.21', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.20 (KHTML, like Gecko) Chrome/11.0.672.2 Safari/534.20', 'Mozilla/5.0 (Windows NT) AppleWebKit/534.20 (KHTML, like Gecko) Chrome/11.0.672.2 Safari/534.20', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-US) AppleWebKit/534.20 (KHTML, like Gecko) Chrome/11.0.672.2 Safari/534.20', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.20 (KHTML, like Gecko) Chrome/11.0.669.0 Safari/534.20', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.19 (KHTML, like Gecko) Chrome/11.0.661.0 Safari/534.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.18 (KHTML, like Gecko) Chrome/11.0.661.0 Safari/534.18', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-US) AppleWebKit/534.18 (KHTML, like Gecko) Chrome/11.0.660.0 Safari/534.18', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.17 (KHTML, like Gecko) Chrome/11.0.655.0 Safari/534.17', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.17 (KHTML, like Gecko) Chrome/11.0.655.0 Safari/534.17', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.17 (KHTML, like Gecko) Chrome/11.0.654.0 Safari/534.17', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.17 (KHTML, like Gecko) Chrome/11.0.652.0 Safari/534.17', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.17 (KHTML, like Gecko) Chrome/10.0.649.0 Safari/534.17', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; de-DE) AppleWebKit/534.17 (KHTML, like Gecko) Chrome/10.0.649.0 Safari/534.17', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.82 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux armv7l; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204 Safari/534.16', 'Mozilla/5.0 (X11; U; FreeBSD x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204 Safari/534.16', 'Mozilla/5.0 (X11; U; FreeBSD i386; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204 Safari/534.16', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.204', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.134 Safari/534.16', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.134 Safari/534.16', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.134 Safari/534.16', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.134 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.648.133 Chrome/10.0.648.133 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.648.133 Chrome/10.0.648.133 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.648.127 Chrome/10.0.648.127 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.127 Safari/534.16', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.127 Safari/534.16', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.127 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.11 Safari/534.16', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; ru-RU; AppleWebKit/534.16; KHTML; like Gecko; Chrome/10.0.648.11;Safari/534.16)', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; ru-RU) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.11 Safari/534.16', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.11 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.648.0 Chrome/10.0.648.0 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.648.0 Chrome/10.0.648.0 Safari/534.16', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.0 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.642.0 Chrome/10.0.642.0 Safari/534.16', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.639.0 Safari/534.16', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.638.0 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.634.0 Safari/534.16', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.634.0 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 SUSE/10.0.626.0 (KHTML, like Gecko) Chrome/10.0.626.0 Safari/534.16', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.15 (KHTML, like Gecko) Chrome/10.0.613.0 Safari/534.15', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.15 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.613.0 Chrome/10.0.613.0 Safari/534.15', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.15 (KHTML, like Gecko) Ubuntu/10.04 Chromium/10.0.612.3 Chrome/10.0.612.3 Safari/534.15', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.15 (KHTML, like Gecko) Chrome/10.0.612.1 Safari/534.15', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.15 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.611.0 Chrome/10.0.611.0 Safari/534.15', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/10.0.602.0 Safari/534.14', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/10.0.601.0 Safari/534.14', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/10.0.601.0 Safari/534.14', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/540.0 (KHTML,like Gecko) Chrome/9.1.0.0 Safari/540.0', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/540.0 (KHTML, like Gecko) Ubuntu/10.10 Chrome/9.1.0.0 Safari/540.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/9.0.601.0 Safari/534.14', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Ubuntu/10.10 Chromium/9.0.600.0 Chrome/9.0.600.0 Safari/534.14', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/9.0.600.0 Safari/534.14', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.599.0 Safari/534.13', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-CA) AppleWebKit/534.13 (KHTML like Gecko) Chrome/9.0.597.98 Safari/534.13', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.84 Safari/534.13', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.44 Safari/534.13', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.19 Safari/534.13', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.15 Safari/534.13', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.15 Safari/534.13', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.107 Safari/534.13 v1333515017.9196', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.0 Safari/534.13', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.0 Safari/534.13', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.0 Safari/534.13', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.0 Safari/534.13', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.0 Safari/534.13', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.0 Safari/534.13', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.596.0 Safari/534.13', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Ubuntu/10.04 Chromium/9.0.595.0 Chrome/9.0.595.0 Safari/534.13', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Ubuntu/9.10 Chromium/9.0.592.0 Chrome/9.0.592.0 Safari/534.13', 'Mozilla/5.0 (X11; U; Windows NT 6; en-US) AppleWebKit/534.12 (KHTML, like Gecko) Chrome/9.0.587.0 Safari/534.12', 'Mozilla/5.0 (Windows U Windows NT 5.1 en-US) AppleWebKit/534.12 (KHTML, like Gecko) Chrome/9.0.583.0 Safari/534.12', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.12 (KHTML, like Gecko) Chrome/9.0.579.0 Safari/534.12', 'Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US) AppleWebKit/534.12 (KHTML, like Gecko) Chrome/9.0.576.0 Safari/534.12', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/540.0 (KHTML, like Gecko) Ubuntu/10.10 Chrome/8.1.0.0 Safari/540.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.558.0 Safari/534.10', 'Mozilla/5.0 (X11; U; CrOS i686 0.9.130; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.344 Safari/534.10', 'Mozilla/5.0 (X11; U; CrOS i686 0.9.128; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.343 Safari/534.10', 'Mozilla/5.0 (X11; U; CrOS i686 0.9.128; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.341 Safari/534.10', 'Mozilla/5.0 (X11; U; CrOS i686 0.9.128; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.339 Safari/534.10', 'Mozilla/5.0 (X11; U; CrOS i686 0.9.128; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.339', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Ubuntu/10.10 Chromium/8.0.552.237 Chrome/8.0.552.237 Safari/534.10', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; de-DE) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/533.3', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.215 Safari/534.10', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.215 Safari/534.10', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.215 Safari/534.10', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.210 Safari/534.10', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.200 Safari/534.10', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.551.0 Safari/534.10', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/7.0.548.0 Safari/534.10', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/7.0.544.0 Safari/534.10', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.15) Gecko/20101027 Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/7.0.540.0 Safari/534.10', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/7.0.540.0 Safari/534.10', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; de-DE) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/7.0.540.0 Safari/534.10', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/7.0.540.0 Safari/534.10', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.9 (KHTML, like Gecko) Chrome/7.0.531.0 Safari/534.9', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.8 (KHTML, like Gecko) Chrome/7.0.521.0 Safari/534.8', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.24 Safari/534.7', 'Mozilla/5.0 (X11; U; Linux x86_64; fr-FR) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.514.0 Safari/534.7', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.514.0 Safari/534.7', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.514.0 Safari/534.7', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.6 (KHTML, like Gecko) Chrome/7.0.500.0 Safari/534.6', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.6 (KHTML, like Gecko) Chrome/7.0.498.0 Safari/534.6', 'Mozilla/5.0 (ipad Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.6 (KHTML, like Gecko) Chrome/7.0.498.0 Safari/534.6', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/7.0.0 Safari/700.13', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.4 (KHTML, like Gecko) Chrome/6.0.481.0 Safari/534.4', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.53 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.33 Safari/534.3', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.470.0 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.464.0 Safari/534.3', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.464.0 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.463.0 Safari/534.3', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.462.0 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.462.0 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.461.0 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.461.0 Safari/534.3', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.461.0 Safari/534.3', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.460.0 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.460.0 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.460.0 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.459.0 Safari/534.3', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.458.1 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.458.1 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.458.1 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.458.1 Safari/534.3', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.458.1 Safari/534.3', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.458.0 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.458.0 Safari/534.3', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.457.0 Safari/534.3', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.456.0 Safari/534.3', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.2 (KHTML, like Gecko) Chrome/6.0.454.0 Safari/534.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/534.2 (KHTML, like Gecko) Chrome/6.0.454.0 Safari/534.2', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.2 (KHTML, like Gecko) Chrome/6.0.453.1 Safari/534.2', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-US) AppleWebKit/534.2 (KHTML, like Gecko) Chrome/6.0.453.1 Safari/534.2', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/534.2 (KHTML, like Gecko) Chrome/6.0.453.1 Safari/534.2', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.2 (KHTML, like Gecko) Chrome/6.0.451.0 Safari/534.2', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.1 SUSE/6.0.428.0 (KHTML, like Gecko) Chrome/6.0.428.0 Safari/534.1', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.1 (KHTML, like Gecko) Chrome/6.0.428.0 Safari/534.1', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB) AppleWebKit/534.1 (KHTML, like Gecko) Chrome/6.0.428.0 Safari/534.1', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-US) AppleWebKit/534.1 (KHTML, like Gecko) Chrome/6.0.428.0 Safari/534.1', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.1 (KHTML, like Gecko) Chrome/6.0.427.0 Safari/534.1', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/534.1 (KHTML, like Gecko) Chrome/6.0.422.0 Safari/534.1', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.1 (KHTML, like Gecko) Chrome/6.0.417.0 Safari/534.1', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.1 (KHTML, like Gecko) Chrome/6.0.416.0 Safari/534.1', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/534.1 (KHTML, like Gecko) Chrome/6.0.414.0 Safari/534.1', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.9 (KHTML, like Gecko) Chrome/6.0.400.0 Safari/533.9', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.8 (KHTML, like Gecko) Chrome/6.0.397.0 Safari/533.8', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/6.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.999 Safari/533.4', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.99 Safari/533.4', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.99 Safari/533.4', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.99 Safari/533.4', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_0; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.99 Safari/533.4', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.99 Safari/533.4', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.86 Safari/533.4', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.86 Safari/533.4', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_0; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.86 Safari/533.4', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.70 Safari/533.4', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.127 Safari/533.4', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.126 Safari/533.4', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; fr-FR) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.126 Safari/533.4', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.370.0 Safari/533.4', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.368.0 Safari/533.4', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.366.2 Safari/533.4', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.366.0 Safari/533.4', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.366.0 Safari/533.4', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.363.0 Safari/533.3', 'Mozilla/5.0 (X11; U; OpenBSD i386; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.359.0 Safari/533.3', 'Mozilla/5.0 (X11; U; x86_64 Linux; en_GB, en_US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.358.0 Safari/533.3', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.358.0 Safari/533.3', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.358.0 Safari/533.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.357.0 Safari/533.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.356.0 Safari/533.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.355.0 Safari/533.3', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.354.0 Safari/533.3', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.354.0 Safari/533.3', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.353.0 Safari/533.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.3 (KHTML, like Gecko) Chrome/5.0.353.0 Safari/533.3', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/5.0.343.0 Safari/533.2', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/5.0.343.0 Safari/533.2', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_7_0; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/5.0.342.7 Safari/533.2', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/5.0.342.7 Safari/533.2', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/5.0.342.5 Safari/533.2', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/5.0.342.3 Safari/533.2', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/5.0.342.3 Safari/533.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/5.0.342.2 Safari/533.2', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/5.0.342.1 Safari/533.2', 'Mozilla/5.0 (X11; U; Linux i586; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/5.0.342.1 Safari/533.2', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.2 (KHTML, like Gecko) Chrome/5.0.342.1 Safari/533.2', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/533.1 (KHTML, like Gecko) Chrome/5.0.335.0 Safari/533.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/533.16 (KHTML, like Gecko) Chrome/5.0.335.0 Safari/533.16', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.310.0 Safari/532.9', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.309.0 Safari/532.9', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.308.0 Safari/532.9', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_0; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.307.11 Safari/532.9', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.9 (KHTML, like Gecko) Chrome/5.0.307.1 Safari/532.9', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.1.249.1025 Safari/532.5', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.302.2 Safari/532.8', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.288.1 Safari/532.8', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.8 (KHTML, like Gecko) Chrome/4.0.277.0 Safari/532.8', 'Mozilla/5.0 (X11; U; Slackware Linux x86_64; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.30 Safari/532.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; it-IT) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.25 Safari/532.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.0 Safari/532.5', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_8; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.0 Safari/532.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.246.0 Safari/532.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.4 (KHTML, like Gecko) Chrome/4.0.241.0 Safari/532.4', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.4 (KHTML, like Gecko) Chrome/4.0.237.0 Safari/532.4 Debian', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.3 (KHTML, like Gecko) Chrome/4.0.227.0 Safari/532.3', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.3 (KHTML, like Gecko) Chrome/4.0.224.2 Safari/532.3', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.3 (KHTML, like Gecko) Chrome/4.0.223.5 Safari/532.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.223.4 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.223.3 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE) Chrome/4.0.223.3 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.223.2 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.223.2 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.223.2 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.223.2 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.223.1 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.223.1 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.223.1 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.223.0 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.8 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.7 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.6 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.6 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.6 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.5 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.5 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.5 Safari/532.2', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.5 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.4 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.4 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.4 Safari/532.2', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.4 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.3 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.3 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.3 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.2 Safari/532.2', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.2 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.12 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.12 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.12 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.1 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.222.0 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.221.8 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.221.8 Safari/532.2', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.221.8 Safari/532.2', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.221.8 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.221.7 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.221.6 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.221.6 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.221.6 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.221.3 Safari/532.2', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.2 (KHTML, like Gecko) Chrome/4.0.221.0 Safari/532.2', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.220.1 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.219.6 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.219.5 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.219.5 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.219.4 Safari/532.1', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.219.3 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.219.3 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.219.3 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.219.0 Safari/532.1', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.213.1 Safari/532.1', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.213.1 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.213.1 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.213.1 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.213.1 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.213.1 Safari/532.1', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.213.0 Safari/532.1', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.213.0 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.213.0 Safari/532.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.213.0 Safari/532.1', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_0; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.212.1 Safari/532.1', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.212.1 Safari/532.1', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.212.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.1 (KHTML, like Gecko) Chrome/4.0.212.0 Safari/532.1', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.212.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.212.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.212.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.212.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.212.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.7 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.7 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.4 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.4 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.4 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.2 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.2 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.2 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.2 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.2 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.2 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.211.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.210.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.210.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.209.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.209.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.209.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.209.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.208.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.208.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.208.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.208.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.208.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.207.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.207.0 Safari/532.0', 'Mozilla/5.0 (X11; U; FreeBSD i386; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.207.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.207.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.207.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.207.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.207.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.206.1 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.206.1 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.206.1 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.206.1 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.206.1 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.206.1 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.206.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.206.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.206.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.206.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.205.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.204.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.204.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.204.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.204.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.204.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.4 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.2 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.2 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.2 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.2 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.2 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.2 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.203.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.2 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.2 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0 (x86_64); de-DE) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.2 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; de-DE) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.2 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/4.0.202.0 Safari/525.13.', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.202.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.201.1 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.201.1 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/4.0.201.1 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.201.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.198.1 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.198.1 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.198.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.198.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.198.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.198.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.198 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.198 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.198 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.197.11 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.197.11 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.197.11 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.197.11 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.197.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.197.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.197.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.197 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.196.2 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.196.2 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.196.2 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.196.0 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.196.0 Safari/532.0', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.196 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.6 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.6 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.6 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.6 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.6 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.4 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.33 Safari/532.0', 'Mozilla/4.0 (Windows; U; Windows NT 5.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.33 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.3 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.3 Safari/532.0', 'Mozilla/6.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.27 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.27 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.27 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.27 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML,like Gecko) Chrome/3.0.195.27', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.27 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.27 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.24 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.24 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.21 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.21 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.21 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.21 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.20 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.20 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.17 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.17 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.10 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.10 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.10 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.1 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.1 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.1 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.1 Safari/532.0', 'Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/531.4 (KHTML, like Gecko) Chrome/3.0.194.0 Safari/531.4', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/531.4 (KHTML, like Gecko) Chrome/3.0.194.0 Safari/531.4', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/531.3 (KHTML, like Gecko) Chrome/3.0.193.2 Safari/531.3', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/531.3 (KHTML, like Gecko) Chrome/3.0.193.2 Safari/531.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/531.3 (KHTML, like Gecko) Chrome/3.0.193.2 Safari/531.3', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/531.3 (KHTML, like Gecko) Chrome/3.0.193.0 Safari/531.3', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_7; en-US) AppleWebKit/531.3 (KHTML, like Gecko) Chrome/3.0.192 Safari/531.3', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/531.2 (KHTML, like Gecko) Chrome/3.0.191.3 Safari/531.2', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/531.0 (KHTML, like Gecko) Chrome/3.0.191.0 Safari/531.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/531.0 (KHTML, like Gecko) Chrome/3.0.191.0 Safari/531.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/531.0 (KHTML, like Gecko) Chrome/2.0.182.0 Safari/532.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/531.0 (KHTML, like Gecko) Chrome/2.0.182.0 Safari/531.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/530.0 (KHTML, like Gecko) Chrome/2.0.182.0 Safari/531.0', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.8 (KHTML, like Gecko) Chrome/2.0.178.0 Safari/530.8', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.8 (KHTML, like Gecko) Chrome/2.0.177.1 Safari/530.8', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.8 (KHTML, like Gecko) Chrome/2.0.177.0 Safari/530.8', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.7 (KHTML, like Gecko) Chrome/2.0.177.0 Safari/530.7', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.7 (KHTML, like Gecko) Chrome/2.0.176.0 Safari/530.7', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.7 (KHTML, like Gecko) Chrome/2.0.176.0 Safari/530.7', 'Mozilla/5.0 (X11; U; Linux i686 (x86_64); en-US) AppleWebKit/530.7 (KHTML, like Gecko) Chrome/2.0.175.0 Safari/530.7', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.7 (KHTML, like Gecko) Chrome/2.0.175.0 Safari/530.7', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.6 (KHTML, like Gecko) Chrome/2.0.175.0 Safari/530.6', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/530.6 (KHTML, like Gecko) Chrome/2.0.174.0 Safari/530.6', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.6 (KHTML, like Gecko) Chrome/2.0.174.0 Safari/530.6', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.6 (KHTML, like Gecko) Chrome/2.0.174.0 Safari/530.6', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.174.0 Safari/530.5', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-US) AppleWebKit/530.6 (KHTML, like Gecko) Chrome/2.0.174.0 Safari/530.6', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.173.1 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.173.1 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.173.0 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.8 Safari/530.5', 'Mozilla/6.0 (Windows; U; Windows NT 6.0; en-US) Gecko/2009032609 Chrome/2.0.172.6 Safari/530.7', 'Mozilla/6.0 (Windows; U; Windows NT 6.0; en-US) Gecko/2009032609 (KHTML, like Gecko) Chrome/2.0.172.6 Safari/530.7', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.6 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.43 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.43 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.43 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.43 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.42 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.40 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.40 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.39 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.39 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.23 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.2 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.2 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/530.4 (KHTML, like Gecko) Chrome/2.0.172.0 Safari/530.4', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; eu) AppleWebKit/530.4 (KHTML, like Gecko) Chrome/2.0.172.0 Safari/530.4', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/530.4 (KHTML, like Gecko) Chrome/2.0.172.0 Safari/530.4', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.0 Safari/530.5', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.4 (KHTML, like Gecko) Chrome/2.0.171.0 Safari/530.4', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.1 (KHTML, like Gecko) Chrome/2.0.170.0 Safari/530.1', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.1 (KHTML, like Gecko) Chrome/2.0.169.0 Safari/530.1', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.1 (KHTML, like Gecko) Chrome/2.0.168.0 Safari/530.1', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.1 (KHTML, like Gecko) Chrome/2.0.164.0 Safari/530.1', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.0 (KHTML, like Gecko) Chrome/2.0.162.0 Safari/530.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/530.0 (KHTML, like Gecko) Chrome/2.0.160.0 Safari/530.0', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/528.10 (KHTML, like Gecko) Chrome/2.0.157.2 Safari/528.10', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/528.10 (KHTML, like Gecko) Chrome/2.0.157.2 Safari/528.10', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_0; en-US) AppleWebKit/528.10 (KHTML, like Gecko) Chrome/2.0.157.2 Safari/528.10', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/528.11 (KHTML, like Gecko) Chrome/2.0.157.0 Safari/528.11', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/528.9 (KHTML, like Gecko) Chrome/2.0.157.0 Safari/528.9', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/528.11 (KHTML, like Gecko) Chrome/2.0.157.0 Safari/528.11', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/528.10 (KHTML, like Gecko) Chrome/2.0.157.0 Safari/528.10', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/528.8 (KHTML, like Gecko) Chrome/2.0.156.1 Safari/528.8', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/528.8 (KHTML, like Gecko) Chrome/2.0.156.1 Safari/528.8', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/528.8 (KHTML, like Gecko) Chrome/2.0.156.1 Safari/528.8', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/528.8 (KHTML, like Gecko) Chrome/2.0.156.0 Version/3.2.1 Safari/528.8', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/528.8 (KHTML, like Gecko) Chrome/2.0.156.0 Safari/528.8', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/528.8 (KHTML, like Gecko) Chrome/1.0.156.0 Safari/528.8', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.59 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.59 Safari/525.19', 'Mozilla/4.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.59 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.55 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.55 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.53 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.53 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.53 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.53 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.53 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.50 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.50 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.48 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.46 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.43 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.43 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.43 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.43 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.42 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.39 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.4.154.31 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.4.154.18 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/528.4 (KHTML, like Gecko) Chrome/0.3.155.0 Safari/528.4', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.3.155.0 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.3.154.9 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.3.154.6 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.2.153.1 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.2.153.0 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.2.153.0 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.2.152.0 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.2.152.0 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.2.151.0 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.2.151.0 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.2.151.0 Safari/525.19', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.6 Safari/525.13', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.6 Safari/525.13', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.30 Safari/525.13', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.30 Safari/525.13', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.29 Safari/525.13', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.29 Safari/525.13', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.29 Safari/525.13', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; de) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13', 'Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13(KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13', 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13', 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13', 'Mozilla/5.0 (Linux; U; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13', 'Mozilla/5.0 (Macintosh; U; Mac OS X 10_6_1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/ Safari/530.5', 'Mozilla/5.0 (Macintosh; U; Mac OS X 10_5_7; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/ Safari/530.5', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-US) AppleWebKit/530.9 (KHTML, like Gecko) Chrome/ Safari/530.9', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-US) AppleWebKit/530.6 (KHTML, like Gecko) Chrome/ Safari/530.6', 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/ Safari/530.5' ] ================================================ FILE: src/proxy_spider/items.py ================================================ # -*- coding: utf-8 -*- # Define here the models for your scraped items # # See documentation in: # https://doc.scrapy.org/en/latest/topics/items.html import scrapy from scrapy import Field, Item class Proxy(Item): ip = Field() port = Field() scheme = Field() # HTTP or HTTPS anonymity = Field() url = Field() # formed need_auth = Field() user = Field() password = Field() speed = Field() last_check = Field() fail_times = Field() ================================================ FILE: src/proxy_spider/middlewares.py ================================================ # -*- coding: utf-8 -*- # Define here the models for your spider middleware # # See documentation in: # https://doc.scrapy.org/en/latest/topics/spider-middleware.html import random import time from scrapy import log from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.selector import HtmlXPathSelector from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.response import get_meta_refresh from six.moves.urllib.request import proxy_bypass from service.proxy.proxy import blocking_proxy_srv class CustomRetryMiddleware(RetryMiddleware): def process_response(self, request, response, spider): url = response.url if response.status in [301, 307]: log.msg("trying to redirect us: %s" % url, level=log.INFO) reason = 'redirect %d' % response.status return self._retry(request, reason, spider) or response interval, redirect_url = get_meta_refresh(response) # handle meta redirect if redirect_url: log.msg("trying to redirect us: %s" % url, level=log.INFO) reason = 'meta' return self._retry(request, reason, spider) or response hxs = HtmlXPathSelector(response) # test for captcha page captcha = hxs.select( ".//input[contains(@id, 'captchacharacters')]").extract() if captcha: log.msg("captcha page %s" % url, level=log.INFO) reason = 'capcha' return self._retry(request, reason, spider) or response return response class PureRedisMiddleware(HttpProxyMiddleware): """ Straightly fetch proxy from redis database. Waiting to be reconstructed. """ def __init__(self, crawler, auth_encoding): self.anonymity = 'anonymous' self.crawler = crawler self.logger = crawler.spider.logger self.auth_encoding = auth_encoding self.srv = blocking_proxy_srv self.settings = crawler.settings self.use_proxy_rate = self.settings.get('USE_PROXY_TO_CRAWL', 1) # print(self.raw_rate) def fetch_proxy(self, scheme): """ Get proxy from fpserver by given scheme. :scheme: `str` proxy protocol :return: url, scheme """ result = None params = { "scheme": scheme, # "anonymity": self.anonymity, "count": 1, } try: keys = self.srv.query(params, return_keys=True) except: self.logger.exception("Failed to fetch proxy") else: if keys: url = self.srv.cli.hget(keys[0], 'url') result = self._get_proxy(url, scheme) return result @classmethod def from_crawler(cls, crawler): auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING', 'latin-l') return cls(crawler, auth_encoding) def _set_proxy(self, request, scheme): _fetched = self.fetch_proxy(scheme) if not _fetched: return creds, proxy = _fetched request.meta['proxy'] = proxy self.logger.debug('Applied proxy: %s' % proxy) if creds: request.headers['Proxy-Authorization'] = b'Basic' + creds def process_request(self, request, spider): # ignore if proxy is already set if 'proxy' in request.meta: if request.meta['proxy'] is None: return # extract credentials if present creds, proxy_url = self._get_proxy(request.meta['proxy'], '') request.meta['proxy'] = proxy_url if creds and not request.headers.get('Proxy-Authorization'): request.headers['Proxy-Authorization'] = b'Basic ' + creds return parsed = urlparse_cached(request) scheme = parsed.scheme if scheme in ('http', 'https') and proxy_bypass(parsed.hostname): return if self.use_proxy_rate < 1: if random.random() < self.use_proxy_rate: self._set_proxy(request, scheme) else: self._set_proxy(request, scheme) class RandomUserAgentMiddleware: """This middleware allows spiders to override the user_agent""" def __init__(self): self.user_agents = None def spider_opened(self, spider): from proxy_spider.const import user_agents self.user_agents = user_agents def process_request(self, request, spider): if self.user_agents: ua = random.choice(self.user_agents) request.headers.setdefault(b'User-Agent', ua) class TimerMiddleware: def process_request(self, request, spider): request.meta['_start_time'] = time.time() ================================================ FILE: src/proxy_spider/pipelines.py ================================================ # -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html from scrapy.exceptions import DropItem from proxy_spider.items import Proxy from service.proxy.proxy import blocking_proxy_srv from utils.tools import subdict class PersistencePipeline: def __init__(self): self.srv = blocking_proxy_srv self.cli = self.srv.cli def process_item(self, item, spider): if item and isinstance(item, Proxy): try: self.delete_existed(item, spider) key = self.srv.save_proxy(item) spider.logger.debug('Stored: %s' % key) except: spider.logger.exception("Item: %s" % item) return item def delete_existed(self, item, spider): _f = subdict(item, ['ip', 'port', 'scheme']) for k in self.srv.keys_by_dict(_f): self.srv.delete(k) spider.logger.debug('Deleted: %s' % k) ================================================ FILE: src/proxy_spider/settings.py ================================================ # -*- coding: utf-8 -*- # Scrapy settings for proxy_spider project # # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # # https://doc.scrapy.org/en/latest/topics/settings.html # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html # https://doc.scrapy.org/en/latest/topics/spider-middleware.html import os import config SPIDER_MODULES = ['proxy_spider.spiders'] NEWSPIDER_MODULE = 'proxy_spider.spiders' BOT_NAME = 'proxy_spider' def _get_log_config(): """ :return: file, level """ _LEVEL = 'INFO' _FILE = None log_config = getattr(config, 'LOG', {}) # fix: divide log from tornado log if not getattr(config, 'CONSOLE_OUTPUT', 0): dirname = log_config.get('dir', './logs') dirname = os.path.expanduser(dirname) if not os.path.exists(dirname): os.mkdir(dirname) _FILE = os.path.join(dirname, 'spider.log') main_level = log_config.get('level') if main_level: _LEVEL = main_level.upper() return _FILE, _LEVEL # LOGGING LOG_STDOUT = True LOG_FILE, LOG_LEVEL = _get_log_config() RETRY_TIMES = 10 RETRY_HTTP_CODES = [400, 500, 502, 503, 504, 408] # Enable or disable downloader middlewares # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html DOWNLOADER_MIDDLEWARES = { 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None, 'proxy_spider.middlewares.RandomUserAgentMiddleware': 543, 'proxy_spider.middlewares.PureRedisMiddleware': 745, 'proxy_spider.middlewares.TimerMiddleware': 746, } # Enable and configure HTTP caching (disabled by default) # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings HTTPCACHE_ENABLED = True HTTPCACHE_EXPIRATION_SECS = 900 HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' # Enable and configure the AutoThrottle extension (disabled by default) # See https://doc.scrapy.org/en/latest/topics/autothrottle.html AUTOTHROTTLE_ENABLED = True # The initial download delay AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies AUTOTHROTTLE_MAX_DELAY = 30 # The average number of requests Scrapy should be sending in parallel to # each remote server # AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: AUTOTHROTTLE_DEBUG = False # Obey robots.txt rules ROBOTSTXT_OBEY = False # Configure item pipelines # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html ITEM_PIPELINES = { 'proxy_spider.pipelines.PersistencePipeline': 333, } # Configure maximum concurrent requests performed by Scrapy (default: 16) CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs DOWNLOAD_DELAY = 1 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 8 #CONCURRENT_REQUESTS_PER_IP = 16 # Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False # Override the default request headers: DEFAULT_REQUEST_HEADERS = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'Accept-Language': 'h-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4', 'Accept-Encoding': 'gzip, deflate, br', 'Connection': 'keep-alive', } # Enable or disable spider middlewares # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html # SPIDER_MIDDLEWARES = { # 'proxy_spider.middlewares.ProxyspiderSpiderMiddleware': 543, # } # Enable or disable extensions # See https://doc.scrapy.org/en/latest/topics/extensions.html # EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None, # } EDITOR = 'vim' # A rate USE_PROXY_TO_CRAWL = 0.7 ================================================ FILE: src/proxy_spider/spiders/__init__.py ================================================ # This package will contain the spiders of your Scrapy project # # Please refer to the documentation for information on how to create and manage # your spiders. import json import random import time from scrapy import Request, exceptions from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import CrawlSpider from twisted.internet.error import (DNSLookupError, TCPTimedOutError, TimeoutError) import requests import config from proxy_spider import utils from proxy_spider.items import Proxy from service.proxy.functions import build_key, valid_format from service.proxy.proxy import blocking_proxy_srv LOCAL_IP = None IP_CHECKER_API = 'http://api.ipify.org/?format=json' def get_local_ip(): global LOCAL_IP if LOCAL_IP: return LOCAL_IP else: r = requests.get(IP_CHECKER_API) j = json.loads(r.text) LOCAL_IP = j['ip'] return LOCAL_IP class _BaseSpider(CrawlSpider): srv = blocking_proxy_srv def complete_condition(self): num = int(getattr(config, 'PROXY_STORE_NUM', 0)) keys = self.srv.get_all_keys() return len(keys) >= num def already_exists(self, spec): result = False existed = self.srv.keys_by_dict(spec) if existed: result = True return result def get_check_approach(self, scheme): # default_timeout = 10 _both = [ ('{scheme}://httpbin.org/ip', self.parse_httpbin), ('{scheme}://ipduh.com/anonymity-check/', self.parse_ipduh), ('{scheme}://api.ipify.org/?format=json', self.parse_ipify), ] _http = [ # ('http://ip-check.info/?lang=en/', self.parse_ipcheck), ] _https = [ ] if scheme == 'http': result = random.choice(_both + _http) else: result = random.choice(_both + _https) return result def build_check_recipient(self, ip, port, scheme, user=None, password=None): """ 1. build a request for availability checking 2. drop it if already existed :return: Request """ if self.complete_condition(): raise exceptions.CloseSpider('Enough items') spec = dict(ip=ip, port=port, scheme=scheme) if not valid_format(spec): self.logger.debug('Got wrong format (%s, %s). Clear it.' % (ip, port)) return {} if self.already_exists(spec): self.logger.debug('Got duplicated %s. Clear it.' % spec.values()) return {} # drop it proxy_url = utils.build_proxy_url(ip, port, scheme, user, password) need_auth = int(bool(user and password)) item = Proxy( ip=ip, scheme=scheme, port=port, need_auth=need_auth, url=proxy_url, ) if need_auth: item['user'], item['password'] = user, password self.logger.debug('Got unchecked %s' % item) return self.build_check_request(item) def build_check_request(self, item: Proxy): scheme = item.get('scheme') proxy_url = item.get('url') self.logger.debug('Checking %s' % proxy_url) url, response_parser = self.get_check_approach(scheme) url = url.format(scheme=scheme) timeout = getattr(config, 'CHECK_TIMEOUT', 20) meta = { 'proxy': proxy_url, 'max_retry_times': 5, 'download_timeout': timeout, '_item_obj': item, '_response_parser': response_parser, } req = Request(url, callback=self.check_ip, meta=meta, dont_filter=True) if self.name == 'checker': req.errback = self.check_ip_failed return req def check_ip(self, response): """ check ip's availability and anonymity """ item = response.meta['_item_obj'] cur_ts = time.time() item['last_check'] = int(cur_ts) item['speed'] = cur_ts - response.meta['_start_time'] item['fail_times'] = 0 parser = response.meta['_response_parser'] got_ip = '' try: got_ip = parser(response) or '' except: self.logger.exception( 'While checking %s with %s, ' 'Failed to parse response %s. ' % (item, response.url, response.text) ) if got_ip.strip() == get_local_ip(): item['anonymity'] = 'transparent' else: item['anonymity'] = 'anonymous' yield item def check_ip_failed(self, failure): self.logger.error(repr(failure)) item = failure.request.meta['_item_obj'] key = build_key(item) self.srv.add_failure(key) if failure.check(HttpError): response = failure.value.response self.logger.error('HttpError on %s', response.url) elif failure.check(DNSLookupError): request = failure.request self.logger.error('DNSLookupError on %s', request.url) elif failure.check(TimeoutError, TCPTimedOutError): request = failure.request self.logger.error('TimeoutError on %s', request.url) def parse_httpbin(self, response): return json.loads(response.text)['origin'] def parse_ipcheck(self, response): return response.xpath('//section[@id="content"]/h1[2]/span/a/text()').get() def parse_ipduh(self, response): return response.xpath( '//table[@id="hm"]/tr/td[contains(text(), "public IP address")]/following-sibling::td/text()').get() def parse_ipify(self, response): return json.loads(response.text)['ip'] ================================================ FILE: src/proxy_spider/spiders/a3464.py ================================================ # -*- coding: utf-8 -*- from proxy_spider.spiders import _BaseSpider class A3464Spider(_BaseSpider): name = '3464' allowed_domains = ['3464.com'] start_urls = ['http://www.3464.com/data/Proxy/http/'] custom_settings = { 'RETRY_TIMES': 3, 'DOWNLOAD_TIMEOUT': 20 } def parse(self, response): info = response.xpath( '//div[@class="CommonBody"]/table[6]/tr[4]/td/table/tr' ) for x in info[1:]: data = x.xpath('td/text()').extract() if data: ip = data[0] port = data[1] scheme = 'http' yield self.build_check_recipient(ip, port, scheme) ================================================ FILE: src/proxy_spider/spiders/checker.py ================================================ # coding: utf-8 from proxy_spider.items import Proxy from proxy_spider.spiders import _BaseSpider from service.proxy.functions import exceed_check_period, valid_format class CheckerSpider(_BaseSpider): """ Check proxy's availability and anonymity. """ name = 'checker' # allowed_domains = ['*'] custom_settings = { 'DOWNLOAD_DELAY': 3, 'CONCURRENT_REQUESTS_PER_DOMAIN': 8, } def start_requests(self): keys = self.srv.get_all_keys() for key in keys: data = self.srv.hgetall_dict(key) last_check = data.get('last_check', 0) if not valid_format(data): self.srv.delete(key, 'Error format %s' % data) continue if exceed_check_period(last_check): item = Proxy(**data) yield self.build_check_request(item) ================================================ FILE: src/proxy_spider/spiders/coderbusy.py ================================================ # -*- coding: utf-8 -*- from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Rule from proxy_spider.spiders import _BaseSpider class CoderbusySpider(_BaseSpider): name = 'coderbusy' allowed_domains = ['coderbusy.com'] start_urls = [ 'https://proxy.coderbusy.com/classical/https-ready.aspx', 'https://proxy.coderbusy.com/classical/anonymous-type/anonymous.aspx', 'https://proxy.coderbusy.com/classical/anonymous-type/highanonymous.aspx', 'https://proxy.coderbusy.com/classical/anonymous-type/transparent.aspx', ] custom_settings = { "DOWNLOAD_TIMEOUT": 20, } rules = ( Rule(LinkExtractor(allow=(r'classical.+aspx.*page=\d+$', )), callback='parse_items', follow=True), ) def parse_items(self, response): for tr in response.xpath('//table[@class="table"]/tbody/tr'): td = tr.xpath('./td[@class="port-box"]') ip = td.xpath('@data-ip').get() port = td.xpath('@data-i').get() for scheme in ('http', 'https'): yield self.build_check_recipient(ip, port, scheme) ================================================ FILE: src/proxy_spider/spiders/coolproxy.py ================================================ # -*- coding: utf-8 -*- import base64 import re from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Rule from proxy_spider.spiders import _BaseSpider from utils.tools import str_rot13 class CoolproxySpider(_BaseSpider): name = 'coolproxy' allowed_domains = ['cool-proxy.net'] start_urls = [ 'https://www.cool-proxy.net/proxies/http_proxy_list/country_code:/port:/anonymous:', 'https://www.cool-proxy.net/proxies/http_proxy_list/country_code:/port:/anonymous:/page:2', 'https://www.cool-proxy.net/proxies/http_proxy_list/country_code:/port:/anonymous:/page:3', ] rules = ( Rule( link_extractor=LinkExtractor( allow=(r'proxies.*page:\d+.*',), restrict_xpaths=('//th[@class="pagination"]',), ), callback='parse_items', follow=True), ) def parse_items(self, response): for tr in response.xpath('//table//tr')[1:]: ip_js = tr.xpath('./td/script/text()').get() if not ip_js: continue searched = re.search(r'str_rot13\("(.*)"', ip_js) if not searched: continue ip_encoded = searched.group(1) ip = base64.b64decode(str_rot13(ip_encoded)).decode('utf-8') if ip: ex = tr.xpath('./td/text()').extract() port = ex[0] for scheme in ('http', 'https'): yield self.build_check_recipient(ip, port, scheme) ================================================ FILE: src/proxy_spider/spiders/data5u.py ================================================ # coding: utf-8 from proxy_spider.spiders import _BaseSpider class Data5uSpider(_BaseSpider): name = "data5u" allowed_domains = ["data5u.com"] # custom_settings = { # # "DOWNLOAD_TIMEOUT": 20, # } start_urls = [ "http://www.data5u.com/free/gngn/index.shtml", "http://www.data5u.com/free/gnpt/index.shtml", "http://www.data5u.com/free/gwgn/index.shtml", "http://www.data5u.com/free/gwpt/index.shtml", "http://www.data5u.com/free/index.shtml", ] def parse(self, response): iplist = response.xpath('//ul[@class="l2"]') for x in iplist[1:-1]: ip = x.xpath('span[1]/li/text()').extract_first() port = x.xpath('span[2]/li/text()').extract_first() scheme = str( x.xpath('span[4]/li/a/text()').extract_first()).upper() # type = x.xpath('span[3]/li/a/text()').extract_first() # print(ip, port, scheme, type) yield self.build_check_recipient(ip, port, scheme) ================================================ FILE: src/proxy_spider/spiders/ihuan.py ================================================ # -*- coding: utf-8 -*- from scrapy import Request from proxy_spider.spiders import _BaseSpider from utils.collections import shuffled_range class IhuanSpider(_BaseSpider): name = 'ihuan' allowed_domains = ['www.ihuan.com'] def start_requests(self): # pages = shuffled_range(1, 100) for _page in range(1, 100): if self.complete_condition(): break url = 'https://ip.ihuan.me/?page=%s' % _page yield Request(url, dont_filter=True) def parse(self, response): for tr in response.xpath('//table/tbody/tr'): ex = tr.xpath('./td//text()').extract() ip = ex[0] port = ex[1] scheme = ['http', 'https'][ex[5] == '支持'] print(ip, port, scheme) if ip and port and scheme in ('http', 'https'): yield self.build_check_recipient(ip, port, scheme) ================================================ FILE: src/proxy_spider/spiders/ip66.py ================================================ # -*- coding: utf-8 -*- from urllib.parse import urljoin from scrapy import Request from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Rule from proxy_spider.spiders import _BaseSpider from utils.collections import shuffled_range class Ip66Spider(_BaseSpider): name = 'ip66' allowed_domains = ['www.66ip.cn'] rules = ( Rule(LinkExtractor(allow=(r'/\d+\.html$',)), callback='parse_items', follow=True), ) def start_requests(self): base = 'http://www.66ip.cn' meta = { 'max_retry_times': 10, # 'download_timeout': 20, } # pages = shuffled_range(1, 100) for _page in range(1, 100): if self.complete_condition(): break url = urljoin(base, '/%s.html' % _page) yield Request(url, meta=meta, callback=self.parse_items, dont_filter=True) def parse_items(self, response): for tr in response.xpath('//div[@id="main"]//table/tr')[1:]: ex = tr.xpath('./td/text()').extract() ip = ex[0] port = ex[1] if ip and port: for scheme in ('http', 'https'): yield self.build_check_recipient(ip, port, scheme) ================================================ FILE: src/proxy_spider/spiders/ip89.py ================================================ # -*- coding: utf-8 -*- from urllib.parse import urljoin from scrapy import Request from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Rule from proxy_spider.spiders import _BaseSpider class Ip89Spider(_BaseSpider): name = 'ip89' allowed_domains = ['www.89ip.cn'] rules = ( Rule(LinkExtractor(allow=(r'index_\d+\.html$',)), callback='parse_items', follow=True), ) def start_requests(self): base = 'http://www.89ip.cn' meta = { 'max_retry_times': 10, # 'download_timeout': 20, } # pages = shuffled_range(1, 100) for _page in range(1, 100): if self.complete_condition(): break url = urljoin(base, '/index_%s.html' % _page) yield Request(url, callback=self.parse_items, meta=meta, dont_filter=True) def parse_items(self, response): for tr in response.xpath('//table[@class="layui-table"]/tbody/tr'): ex = tr.xpath('./td/text()').extract() ip = ex[0].strip() port = ex[1].strip() if ip and port: for scheme in ('http', 'https'): yield self.build_check_recipient(ip, port, scheme) ================================================ FILE: src/proxy_spider/spiders/kuaidaili.py ================================================ # -*- coding: utf-8 -*- from scrapy import Request from proxy_spider.spiders import _BaseSpider from utils.collections import shuffled_range class KuaidailiSpider(_BaseSpider): name = 'kuaidaili' allowed_domains = ['www.kuaidaili.com'] def start_requests(self): meta = { # 'max_retry_times': 10, # 'download_timeout': 20, } pages = range(1, 100) for _type in ('inha', 'intr'): for _page in pages: if self.complete_condition(): break url = 'http://www.kuaidaili.com/free/%s/%s/' % (_type, _page) yield Request(url, meta=meta, dont_filter=True) def parse(self, response): for tr in response.xpath('//table/tbody/tr'): ex = tr.xpath('./td/text()').extract() ip = ex[0] port = ex[1] scheme = ex[3].lower() if ip and port and scheme in ('http', 'https'): yield self.build_check_recipient(ip, port, scheme) ================================================ FILE: src/proxy_spider/spiders/mix.py ================================================ # -*- coding: utf-8 -*- """ for some little spiders """ import json from scrapy import Request from proxy_spider.spiders import _BaseSpider from service.proxy.functions import IP_PORT_PATTERN class MixSpider(_BaseSpider): name = 'mix' # allowed_domains = ['coderbusy.com'] # custom_settings = { # "DOWNLOAD_TIMEOUT": 20, # } def start_requests(self): # ip181 url_181 = 'http://www.ip181.com/' yield Request(url_181, dont_filter=True, callback=self.parse_ip181) # a2u url_a2u = 'https://raw.githubusercontent.com/a2u/free-proxy-list/master/free-proxy-list.txt' yield Request(url_a2u, dont_filter=True, callback=self.parse_a2u) # iphai for url in [ 'http://www.iphai.com/free/ng', 'http://www.iphai.com/free/np', 'http://www.iphai.com/free/wg', 'http://www.iphai.com/free/wp', ]: yield Request(url, dont_filter=True, callback=self.parse_iphai) def parse_iphai(self, response): data = response.xpath( '//table[@class="table table-bordered table-striped table-hover"]/tr') if data: for x in data[1:]: info = x.xpath('td/text()').extract() ip = info[0].strip() scheme = info[3].strip() port = info[1].strip() yield self.build_check_recipient(ip, port, scheme) def parse_a2u(self, response): for p in response.text.split(): p = p.strip() if IP_PORT_PATTERN.match(p): ip, port = p.split(':') for scheme in ('http', 'https'): yield self.build_check_recipient(ip, port, scheme) def parse_ip181(self, response): if response.text: j = json.loads(response.text) for i in j['RESULT']: ip = i.get('ip') port = i.get('port') if ip and port: for scheme in ('http', 'https'): yield self.build_check_recipient(ip, port, scheme) ================================================ FILE: src/proxy_spider/spiders/xicidaili.py ================================================ # -*- coding: utf-8 -*- from scrapy import Request from proxy_spider.spiders import _BaseSpider from utils.collections import shuffled_range class XicidailiSpider(_BaseSpider): name = 'xicidaili' allowed_domains = ['www.xicidaili.com'] def start_requests(self): for _type in ('nn', 'nt'): for _page in range(1, 100): if self.complete_condition(): break url = 'http://www.xicidaili.com/%s/%s' % (_type, _page) yield Request(url, dont_filter=True) def parse(self, response): for tr in response.xpath('//table[@id="ip_list"]//tr[@class]'): ex = tr.xpath('./td/text()').extract() ip = ex[0] port = ex[1] scheme = ex[5].lower() if ip and port and scheme in ('http', 'https'): yield self.build_check_recipient(ip, port, scheme) ================================================ FILE: src/proxy_spider/spiders/yundaili.py ================================================ # -*- coding: utf-8 -*- from scrapy import Request from proxy_spider.spiders import _BaseSpider class YundailiSpider(_BaseSpider): name = 'yundaili' allowed_domains = ['www.ip3366.net'] def start_requests(self): url = 'http://www.ip3366.net/free/?page=1' # url = 'http://example.com' meta = { 'max_retry_times': 10, # 'download_timeout': 10, } yield Request(url, dont_filter=True, meta=meta) def parse(self, response): for p in response.xpath('//table/tbody/tr'): ex = p.xpath('./td/text()').extract() ip = ex[0] port = ex[1] scheme = ex[3].lower() if ip and port and scheme in ('http', 'https'): yield self.build_check_recipient(ip, port, scheme) ================================================ FILE: src/proxy_spider/utils.py ================================================ # coding: utf-8 def build_proxy_url(ip, port, scheme, user=None, password=None, *args, **kw): need_auth = int(bool(user and password)) auth_part = '' if need_auth: auth_part = '%s:%s@' % (user, password) result = '{scheme}://{auth}{ip}:{port}'.format( ip=ip, port=port, scheme=scheme, auth=auth_part, ) return result ================================================ FILE: src/scrapy.cfg ================================================ # Automatically created by: scrapy startproject # # For more information about the [deploy] section see: # https://scrapyd.readthedocs.io/en/latest/deploy.html [settings] default = proxy_spider.settings [deploy] #url = http://localhost:6800/ project = proxy_spider ================================================ FILE: src/service/__init__.py ================================================ ================================================ FILE: src/service/proxy/__init__.py ================================================ ================================================ FILE: src/service/proxy/functions.py ================================================ # coding: utf-8 import re import time import config key_prefix = 'proxy_' searchable_keys = ('anonymity', 'scheme', 'ip', 'port') IP_PATTERN = re.compile(r'^(\d+\.){3}\d+$') PORT_PATTERN = re.compile(r'^\d+$') IP_PORT_PATTERN = re.compile(r'^(\d+\.){3}\d+:\d+$') def get_searchable_spec(spec): """ :spec: filter the search conditions :return: dict """ _spec = {} if spec: _spec = {k: v for k, v in spec.items() if k in searchable_keys} return _spec def build_key(item): """ proxy items' key in redis """ key = '{prefix}{anonymity}:{scheme}:{ip}:{port}'.format( prefix=key_prefix, anonymity=item['anonymity'], scheme=item['scheme'], ip=item['ip'], port=item['port'], ) return key def build_pattern(spec): """ build pattern for searching :return: str """ _pattern = '%s%s:%s:%s:%s' % ( key_prefix, spec.get('anonymity') or '*', spec.get('scheme') or '*', spec.get('ip') or '*', spec.get('port') or '*', ) return _pattern def exceed_check_period(last_check): interval = config.PROXY_STORE_CHECK_SEC now = int(time.time()) return now - int(last_check) > int(interval) def valid_format(proxy_item): """ check the format of ip and port :return: bool """ scheme = proxy_item.get('scheme', '') ip = proxy_item.get('ip', '') port = proxy_item.get('port', '') anonymity = proxy_item.get('anonymity', 'transparent') if not isinstance(port, str): port = str(port) def _conditions(): yield scheme and ip and port yield scheme.lower() in ('http', 'https') yield anonymity in ('anonymous', 'transparent') yield IP_PATTERN.match(ip) yield PORT_PATTERN.match(port) return all(_conditions()) ================================================ FILE: src/service/proxy/proxy.py ================================================ # -*- coding:utf-8 -*- # TODO: recon this ugly module """ Persistence utils for `proxy` only """ import random from itertools import chain from scrapy import Item from service.proxy.functions import (build_pattern, get_searchable_spec, key_prefix) from service.proxy.serializers import ProxySerializer from utils import log as logger FAIL_TIMES_MAXIMUM = 5 class _ProxyServerBase: """ common function """ # TODO class BlockingProxyServer(_ProxyServerBase): """ pyredis version """ def __init__(self): from core.db.redis import pyredis_pool self.cli = pyredis_pool.acquire() self.cli.hmset_dict = self.hmset_dict def get_all_keys(self): return self.cli.keys('%s*' % key_prefix) or [] def hgetall_dict(self, key): keys = self.cli.hkeys(key) vals = self.cli.hvals(key) return dict(zip(keys, vals)) def hmset_dict(self, key, item): if not isinstance(item, (dict, Item)): raise TypeError("Error type: %s" % type(item)) if not item: raise ValueError("item is empty") args = chain.from_iterable(item.items()) return self.cli.hmset(key, *args) def delete(self, key, reason='Not specified.'): self.cli.delete(key) logger.info('Remove key: %s Reason: %s' % (key, reason)) def add_failure(self, key): item = self.hgetall_dict(key) s = ProxySerializer(item) fail_times = int(item.get('fail_times', 0)) current_fail_times = fail_times+1 if current_fail_times >= FAIL_TIMES_MAXIMUM: self.delete(key, 'Failed %s times' % fail_times) else: if s.is_valid(): self.cli.hincrby(key, 'fail_times', 1) logger.debug('Key: %s Fail times: %s' % (key, current_fail_times)) def save_proxy(self, item: dict): s = ProxySerializer(item) if s.is_valid(raise_e=True): save_res = self.cli.hmset_dict(s.key, s.validated_data) logger.debug( 'Saving checked proxy: %s ' 'HMSET result: %s' % (s.validated_data, save_res) ) return s.key def query(self, spec, return_keys=False): result = [] _spec = get_searchable_spec(spec) count = spec.get('count', 1) if count: keys = self.get_random_keys(count, _spec) if return_keys: result = keys else: for key in keys: item = self.hgetall_dict(key) result.append(item) return result def get_random_keys(self, count, spec=None): """ :count: default 1 :spec: search conditions """ keys = self.keys_by_dict(spec) or [] if keys and len(keys) > count: keys = random.sample(keys, count) return keys def keys_by_dict(self, spec): _pattern = build_pattern(spec) keys = self.cli.keys(_pattern) return keys or [] class ProxyServer(_ProxyServerBase): """ aioredis version """ @property def cli(self): from core.db.redis import aioredis_pool return aioredis_pool async def get_all_status(self): all_keys = await self.get_all_keys() num = len(all_keys) http_keys = await self.keys_by_dict({"scheme": "http"}) http_num = len(http_keys) https_num = num - http_num trans_keys = await self.keys_by_dict({"anonymity": "transparent"}) trans_num = len(trans_keys) anony_num = num - trans_num ret = { "total": len(all_keys), "detail": { "http": http_num, "https": https_num, "transparent": trans_num, "anonymous": anony_num, } } return ret async def get_all_keys(self): return await self.cli.keys('%s*' % key_prefix) or [] async def add_failure(self, key): item = await self.cli.hgetall(key) fail_times = int(item.get('fail_times', 0)) s = ProxySerializer(item) current_fail_times = fail_times+1 if current_fail_times >= FAIL_TIMES_MAXIMUM: await self.cli.delete(key) logger.info('Remove key: %s' % key) else: if s.is_valid(): await self.cli.hincrby(key, 'fail_times', 1) logger.debug('Key: %s Fail times: %s' % (key, current_fail_times)) async def new_proxy(self, item): s = ProxySerializer(item) if s.is_valid(raise_e=True): save_res = await self.cli.hmset_dict(s.key, s.validated_data) logger.debug( 'Saving checked proxy: %s ' 'HMSET result: %s' % (s.validated_data, save_res) ) return s.key async def query(self, spec, return_keys=False): result = [] _spec = get_searchable_spec(spec) count = spec.get('count', 1) if count: keys = await self.get_random_keys(count, _spec) if return_keys: result = keys else: for key in keys: item = await self.cli.hgetall(key) result.append(item) return result async def get_random_keys(self, count, spec=None): """ :count: default 1 :spec: search conditions """ keys = await self.keys_by_dict(spec) or [] if keys and len(keys) > count: keys = random.sample(keys, count) return keys async def keys_by_dict(self, spec): _pattern = build_pattern(spec) keys = await self.cli.keys(_pattern) return keys or [] proxy_srv = ProxyServer() blocking_proxy_srv = BlockingProxyServer() __all__ = [proxy_srv, blocking_proxy_srv] ================================================ FILE: src/service/proxy/serializers.py ================================================ # coding: utf-8 from copy import deepcopy from core import exceptions from proxy_spider.items import Proxy from service.proxy import functions from proxy_spider.utils import build_proxy_url class ProxySerializer: """ simple serializer """ excluded = ['speed', 'last_check', 'fail_times'] def __init__(self, item: dict): self._item = item def get_value(self, key): assert hasattr(self, '_is_valid'), ( 'You must call `.is_valid` at first' ) return self._item.get(key) def is_valid(self, raise_e=False): """ :param raise_e: whether raise exception :rtype: bool """ self._is_valid = True validators = ( self._validate_indispensibles, self._validate_type, self._validate_format, ) try: for _v in validators: _v() except Exception as e: self._is_valid = False if raise_e: raise e return self._is_valid @property def data(self): return self._item @property def validated_data(self): assert hasattr(self, '_is_valid'), ( 'You must call `.is_valid` at first' ) assert self._is_valid, 'Data is not valid.' result = dict(self._item) result['scheme'] = result['scheme'].lower() result.setdefault('url', build_proxy_url(**result)) result.setdefault('last_check', 0) result.setdefault('fail_times', 0) result.setdefault('need_auth', 0) result.setdefault('anonymity', 'transparent') print(result) return result @property def key(self): """ key that used in redis """ assert hasattr(self, '_is_valid'), ( 'You must call `.is_valid` at first' ) assert self._is_valid, 'Invalid data.' return functions.build_key(self.validated_data) def to_representation(self): """ only used in api """ return { k: v for k, v in self._item.items() if k not in self.excluded } def _validate_indispensibles(self): indispensibles = ('scheme', 'ip', 'port') for k in indispensibles: if k not in self._item: raise exceptions.ValidationError('%s cannot be empty.' % k) def _validate_type(self): if not isinstance(self._item, Proxy): self._item = Proxy(self._item) def _validate_format(self): fmt_ok = functions.valid_format(self._item) assert fmt_ok, 'Format error %s' % self._item ================================================ FILE: src/service/spider/__init__.py ================================================ ================================================ FILE: src/service/spider/functions.py ================================================ # coding: utf-8 # corresponding key's prefix in redis while spider is running key_prefix = 'fp_server:spider:' key_prefix_checker = '%schecker:' % key_prefix key_prefix_seeker = '%sseeker:' % key_prefix # import config # def get_specific_settings(spider_cls): # result = {} # # if not config.CONSOLE_OUTPUT: # from proxy_spider import settings # filename = 'spider_%s.log' % spider_cls.name # log_file = settings._get_log_path(filename) # result['LOG_FILE'] = log_file # # return result def prefix_by_type(_type=None): if _type: result = { 'checker': key_prefix_checker, 'seeker': key_prefix_seeker, }[_type] else: result = key_prefix return result def build_key(spider_cls, _type): """ corresponding key in redis while spider is running """ _prefix = prefix_by_type(_type) return '%s%s' % (_prefix, spider_cls.name) def updated_crawler_settings(origin_settings, spec: dict): new_settings = origin_settings.copy() new_settings.setdict(spec) return new_settings ================================================ FILE: src/service/spider/spider.py ================================================ # coding: utf-8 import time from core.db.redis import aioredis_pool, pyredis_pool from service.spider.functions import prefix_by_type from utils import log as logger class SpiderServer: def __init__(self): self.cli = aioredis_pool self.blocking_cli = pyredis_pool.acquire() # XXX: It's ugly async def all_status(self, _type=None, with_key=False): """ get all status about spiders :param _type: spider's type :param with_key: whether return key in redis """ key_prefix = prefix_by_type(_type) keys = await self.cli.keys('%s*' % key_prefix) or [] result = [] for key in keys: item = await self.cli.hgetall(key) if item['status'] == 'running': st = int(item.get('last_start_time')) item['total_time'] = int(time.time()) - st if with_key: item['key'] = key result.append(item) return result async def register_status(self, key): """ :param key: will be deleted after crawling or last for some hours :param detail: :return: """ t = int(time.time()) spec = { 'last_start_time': t, 'status': 'running', } await self.cli.hmset_dict(key, spec) return t async def unregister_status(self, st, key, *args, **kw): """ :param st: start time :param key: key in db :param args: preserved :param kw: preserved :return: preserved """ st = await self.cli.hget(key, 'last_start_time') total_time = int(time.time()) - int(st) await self.cli.hset(key, "status", 'stopped') return total_time def callback_unregister_status(self, _, st, key, *args, **kw): """ as callback of crawling * Twisted doesn't support aioredis :param _: preserved for fired deffer :param st: start time :param key: key in db :param args: preserved :param kw: preserved :return: preserved """ total_time = int(time.time()) - int(st) logger.info("One spider finished working. " "key: %s. Total time: %s" % (key, total_time)) return self.blocking_cli.hset(key, 'status', 'stopped'), total_time spider_srv = SpiderServer() __all__ = [spider_srv] ================================================ FILE: src/service/tasks/__init__.py ================================================ ================================================ FILE: src/service/tasks/spider.py ================================================ # -*- coding:utf-8 -*- from scrapy.crawler import Crawler from tornado.ioloop import IOLoop import config from core.crawler import CRAWLER_RUNNER as crawler_runner from core.crawler import spider_keymap from service.proxy.proxy import proxy_srv from service.spider.spider import spider_srv from utils import log as logger class SpiderTasks: """ there are 2 kinds of spiders: seeker: to crawler new proxies checker: to check crawled proxies """ async def start(self, *args, **kwargs): heart_beat_count = kwargs['heart_beat_count'] def _checker_timers(): # every 5 sec yield heart_beat_count < 60 and heart_beat_count % 5 == 0 # every 1 min yield heart_beat_count >= 60 and heart_beat_count % 60 == 0 def _crawler_timers(): # every 10 sec yield heart_beat_count < 60 and heart_beat_count % 10 == 0 # every 10 min yield heart_beat_count >= 60 and heart_beat_count % 600 == 0 # consider using run_in_executor if any(_checker_timers()) and await self.checker_condition(): IOLoop.current().add_callback(self.execute_task, 'checker') if any(_crawler_timers()) and await self.seeker_condition(): IOLoop.current().add_callback(self.execute_task, 'seeker') async def execute_task(self, _type): key_and_spiders = await self.get_spiders_to_run(_type) return await self.start_crawling(key_and_spiders) async def checker_condition(self): """ placeholder :return: bool """ return True async def seeker_condition(self): """ whether to start crawlers :return: bool """ keys = await proxy_srv.get_all_keys() return len(keys) < config.PROXY_STORE_NUM def get_max_running_rum(self, _type): _map = getattr(config, 'MAX_RUNNING_NUM', {}) max_num = _map.get(_type, 0) if max_num is None: max_num = 999999 return max_num async def get_spiders_to_run(self, _type): """ get spiders to run according to current status :param _type: :return: [(key, spider_cls)] """ max_num = self.get_max_running_rum(_type) # all is turned off if not max_num: logger.info('No %s will run.' % _type) return [] keymap = spider_keymap[_type] items = await spider_srv.all_status(_type=_type, with_key=True) alternatives = [] # enough running spiders running_num = sum(1 for _ in items if _.get('status') == 'running') if (running_num >= max_num) or running_num >= len(keymap): logger.debug( 'There are already %s running %ss.' % (running_num, _type) ) return [] # in keymap but not in redis existed_keys = [_['key'] for _ in items] if len(items) < len(keymap): alternatives += [k for k in keymap if k not in existed_keys] # not running not_running = [_ for _ in items if not (_.get('status') == 'running')] if not_running: sorted_spiders = sorted( not_running, key=lambda _: int(_.get('last_start_time', 0)), ) alternatives += [_.get('key') for _ in sorted_spiders] if max_num: alternatives = alternatives[:max_num - running_num] return [(k, v) for k, v in keymap.items() if k in alternatives] async def start_crawling(self, key_and_spiders): """ trigger spiders :param key_and_spiders: [(key, spider_cls)] spiders to run :return: [key] """ deployed = [] for key, spider in key_and_spiders: await self.deploy_spider(key, spider) deployed.append(key) return deployed async def deploy_spider(self, key, spider): """ deploy a spider :param key: :param spider: :return: """ # register st = await spider_srv.register_status(key) logger.info('Started %s at %s. Key: %s.' % (spider, st, key)) # build a new thread # and run the crawler in it IOLoop.current().run_in_executor(None, self.run_spider, spider, st, key) def run_spider(self, spider, st, key): """ run a crawler, then unregister it * moved to another thread :st: start time :key: key in redis """ crawler = self.build_crawler(spider) logger.info('Got crawler: %s' % crawler) d = crawler_runner.crawl(crawler) # unregister d.addBoth(spider_srv.callback_unregister_status, st=st, key=key) def build_crawler(self, spider): """ do some specific settings for spider and return the wrapped crawler :param spider: spider class :return: crawler """ # TODO: specify settings settings = crawler_runner.settings # FIXME !!! # conf = {} # log_file = crawler_runner.settings.get('LOG_FILE') # if log_file: # conf['LOG_FILE'] = '%s.%s' % (log_file, spider.name) # conf['LOG_FILE'] = None # conf['LOG_FORMAT'] = ('%(levelname)1.1s [%(asctime)s]' # ' [spider-{spider}]' # ' %(message)s' # ).format(spider=spider.name) # settings = updated_crawler_settings(settings, conf) # configure_logging(settings) return Crawler(spider, settings) ================================================ FILE: src/utils/__init__.py ================================================ ================================================ FILE: src/utils/collections.py ================================================ # coding: utf-8 import random def shuffled_range(start, stop, step=1): r = range(start, stop, step) lr = list(r) random.shuffle(lr) return lr ================================================ FILE: src/utils/docker.py ================================================ # coding: utf-8 import os def check_if_inside_docker(): """ check if current env is incide a docker container :return bool """ result = False cgroup_path = '/proc/1/cgroup' if os.path.exists(cgroup_path): with open(cgroup_path) as _f: result = 'docker' in _f.read() return result ================================================ FILE: src/utils/http_client.py ================================================ # -*- coding:utf-8 -*- import json from tornado.httpclient import AsyncHTTPClient from tornado.httputil import url_concat, urlencode from core import exceptions from utils import log as logger class AsyncHttpRequests(object): """ a wrapper of async http request """ @classmethod async def get(cls, url, params=None, headers=None, decode_type='utf-8', parse_json=True, timeout=30): """ HTTP GET 请求 @param url 请求url @param params 请求的uri qurey参数 @param headers 请求的header参数 @param decode_type 返回body解码格式，默认使用utf-8解码 @param parse_json 是否解析返回body为json格式，默认为True @param timeout 请求超时时间，默认30秒 @return data 返回的http body """ if params: url = url_concat(url, params) http_client = AsyncHTTPClient() response = await http_client.fetch(url, method='GET', headers=headers, request_timeout=timeout) if response.code not in (200, 201, 202, 203, 204, 205, 206): logger.error('url:', url, 'response code:', response.code, 'response body:', response.body, caller=cls) msg = '请求url失败: {url}'.format(url=url) raise exceptions.CustomException(msg=msg) if response.body: data = response.body if decode_type: data = data.decode(decode_type) if parse_json: return json.loads(data) else: return data else: return None @classmethod async def post(cls, url, params=None, body=None, headers=None, encode_type='utf-8', decode_type='utf-8', parse_json=True, timeout=30): """ HTTP POST 请求 @param url 请求url @param params 请求的uri qurey参数 @param body 请求的body参数 @param headers 请求的header参数 @param encode_type 请求body编码格式，默认使用utf-8编码 @param decode_type 返回body解码格式，默认使用utf-8解码 @param parse_json 是否解析返回body为json格式，默认为True @param timeout 请求超时时间，默认30秒 @return data 返回的http body """ if params: url = url_concat(url, params) if body: if not encode_type: pass elif encode_type == 'utf-8': body = json.dumps(body) else: body = urlencode(body, encoding=encode_type) http_client = AsyncHTTPClient() response = await http_client.fetch(url, method='POST', body=body, headers=headers, request_timeout=timeout) if response.code not in (200, 201, 202, 203, 204, 205, 206): logger.error('url:', url, 'post data:', body, 'response code:', response.code, 'response body:', response.body, caller=cls) msg = '请求url失败: {url}'.format(url=url) raise exceptions.CustomException(msg=msg) if response.body: data = response.body if decode_type: data = data.decode(decode_type) if parse_json: return json.loads(data) else: return data else: return None ================================================ FILE: src/utils/log.py ================================================ # -*- coding:utf-8 -*- """ 日志打印 """ import logging import os import sys from tornado import log from tornado.options import options def initLogger(log_level='debug', log_path=None, logfile_name=None): """ 初始化日志输出 @param log_level 日志级别 debug info @param log_path 日志输出路径 @param logfile_name 日志文件名 """ if log_level == 'info': options.logging = 'info' else: options.logging = 'debug' logger = logging.getLogger() if logfile_name: if not os.path.isdir(log_path): os.makedirs(log_path) logfile = os.path.join(log_path, logfile_name) print('init logger ...:', logfile) handler = logging.handlers.TimedRotatingFileHandler( logfile, 'midnight') else: handler = logging.StreamHandler() fmt_str = '%(levelname)1.1s [%(asctime)s] %(message)s' fmt = log.LogFormatter(fmt=fmt_str, datefmt=None) handler.setFormatter(fmt) logger.addHandler(handler) def info(*args, **kwargs): func_name, kwargs = _log_msg_header(*args, **kwargs) logging.info(_log(func_name, *args, **kwargs)) def warn(*args, **kwargs): msg_header, kwargs = _log_msg_header(*args, **kwargs) logging.warning(_log(msg_header, *args, **kwargs)) def debug(*args, **kwargs): msg_header, kwargs = _log_msg_header(*args, **kwargs) logging.debug(_log(msg_header, *args, **kwargs)) def error(*args, **kwargs): logging.error('*' * 40) msg_header, kwargs = _log_msg_header(*args, **kwargs) logging.error(_log(msg_header, *args, **kwargs)) logging.error('*' * 40) exception = error def _log(msg_header, *args, **kwargs): _log_msg = msg_header for l in args: if type(l) == tuple: ps = str(l) else: try: ps = '%r' % l except: ps = str(l) if type(l) == str: _log_msg += ps[1:-1] + ' ' else: _log_msg += ps + ' ' if len(kwargs) > 0: _log_msg += str(kwargs) return _log_msg def _log_msg_header(*args, **kwargs): """ 打印日志的message头 @param kwargs['caller'] 调用的方法所属类对象 @param kwargs['session_id'] 调用的方法所带的session_id * NOTE: logger.xxx(... caller=self) for instance method logger.xxx(... caller=cls) for @classmethod """ cls_name = '' func_name = sys._getframe().f_back.f_back.f_code.co_name session_id = '-' try: _caller = kwargs.get('caller', None) if _caller: if not hasattr(_caller, '__name__'): _caller = _caller.__class__ cls_name = _caller.__name__ del kwargs['caller'] session_id = kwargs.get('session_id', '-') if session_id: del kwargs['session_id'] except: pass finally: msg_header = '[{cls_name}.{func_name}] [{session_id}] '.format( cls_name=cls_name, func_name=func_name, session_id=session_id, ) return msg_header, kwargs ================================================ FILE: src/utils/routes.py ================================================ # -*- coding:utf-8 -*- """ http uri 路由装饰器 """ from utils import log as logger class route(object): """ @route('/some/path') class SomeRequestHandler(RequestHandler): pass @route('/some/path', name='other') class SomeOtherRequestHandler(RequestHandler): pass my_routes = route.make_routes(['api']) """ _routes = [] def __init__(self, uri, name=None): """ 装饰器 @param uri 注册的uri名字，支持uri正则表达式 @param name 注册的uri别名 """ self.uri = uri if not name: name = '-'.join(uri.split('/')) self.name = name def __call__(self, _handler): """ gets called when we class decorate """ for item in self._routes: if item.get('uri') == self.uri: logger.error('uri aleady exists! uri:', self.uri, 'name:', self.name, 'handler:', _handler, caller=self) if item.get('name') == self.name: logger.warn('name aleady exists! uri:', self.uri, 'name:', self.name, 'handler:', _handler, caller=self) self._routes.append({'uri': self.uri, 'name': self.name, 'handler': _handler}) return _handler @classmethod def make_routes(cls, dirs): """ 注册并返回所有的handler @param dirs list，需要注册uri路由的处理方法路径 """ for dir in dirs: s = 'import %s' % dir exec(s) routes = [] for handler_dic in cls._routes: logger.info('register uri:', handler_dic['uri'], 'handler:', handler_dic.get('handler'), caller=cls) routes.append( (handler_dic.get('uri'), handler_dic.get('handler')) ) return routes ================================================ FILE: src/utils/send_email.py ================================================ # -*- coding:utf-8 -*- """ 发送邮件 """ import email import smtplib from email.mime.text import MIMEText from email.mime.multipart import MIMEMultipart class MailSender(object): def __init__(self, host, username, password, to_emails, subject, content): """ 初始化 @param host 邮件服务端host @param username 用户名 @param password 密码 @param to_emails 发送到邮箱列表 @param title 标题 @param content 内容 """ self.host = host self.username = username self.password = password self.to_emails = to_emails self.subject = subject self.content = content def send_mail(self): """ 发送邮件 """ msgRoot = MIMEMultipart('related') msgRoot['Subject'] = self.subject msgRoot['From'] = self.username msgRoot['To'] = ",".join(self.to_emails) msgRoot['Date'] = email.utils.formatdate() msgRoot.preamble = 'This is a multi-part message in MIME format.' msgAlternative = MIMEMultipart('alternative') msgRoot.attach(msgAlternative) msgText = MIMEText(self.content, 'plain', 'GB2312') msgAlternative.attach(msgText) # msgText = MIMEText(self.content, 'html', 'GB2312') # msgAlternative.attach(msgText) # sending mail svr = smtplib.SMTP(self.host, 25) # svr = smtplib.SMTP(self.host) svr.set_debuglevel(0) svr.ehlo() svr.starttls() svr.login(self.username, self.password) svr.sendmail(self.username, self.to_emails, msgRoot.as_string()) svr.quit() def test(): host = 'smtp.xxx.com' username = 'test@xxxx.com' password = '123456' to_emails = ['valesail7@gmail.com'] subject = 'Test Send Email' content = "Just a test." sender = MailSender(host, username, password, to_emails, subject, content) sender.send_mail() if __name__ == "__main__": test() ================================================ FILE: src/utils/time_ext.py ================================================ # -*- coding: utf-8 -*- import datetime import re import time ZERO_TIME_DELTA = datetime.timedelta(0) LOCAL_TIME_DELTA = datetime.timedelta(hours=8) # 本地时区偏差 class UTC(datetime.tzinfo): def utcoffset(self, dt): return ZERO_TIME_DELTA def dst(self, dt): return ZERO_TIME_DELTA class LocalTimeZone(datetime.tzinfo): def utcoffset(self, dt): return LOCAL_TIME_DELTA def dst(self, dt): return ZERO_TIME_DELTA def tzname(self, dt): return '+08:00' # singleton UTC = UTC() LocalTimeZone = LocalTimeZone() date_re = re.compile( r'(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})$' ) datetime_re = re.compile( r'(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})' r'[T ](?P\d{1,2}):(?P\d{1,2})' r'(?::(?P\d{1,2})(?:\.(?P\d{1,6})\d{0,6})?)?' r'(?PZ|[+-]\d{2}(?::?\d{2})?)?$' ) def parse_date(value): """Parses a string(ISO_8601) and return a datetime.date. Raises ValueError if the input is well formatted but not a valid date. Returns None if the input isn't well formatted. """ match = date_re.match(value) if match: kw = {k: int(v) for k, v in match.groupdict().items()} return datetime.date(**kw) def parse_datetime(value): """Parses a string(ISO_8601) and return a datetime.datetime base UTC, or parse datetime.datetime base other timezone and return a datetime.datetime base UTC timezone """ if isinstance(value, datetime.datetime): if not value.tzinfo: value = value.replace(tzinfo=LocalTimeZone) return value.astimezone(UTC) match = datetime_re.match(value) if match: kw = match.groupdict() if kw['microsecond']: kw['microsecond'] = kw['microsecond'].ljust(6, '0') tzinfo = kw.pop('tzinfo') tz = UTC offset = 0 if tzinfo == 'Z': offset = 0 elif tzinfo is not None: offset_mins = int(tzinfo[-2:]) if len(tzinfo) > 3 else 0 offset = 60 * int(tzinfo[1:3]) + offset_mins if tzinfo[0] == '-': offset = -offset else: tz = LocalTimeZone kw = {k: int(v) for k, v in kw.items() if v is not None} kw['tzinfo'] = tz dt = datetime.datetime(**kw) dt += datetime.timedelta(minutes=offset) return dt.astimezone(UTC) def convert_zone(dt: datetime.datetime, tz_to=UTC, tz_default=LocalTimeZone): """ @param dt: @param tz_to: 转换后的目标时区 @param tz_default: dt无时区信息时的默认时区 """ if not dt.tzinfo: dt = dt.replace(tzinfo=tz_default) return dt.astimezone(tz_to) def get_utc_time(dt: datetime.datetime = None, tz_default=LocalTimeZone): """ @param dt 为None时，返回当前时间 @param tz_default dt无时区信息时的默认时区 """ if dt is None: dt = datetime.datetime.now() return convert_zone(dt, tz_default=tz_default) def get_time_str(dt: datetime.datetime = None, tz_default=LocalTimeZone): """ @param dt 为None时，返回当前时间 @param tz_default dt无时区信息时的默认时区 """ if not dt: dt = datetime.datetime.now() dt = convert_zone(dt, tz_default=tz_default) time_str = dt.isoformat().split('+')[0] return time_str + 'Z' def get_date_str(dt: datetime.date = None): """ @param dt 为None时，返回当前日期 """ if not dt: dt = datetime.date.today() return dt.isoformat() def get_cur_timestamp(): """ 获取当前时间戳 """ ts = int(time.time()) return ts def get_cur_datetime_m(): """ 获取当前日期时间字符串，包含 年 + 月 + 日 + 时 + 分 + 秒 + 微妙 """ today = datetime.datetime.today() str_m = today.strftime('%Y%m%d%H%M%S%f') return str_m def get_datetime(): """ 获取日期时间字符串，包含 年 + 月 + 日 + 时 + 分 + 秒 """ today = datetime.datetime.today() str_dt = today.strftime('%Y%m%d%H%M%S') return str_dt def get_date(fmt='%Y%m%d', delta_day=0): """ 获取日期字符串，包含 年 + 月 + 日 @param fmt 返回的日期格式 """ day = datetime.datetime.today() if delta_day: day += datetime.timedelta(days=delta_day) str_d = day.strftime(fmt) return str_d def date_str_to_dt(date_str=None, fmt='%Y%m%d', delta_day=0): """ 日期字符串转换到datetime对象 @param date_str 日期字符串 @param fmt 日期字符串格式 @param delta_day 相对天数，<0减相对天数，>0加相对天数 """ if not date_str: dt = datetime.datetime.today() else: dt = datetime.datetime.strptime(date_str, fmt) if delta_day: dt += datetime.timedelta(days=delta_day) return dt def dt_to_date_str(dt=None, fmt='%Y%m%d', delta_day=0): """ datetime对象转换到日期字符串 @param dt datetime对象 @param fmt 返回的日期字符串格式 @param delta_day 相对天数，<0减相对天数，>0加相对天数 """ if not dt: dt = datetime.datetime.today() if delta_day: dt += datetime.timedelta(days=delta_day) str_d = dt.strftime(fmt) return str_d def ts_to_datetime_str(ts): """ 将时间戳转换为日期时间格式，年-月-日 时:分:秒 @param ts 时间戳 """ if not ts: return '00-00-00 00:00:00' dt = datetime.datetime.fromtimestamp(int(ts)) return dt.strftime('%Y-%m-%d %H:%M:%S') def datetime_str_to_ts(dt_str, fmt='%Y-%m-%d %H:%M:%S'): """ 将日期时间格式字符串转换成时间戳 @param dt_str 日期时间字符串 @param fmt 日期时间字符串格式 """ ts = int(time.mktime(datetime.datetime.strptime(dt_str, fmt).timetuple())) return ts def current_timestamp(_int=True): res = time.time() if _int: res = int(res) return res ================================================ FILE: src/utils/tools.py ================================================ # -*- coding:utf-8 -*- import uuid import yaml def get_uuid1(): """ make a UUID based on the host ID and current time """ s = uuid.uuid1() return str(s) def get_uuid3(str_in): """ make a UUID using an MD5 hash of a namespace UUID and a name @param str_in 输入字符串 """ s = uuid.uuid3(uuid.NAMESPACE_DNS, str_in) return str(s) def get_uuid4(): """ make a random UUID """ s = uuid.uuid4() return str(s) def get_uuid5(str_in): """ make a UUID using a SHA-1 hash of a namespace UUID and a name @param str_in 输入字符串 """ s = uuid.uuid5(uuid.NAMESPACE_DNS, str_in) return str(s) def parse_yaml(path): with open(path) as f: return yaml.load(f) def merge_configure(old, new): """ this is just for configure updating :old: to be updated :new: """ for key, value in new.items(): if key in old: old_value = old[key] # if type(old_value) != type(value): # raise AssertionError("New value's type must" # "be the same as the old's") if isinstance(value, dict): old[key].update(value) elif isinstance(value, list): old[key] = old_value + value elif isinstance(value, tuple): old[key] = tuple(list(old_value) + list(value)) else: old[key] = value else: old[key] = value # from copy import deepcopy # def recursive_updated(old, new): # """ # for general scene # :old: to be updated # :new: # :return: a new dict # """ # result = deepcopy(old) # for key, value in new.items(): # if key in result: # old_value = result[key] # if type(old_value) != type(value): # raise TypeError( # "[key: %s] old: %r new: %r" % (key, old_value, value) # ) # if isinstance(value, dict): # recursive_update(old_value, value) # # elif isinstance(value, list): # # result[key] = old_value + value # # elif isinstance(value, set): # # result[key] = old_value | value # else: # result[key] = value # else: # result[key] = value # return result def subdict(base, keys): return {k: v for k, v in base.items() if k in keys} def str_rot13(txt): rot13 = str.maketrans( "ABCDEFGHIJKLMabcdefghijklmNOPQRSTUVWXYZnopqrstuvwxyz", "NOPQRSTUVWXYZnopqrstuvwxyzABCDEFGHIJKLMabcdefghijklm" ) return str.translate(txt, rot13) ================================================ FILE: src/utils/validators.py ================================================ # -*- coding:utf-8 -*- import json from core import exceptions from utils import time_ext def _field(data, field, required): if field: data = data or {} if not isinstance(data, dict): raise exceptions.SystemError() if required and field not in data: raise exceptions.ValidationError('{field}必填'.format(field=field)) return data.get(field) else: return data def bool_field(data, field=None, required=True): """ bool类型检查 @param data 如果field不为None，那么field从data里取值，否则判断data是否为bool类型 @param field 如果不为None，那么需要从data里提取值 @param required 是否data里必须存在field字段，如果字段不存在且required为False，返回None """ field_data = _field(data, field, required) if str(field_data).lower() == 'true': return True if str(field_data).lower() == 'false': return False if not required: return None raise exceptions.ValidationError('{field}是bool类型'.format(field=field)) def int_field(data, field=None, required=True): """ int类型检查 @param data 如果field不为None，那么field从data里取值，否则判断data是否为int类型 @param field 如果不为None，那么需要从data里提取值 @param required 是否data里必须存在field字段，如果字段不存在且required为False，返回None """ field_data = _field(data, field, required) if not field_data and not required: return None try: return int(field_data) except: raise exceptions.ValidationError('{field}是int类型'.format(field=field)) def float_field(data, field=None, required=True): """ float类型检查 @param data 如果field不为None，那么field从data里取值，否则判断data是否为float类型 @param field 如果不为None，那么需要从data里提取值 @param required 是否data里必须存在field字段，如果字段不存在且required为False，返回None """ field_data = _field(data, field, required) if not field_data and not required: return None try: return float(field_data) except: raise exceptions.ValidationError('{field}是float类型'.format(field=field)) def string_field(data, field=None, required=True): """ str类型检查 @param data 如果field不为None，那么field从data里取值，否则判断data是否为str类型 @param field 如果不为None，那么需要从data里提取值 @param required 是否data里必须存在field字段，如果字段不存在且required为False，返回None """ field_data = _field(data, field, required) if not field_data and not required: return None return str(field_data) def list_field(data, field=None, required=True): """ list类型检查 @param data 如果field不为None，那么field从data里取值，否则判断data是否为list类型 @param field 如果不为None，那么需要从data里提取值 @param required 是否data里必须存在field字段，如果字段不存在且required为False，返回None """ field_data = _field(data, field, required) if not field_data and not required: return None if isinstance(field_data, str): try: field_data = json.loads(field_data) except: raise exceptions.ValidationError( '{field}是list类型'.format(field=field)) if not isinstance(field_data, (list, set, tuple)): raise exceptions.ValidationError('{field}是list类型'.format(field=field)) return list(field_data) def dict_field(data, field=None, required=True): """ dict类型检查 @param data 如果field不为None，那么field从data里取值，否则判断data是否为dict类型 @param field 如果不为None，那么需要从data里提取值 @param required 是否data里必须存在field字段，如果字段不存在且required为False，返回None """ field_data = _field(data, field, required) if not field_data and not required: return None if isinstance(field_data, str): try: field_data = json.loads(field_data) except: raise exceptions.ValidationError( '{field}是dict类型'.format(field=field)) if not isinstance(field_data, dict): raise exceptions.ValidationError('{field}是dict类型'.format(field=field)) return field_data def datetime_field(data, field=None, required=True): field_data = _field(data, field, required) try: return time_ext.parse_datetime(field_data) if field_data is not None else None except: raise exceptions.ValidationError('%s是ISO_8601格式的时间字符串' % field) def date_field(data, field=None, required=True): field_data = _field(data, field, required) try: return time_ext.parse_date(field_data) if field_data is not None else None except: raise exceptions.ValidationError('%s是ISO_8601格式的日期字符串' % field)