Repository: hunterhug/marmot Branch: master Commit: 83535771ee8a Files: 38 Total size: 150.0 KB Directory structure: gitextract__m_tjoi8/ ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── README_ZH.md ├── example/ │ ├── lesson/ │ │ ├── lesson1/ │ │ │ └── lesson1.go │ │ ├── lesson2/ │ │ │ └── lesson2.go │ │ ├── lesson3/ │ │ │ └── lesson3.go │ │ ├── lesson4/ │ │ │ └── lesson4.go │ │ └── lesson5/ │ │ └── lesson5.go │ └── practice/ │ ├── helloworld/ │ │ ├── README.md │ │ └── main.go │ ├── pictures/ │ │ ├── README.md │ │ └── main.go │ └── upload/ │ ├── README.md │ ├── client/ │ │ └── main.go │ └── main.go ├── expert/ │ ├── query.go │ └── query_test.go ├── go.mod ├── go.sum ├── miner/ │ ├── client.go │ ├── config/ │ │ └── ua.txt │ ├── config.go │ ├── cons.go │ ├── default.go │ ├── doc.go │ ├── log.go │ ├── multworker.go │ ├── ua.go │ ├── util.go │ └── worker.go └── util/ ├── crypto.go ├── file.go ├── io.go ├── json.go ├── time.go └── util.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .gitignore ================================================ .idea/ /vendor /picture .DS_Store doc.go.xxxxxx ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: Makefile ================================================ .PHONY: clean fmt default: fmt clean: rm -rf bin/* fmt: go fmt ./ ================================================ FILE: README.md ================================================ # Project: Marmot | HTTP Download [![GitHub forks](https://img.shields.io/github/forks/hunterhug/marmot.svg?style=social&label=Forks)](https://github.com/hunterhug/marmot/network) [![GitHub stars](https://img.shields.io/github/stars/hunterhug/marmot.svg?style=social&label=Stars)](https://github.com/hunterhug/marmot/stargazers) [![GitHub last commit](https://img.shields.io/github/last-commit/hunterhug/marmot.svg)](https://github.com/hunterhug/marmot) [![Go Report Card](https://goreportcard.com/badge/github.com/hunterhug/marmot)](https://goreportcard.com/report/github.com/hunterhug/marmot) [![GitHub issues](https://img.shields.io/github/issues/hunterhug/marmot.svg)](https://github.com/hunterhug/marmot/issues) [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [中文介绍](/README_ZH.md) HTTP Download Helper, Supports Many Features such as Cookie Persistence, HTTP(S) and SOCKS5 Proxy.... ![Marmot](logo.png) ## 1. Introduction World-Wide-Web robot, also known as spiders and crawlers. The principle is to falsify network data by constructing appointed HTTP protocol data packet, then request resource to the specified host, goal is to access the data returned. There are a large number of web information, human's hand movement such as `copy-paste data` from web page is `time-consuming` and `laborious`, thus inspired the data acquisition industry. Batch access to public network data does not break the law, but because there is no difference, no control, very violent means will lead to other services is not stable, therefore, most of the resources provider will filtering some data packets(falsify), in this context, batch small data acquisition has become a problem. Integrated with various requirements, such as various API development, automated software testing(all this have similar technical principle). So this project come into the world(very simple). The `Marmot` is very easy to understand, just like Python's library `requests`(Not yet Smile~ --| ). By enhancing native Golang HTTP library, help you deal with some trivial logic (such as collecting information, checking parameters), and add some fault-tolerant mechanisms (such as add lock, close time flow, ensure the high concurrent run without accident). It provides a human friendly API interface, you can reuse it often. Very convenient to support `Cookie Persistence`, `Crawler Proxy Settings`, as well as others general settings, such as `HTTP request header settings, timeout/pause settings, data upload/post settings`. It supports all the HTTP methods `POST/PUT/GET/DELETE/...` and has built-in spider pool and browser UA pool, easy to develop UA+Cookie persistence distributed spider. The library is simple and practical, just a few lines of code to replace the previous `Spaghetti code`, has been applied in some large projects. The main uses: `WeChat development`/ `API docking` / `Automated test` / `Rush Ticket Scripting` / `Vote Plug-in` / `Data Crawling` Now We support Default Worker, You can easy use: `lesson1.go` ```go package main import ( "fmt" "github.com/hunterhug/marmot/miner" ) func main() { miner.SetLogLevel(miner.DEBUG) // Use Default Worker, You can Also New One: //worker, _ := miner.New(nil) //worker = miner.NewWorkerWithNoProxy() //worker = miner.NewAPI() //worker, _ = miner.NewWorkerWithProxy("socks5://127.0.0.1:1080") worker := miner.Clone() _, err := worker.SetUrl("https://www.bing.com").Go() if err != nil { fmt.Println(err.Error()) } else { fmt.Println(worker.ToString()) } } ``` See the [example](example) dir. such as lesson or practice. ## 2. How To Use You can get it by: ``` go get -v github.com/hunterhug/marmot/miner ``` ### 2.1 The First Step There are four kinds of worker: 1. `worker, err := miner.NewWorker("http://xx:xxxx@103.118.111.26:808") ` // proxy worker, format: `protocol://user(optional):password(optional)@ip:port`, alias to`New()`, support http(s), socks5 2. `worker, err := miner.NewWorker(nil)` // normal worker, default keep Cookie, alias to `New()` 3. `worker := miner.NewAPI()` // API worker, will not keep Cookie 4. `worker, err := miner.NewWorkerByClient(&http.Client{})` // You can also pass a `http.Client` if you want if you want to use worker twice, you can call `Clone()` method to clone a new worker, it can isolate the request and response of http, otherwise, you should deal concurrent program carefully. ### 2.2 The Second Step Camouflage our worker: 1. `worker.SetUrl("https://www.bing.com")` // required: set url you want to 2. `worker.SetMethod(miner.GET)` // optional: set http method `POST/GET/PUT/POSTJSON` and so on 3. `worker.SetWaitTime(2)` // optional: set timeout of http request 4. `worker.SetUa(miner.RandomUa())` // optional: set http browser user agent, you can see miner/config/ua.txt 5. `worker.SetRefer("https://www.bing.com")` // optional: set http request Refer 6. `worker.SetHeaderParam("diyheader", "diy")` // optional: set http diy header 7. `worker.SetBData([]byte("file data"))` // optional: set binary data for post or put 8. `worker.SetFormParam("username","hunterhug")` // optional: set form data for post or put 9. `worker.SetCookie("xx=dddd")` // optional: you can set a init cookie, some website you can login and F12 copy the cookie 10. `worker.SetCookieByFile("/root/cookie.txt")` // optional: set cookie which store in a file ### 2.3 The Third Step Run our worker: 1. `body, err := worker.Go()` // if you use SetMethod(), auto use following ways, otherwise use Get() 2. `body, err := worker.Get()` // default 3. `body, err := worker.Post()` // post form request, data fill by SetFormParam() 4. `body, err := worker.PostJSON()` // post JSON request, data fill by SetBData() 5. `body, err := worker.PostXML()` // post XML request, data fill by SetBData() 6. `body, err := worker.PostFILE()` // upload file, data fill by SetBData(), and should set SetFileInfo(fileName, fileFormName string) 7. `body, err := worker.Delete()` // you know! 8. `body, err := worker.Put()` // ones http method... 9. `body, err := worker.PutJSON()` // put JSON request 10. `body, err := worker.PutXML()` 11. `body, err := worker.PutFILE()` 12. `body, err := worker.OtherGo("OPTIONS", "application/x-www-form-urlencoded")` // Other http method, Such as OPTIONS etc., can not sent binary. 13. `body, err := worker.OtherGoBinary("OPTIONS", "application/x-www-form-urlencoded")` // Other http method, Such as OPTIONS etc., just sent binary. 14. `body, err := worker.GoByMethod("POST")` // you can override SetMethod() By this, equal SetMethod() then Go() ### 2.4 The Fourth Step Deal the return data, all data will be return as binary, You can immediately store it into a new variable: 1. `fmt.Println(string(html))` // type change directly 2. `fmt.Println(worker.ToString())` // use spider method, after http response, data will keep in the field `Raw`, just use ToString 3. `fmt.Println(worker.JsonToString())` // some json data will include chinese and other multibyte character, such as `我爱你,我的小绵羊`,`사랑해` Attention: After every request for an url, the next request you should cover your http request header, otherwise http header you set still exist, if just want clear post data, use `Clear()`, and want clear HTTP header too please use `ClearAll()`, but I suggest use `Clone()` to avoid this. ### 2.5 Other Hook: 1. `SetBeforeAction(fc func(context.Context, *Worker))` 2. `SetAfterAction(fc func(context.Context, *Worker))` # License ``` Copyright [2016-2022] [github.com/hunterhug] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ``` ================================================ FILE: README_ZH.md ================================================ # 项目代号：土拨鼠 [![GitHub forks](https://img.shields.io/github/forks/hunterhug/marmot.svg?style=social&label=Forks)](https://github.com/hunterhug/marmot/network) [![GitHub stars](https://img.shields.io/github/stars/hunterhug/marmot.svg?style=social&label=Stars)](https://github.com/hunterhug/marmot/stargazers) [![GitHub last commit](https://img.shields.io/github/last-commit/hunterhug/marmot.svg)](https://github.com/hunterhug/marmot) [![Go Report Card](https://goreportcard.com/badge/github.com/hunterhug/marmot)](https://goreportcard.com/report/github.com/hunterhug/marmot) [![GitHub issues](https://img.shields.io/github/issues/hunterhug/marmot.svg)](https://github.com/hunterhug/marmot/issues) [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) [English README](/README.md) ![Marmot](logo.png) >万维网网络机器人，又称蜘蛛，爬虫，原理主要是通过构造符合HTTP协议的网络数据包，向指定主机请求资源，获取返回的数据.万维网有大量的公开信息，人力采集数据费时费力，故激发了爬虫的产业化. 批量获取公开网络数据并不违反，但由于无差别性，无节制，十分暴力的手段会导致对方服务的不稳定，因此，大部分资源提供商对数据包进行了某些过滤，在此背景下，小批量数据获取成为了难题. 综合各种需求，如各种API对接，自动化测试等原理均一样，故开发了此爬虫库. >土拨鼠项目是一个人类友好姿势的代码库，开发采用面向对象的方式，易于理解.通过对Golang原生HTTP库的封装，帮用户处理了一些琐碎逻辑(如收集信息，检测参数)，并加入了一些容错机制(如加锁，及时关闭流)，保证了爬虫高并发的安全.此库提供了大量优美的API接口，复用率高，十分方便地支持Cookie接力，爬虫代理设置，以及一般的HTTP请求设置如头部设置，超时，暂停设置，数据设置等，支持全部的HTTP方法如POST/PUT/GET/DELETE等，内置爬虫池和浏览器UA池，易于开发多UA多Cookie分布式爬虫. >该库简单实用，短短几行代码即可取代以往杂乱无章的面包条代码片段，已经应用在某些大项目中。 >该库主要用途：微信开发/API对接/自动化测试/抢票脚本/网站监控/点赞插件/数据爬取 ## 一. 下载 自己封装的 `Golang` 爬虫下载库，支持各种代理模式和伪装功能，你只需通过该方式获取库： ``` go get -v github.com/hunterhug/marmot/miner ``` 代码结构： ``` ├── miner 核心库（HTTP请求封装） ├── expert 解析库（HTML解析封装） ├── example Example示例库 ├── lession 示例 ├── practice 练习 ``` 以下是几个实战例子： 1. [多线程批量抓图片](/example/practice/pictures/README.md)。 2. [模拟上传文件](/example/practice/upload/README.md)。 ## 二. 使用 此库可模拟上传文件，模拟表单提交，模拟各种各样的操作。 `lesson1.go` ```go package main import ( "fmt" "github.com/hunterhug/marmot/miner" ) func main() { miner.SetLogLevel(miner.DEBUG) // 使用默认的全局矿工，当然你可以自己建一个 //worker, _ := miner.New(nil) //worker = miner.NewWorkerWithNoProxy() //worker = miner.NewAPI() //worker, _ = miner.NewWorkerWithProxy("socks5://127.0.0.1:1080") worker := miner.Clone() _, err := worker.SetUrl("https://www.bing.com").Go() if err != nil { fmt.Println(err.Error()) } else { fmt.Println(worker.ToString()) } } ``` 官方部分示例已经合进本库，参见 [example](example) 文件夹。 ### 第一步 矿工有四种类型： 1. `miner.NewWorker("http://user:password@103.118.111.26:808") ` // 代理矿工，默认自动化Cookie接力 格式:`协议://代理帐号(可选):代理密码(可选)@ip:port`，支持http(s)，socks5，别名函数 `New()` 2. `miner.NewWorker(nil)` // 正常矿工，默认自动化Cookie接力，别名函数`New()` 3. `miner.NewAPI()` // API矿工，默认Cookie不接力，主要用来对接服务端 API 4. `miner.NewWorkerByClient(&http.Client{})` // 可自定义客户端 如果你想使用一个矿工多次，使用前请先调用 `Clone()` 方法，它会创建新的矿工来进行 HTTP 请求的数据隔离，避免你并发调用多次导致的混乱。 ### 第二步 模拟矿工设置头部: 1. `worker.SetUrl("https://www.bing.com")` // 设置HTTP请求要抓取的网址，**必须** 2. `worker.SetMethod(miner.GET)` // 设置HTTP请求的方法:`POST/GET/PUT/POSTJSON`等 3. `worker.SetWaitTime(2)` // 设置HTTP请求超时时间 4. `worker.SetUa(miner.RandomUa())` // 设置HTTP请求浏览器标志，本项目提供445个浏览器标志，可选设置 5. `worker.SetRefer("http://www.baidu.com")` // 设置HTTP请求Refer头 6. `worker.SetHeaderParam("diyheader", "diy")` // 设置HTTP请求自定义头部 7. `worker.SetBData([]byte("file data"))` // HTTP请求需要上传数据 8. `worker.SetFormParam("username","hunterhug")` // HTTP请求需要提交表单 9. `worker.SetCookie("xx=dddd")` // HTTP请求设置cookie，某些网站需要登录后F12复制cookie ### 第三步 矿工启动方式有： 1. `body, err := worker.Go()` // 如果设置SetMethod()，会调用下方对应的方法，否则使用Get() 2. `body, err := worker.Get()` // 默认 3. `body, err := worker.Post()` // POST表单请求，数据在SetFormParam() 4. `body, err := worker.PostJSON()` // 提交JSON请求，数据在SetBData() 5. `body, err := worker.PostXML()` // 提交XML请求，数据在SetBData() 6. `body, err := worker.PostFILE()` // 提交文件上传请求，文件二进制数据通过SetBData()设置，然后设置SetFileInfo(fileName，fileFormName string) 表明文件名和表单field Name 7. `body, err := worker.Delete()` 8. `body, err := worker.Put()` 9. `body, err := worker.PutJSON()` 10. `body, err := worker.PutXML()` 11. `body, err := worker.PutFILE()` 12. `body, err := worker.OtherGo("OPTIONS", "application/x-www-form-urlencoded")` // 其他自定义的HTTP方法，不能模拟二进制 13. `body, err := worker.OtherGoBinary("OPTIONS", "application/x-www-form-urlencoded")` // 其他自定义的HTTP方法，模拟二进制 14. `body, err := worker.GoByMethod("POST")` // 等同于 SetMethod() 然后 Go() ### 第四步 每次下载会返回 `[]byte`，请自行解析，调试时可以使用以下方法： 1. `fmt.Println(string(html))` // 每次抓取后会返回二进制数据，直接类型转化 2. `fmt.Println(worker.ToString())` // http响应后二进制数据也会保存在矿工对象的Raw字段中，使用ToString可取出来 3. `fmt.Println(worker.JsonToString())` // 如果获取到的是JSON数据，请采用此方法转义回来，不然字符串会乱码 注意：每次下载后，需要使用以下方法将表单等数据重置（我推荐每次使用 `Clone()` 来避免这种做法，也就是克隆一个新的矿工来进行下一次操作）： 1. `Clear()` // 清除表单和二进制数据 2. `ClearAll()` // 清除表单和二进制数据，还清除全部HTTP头部 3. `ClearCookie()` // 可清除Cookie ### 其他 勾子: 1. `SetBeforeAction(fc func(context.Context, *Worker))` 爬虫动作前可AOP注入。 2. `SetAfterAction(fc func(context.Context, *Worker))` 爬虫动作完成后。 # License ``` Copyright [2016-2022] [github.com/hunterhug] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ``` ================================================ FILE: example/lesson/lesson1/lesson1.go ================================================ package main import ( "fmt" "github.com/hunterhug/marmot/miner" ) /* Most Simple: Use Default Worker! */ func main() { miner.SetLogLevel(miner.DEBUG) // Use Default Worker, You can Also New One: //worker, _ := miner.New(nil) //worker = miner.NewWorkerWithNoProxy() //worker = miner.NewAPI() //worker, _ = miner.NewWorkerWithProxy("socks5://127.0.0.1:1080") worker := miner.Clone() _, err := worker.SetUrl("https://www.bing.com").Go() if err != nil { fmt.Println(err.Error()) } else { fmt.Println(worker.ToString()) } } ================================================ FILE: example/lesson/lesson2/lesson2.go ================================================ package main import ( "fmt" "github.com/hunterhug/marmot/miner" ) /* New A Worker! */ func main() { // 1. New a worker worker, _ := miner.New(nil) // 2. Set a URL And Fetch html, err := worker.SetUrl("https://www.bing.com").SetUa(miner.RandomUa()).SetMethod(miner.GET).Go() if err != nil { fmt.Println(err.Error()) } // 3.Print content equal to fmt.Println(worker.ToString()) fmt.Println(string(html)) } ================================================ FILE: example/lesson/lesson3/lesson3.go ================================================ package main /* More detail Example */ import ( // 1:import package "github.com/hunterhug/marmot/miner" "github.com/hunterhug/marmot/util" ) func init() { // 2:Optional global setting miner.SetLogLevel(miner.DEBUG) // optional, set log to debug miner.SetGlobalTimeout(3) // optional, http request timeout time } func main() { log := miner.Log() // optional, miner log you can choose to use // 3: Must new a Worker object, three ways //worker, err := miner.NewWorker("http://xx:xxxx@103.118.111.26:808") // proxy format: protocol://user(optional):password(optional)@ip:port //worker, err := miner.NewWorker(nil) // normal worker, default keep Cookie //worker := miner.NewAPI() // API worker, not keep Cookie worker, err := miner.New(nil) // NewWorker alias if err != nil { panic(err) } // 4: Set the request Method/URL and some others, can chain set, only SetUrl is required. // SetUrl: required, the Url // SetMethod: optional, HTTP method: POST/GET/..., default GET // SetWaitTime: optional, HTTP request wait/pause time worker.SetUrl("https://www.bing.com").SetMethod(miner.GET).SetWaitTime(2) worker.SetUa(miner.RandomUa()) // optional, browser user agent: IE/Firefox... worker.SetRefer("https://www.bing.com") // optional, url refer worker.SetHeaderParam("diy_header", "diy") // optional, some other diy http header //worker.SetBData([]byte("file data")) // optional, if you want post JSON data or upload file //worker.SetFormParam("username","root") // optional: if you want post form //worker.SetFormParam("password","123") // 5: Start Run //worker.Get() // default GET //worker.Post() // POST form request data, data can fill by SetFormParam() //worker.PostJSON() // POST JSON dara, use SetBData() //worker.PostXML() // POST XML, use SetBData() //worker.PostFILE() // POST to Upload File, data in SetBData() too //worker.OtherGo("OPTIONS", "application/x-www-form-urlencoded") // Other http method, Such as OPTIONS etcd body, err := worker.Go() // if you use SetMethod(), otherwise equal to Get() if err != nil { log.Error(err.Error()) } else { log.Infof("%s", string(body)) // Print return data } log.Debugf("%#v", worker.GetCookies) // if you not set log as debug, it will not appear // You must Clear it! If you want to POST Data by SetFormParam()/SetBData() again // After get the return data by post data, you can clear the data you fill // I suggest use Clone() worker.Clear() //worker.ClearAll() // you can also want to clear all, include http header you set // Worker pool for concurrent, every Worker Object is serial as the browser. if you want collateral execution, use this. miner.Pool.Set("myfirstworker", worker) if w, ok := miner.Pool.Get("myfirstworker"); ok { go func() { data, _ := w.Clone().SetUrl("https://www.bing.com").Get() log.Info(string(data)) }() util.Sleep(10) } } ================================================ FILE: example/lesson/lesson4/lesson4.go ================================================ package main import ( "context" "fmt" "github.com/hunterhug/marmot/util" "strings" "github.com/hunterhug/marmot/expert" "github.com/hunterhug/marmot/miner" ) func main() { // We can debug, to see whether SetBeforeAction make sense miner.SetLogLevel(miner.DEBUG) // The url we want url := "https://www.bing.com" // IAM we can NewAPI worker := miner.NewAPI() // We can aop by context ctx, cancel := context.WithCancel(context.Background()) //ctx := context.TODO() worker.SetContext(ctx) // we cancel it after 5 second go func() { fmt.Println("I stop and sleep 5") util.Sleep(5) fmt.Println("I wake up after sleep 5") cancel() }() // Before we make some change, And every GET Or POST it will action worker.SetBeforeAction(func(ctx context.Context, this *miner.Worker) { fmt.Println("Before Action, I will add a HTTP header, then sleep wait cancel") this.SetHeaderParam("Marmot", "v2") this.SetHeaderParam("DUDUDUU", "DUDU") select { case <-ctx.Done(): // block in here util cancel() //fmt.Println(ctx.Err()) fmt.Println("after sleep, i do action.") } }) worker.SetAfterAction(func(ctx context.Context, this *miner.Worker) { fmt.Println("After Action, I just print this sentence") }) // Let's Go body, err := worker.SetUrl(url).GoByMethod(miner.GET) if err != nil { fmt.Println(err.Error()) } else { // Parse We want fmt.Printf("Output:\n %s\n", MyParse(body)) } } // MyParse Parse HTML page func MyParse(data []byte) string { doc, err := expert.QueryBytes(data) if err != nil { fmt.Println(err.Error()) } return strings.TrimSpace(doc.Find("title").Text()) } ================================================ FILE: example/lesson/lesson5/lesson5.go ================================================ package main /* Proxy Worker! You first should own a remote machine, Then in your local tap: `ssh -ND 1080 ubuntu@remoteIp` It will generate socks5 proxy client in your local, which port is 1080 */ import ( "fmt" "os" "github.com/hunterhug/marmot/expert" "github.com/hunterhug/marmot/miner" ) func init() { miner.SetLogLevel(miner.DEBUG) } func main() { // You can use a lot of proxy ip such "https/http/socks5" proxyIp := "socks5://127.0.0.1:1080" url := "https://www.google.com" worker, err := miner.New(proxyIp) if err != nil { fmt.Println(err.Error()) os.Exit(1) } body, err := worker.SetUa(miner.RandomUa()).SetUrl(url).SetMethod(miner.GET).Go() if err != nil { fmt.Println(err.Error()) } else { fmt.Println(parse(body)) } } // Parse HTML page func parse(data []byte) string { doc, err := expert.QueryBytes(data) if err != nil { fmt.Println(err.Error()) return "" } return doc.Find("title").Text() } ================================================ FILE: example/practice/helloworld/README.md ================================================ # 简单示例 1. simple.go 最简单的一个 ================================================ FILE: example/practice/helloworld/main.go ================================================ package main import ( // 第一步：引入库 "github.com/hunterhug/marmot/miner" "github.com/hunterhug/marmot/util" ) func init() { // 第二步：可选设置全局 miner.SetLogLevel(miner.DEBUG) // 设置全局矿工日志,可不设置,设置debug可打印出http请求轨迹 miner.SetGlobalTimeout(3) // 矿工超时时间,可不设置 } func main() { log := miner.Log() // 矿工为你提供的日志工具,可不用 // 第三步： 必须新建一个矿工对象 // 代理IP格式: 协议://代理帐号(可选):代理密码(可选)@ip:port //worker, err := miner.NewWorker("http://xx:xxxx@103.118.111.26:808") //worker, err := miner.NewWorker(nil) // 正常矿工 默认带Cookie //worker := miner.NewAPI() // API矿工 默认不带Cookie worker, err := miner.New(nil) // NewWorker同名函数 if err != nil { panic(err) } // 第四步：设置抓取方式和网站,可链式结构设置,只有SetUrl是必须的 // SetUrl:Url必须设置 // SetMethod:HTTP方法可以是POST或GET,可不设置,默认GET,传错值默认为GET // SetWaitTime:暂停时间,可不设置,默认不暂停 worker.SetUrl("https://www.bing.com").SetMethod(miner.GET).SetWaitTime(2) worker.SetUa(miner.RandomUa()) //设置随机浏览器标志 worker.SetRefer("https://www.bing.com") // 设置Refer头 worker.SetHeaderParam("diyheader", "diy") // 自定义头部 //worker.SetBData([]byte("file data")) // 如果你要提交JSON数据/上传文件 //worker.SetFormParam("username","hunterhug") // 提交表单 //worker.SetFormParam("password","123") // 第五步：开始爬 //worker.Get() // 默认GET //worker.Post() // POST表单请求,数据在SetFormParam() //worker.PostJSON() // 提交JSON请求,数据在SetBData() //worker.PostXML() // 提交XML请求,数据在SetBData() //worker.PostFILE() // 提交文件上传请求,数据在SetBData() body, err := worker.Go() // 如果设置SetMethod(),采用,否则Get() if err != nil { log.Error(err.Error()) } else { log.Infof("%s", string(body)) // 打印获取的数据 } log.Debugf("%#v", worker.GetCookies()) // 不设置全局log为debug是不会出现这个东西的 // 爬取完毕后可以清除POST的表单数据/文件数据/JSON数据 // 我建议使用 Clone()，这样也可以不用清除 worker.Clear() // 爬取完毕后可以清除设置的Http头部和POST的表单数据/文件数据/JSON数据 //worker.ClearAll() // 矿工池子 miner.Pool.Set("worker1", worker) if w, ok := miner.Pool.Get("worker1"); ok { go func() { data, _ := w.Clone().SetUrl("https://www.bing.com").Get() log.Info(string(data)) }() util.Sleep(10) } } ================================================ FILE: example/practice/pictures/README.md ================================================ # 多线程批量抓图片 直接运行程序即可。 ```go go run main.go Welcome: Input "url" and picture keep "dir" --------------------------------------------- URL(Like: "https://www.bizhizj.com") DIR(Default: "./picture") ``` ================================================ FILE: example/practice/pictures/main.go ================================================ package main import ( "errors" "fmt" "github.com/hunterhug/marmot/expert" "github.com/hunterhug/marmot/miner" "github.com/hunterhug/marmot/util" "net/url" "strings" ) // MinerNum Num of miner, We can run it at the same time to crawl data fast var MinerNum = 5 // ProxyAddress You can update this decides whether to proxy var ProxyAddress interface{} func main() { // You can Proxy! // ProxyAddress = "socks5://127.0.0.1:1080" fmt.Println(`Welcome: Input "url" and picture keep "dir"`) fmt.Println("---------------------------------------------") urlPicture := util.Input(`URL(Like: "https://www.bizhizj.com")`, "https://www.bizhizj.com/dongman") dir := util.Input(`DIR(Default: "./picture")`, "./picture") fmt.Printf("You will keep %s picture in dir %s\n", urlPicture, dir) fmt.Println("---------------------------------------------") // Start Catch err := DownloadHTMLPictures(urlPicture, dir, MinerNum, ProxyAddress) if err != nil { fmt.Println("Error:" + err.Error()) } } // DownloadHTMLPictures Download one HTML page's all pictures // @URL: http://image.baidu.com // @SaveDir /home/images // @ProxyAddress : "socks5://127.0.0.1:1080" func DownloadHTMLPictures(URL string, SaveDir string, MinerNum int, ProxyAddress interface{}) error { // Check valid _, err := url.Parse(URL) if err != nil { return err } // New a worker to get url worker, err := miner.New(ProxyAddress) if err != nil { return err } result, err := worker.SetUrl(URL).SetUa(miner.RandomUa()).Get() if err != nil { return err } // Find all picture pictures := expert.FindPicture(string(result)) return DownloadURLPictures(pictures, SaveDir, MinerNum, worker) } // DownloadURLPictures Download pictures faster! func DownloadURLPictures(PictureUrls []string, SaveDir string, MinerNum int, initWorker *miner.Worker) error { // Empty, What a pity! if len(PictureUrls) == 0 { return errors.New("empty") } // Make dir! err := util.MakeDir(SaveDir) if err != nil { return err } // Divide pictures into several worker xxx, _ := util.DivideStringList(PictureUrls, MinerNum) // Chanel to info exchange chs := make(chan int, len(PictureUrls)) // Go at the same time for num, pictureList := range xxx { // Clone new worker workerPicture := initWorker.Clone() workerPicture.SetUa(miner.RandomUa()) // Go save picture! go func(images []string, worker *miner.Worker, num int) { for _, img := range images { // Check, May be Pass _, err := url.Parse(img) if err != nil { continue } // Change Name of our picture filename := strings.Replace(util.ValidFileName(img), "#", "_", -1) // Exist? if util.FileExist(SaveDir + "/" + filename) { fmt.Println("File Exist：" + SaveDir + "/" + filename) chs <- 0 } else { // Not Exsit? imgsrc, e := worker.SetUrl(img).Get() if e != nil { fmt.Println("Download " + img + " error:" + e.Error()) chs <- 0 return } // Save it! e = util.SaveToFile(SaveDir+"/"+filename, imgsrc) if e == nil { fmt.Printf("SP%d: Keep in %s/%s\n", num, SaveDir, filename) } chs <- 1 } } }(pictureList, workerPicture, num) } // Every picture should return for i := 0; i < len(PictureUrls); i++ { <-chs } return nil } ================================================ FILE: example/practice/upload/README.md ================================================ # 模拟上传文件 ## 启动服务端 ``` go run main.go ``` 打开 127.0.0.1:1789 可以看到上传文件的界面。 但我们使用程序模拟上传。 ## 启动模拟上传程序 ``` cd client go run main.go ``` 将会看到出现一个 `doc.go.xxxx` 的文件出现在目录下。 ================================================ FILE: example/practice/upload/client/main.go ================================================ package main import ( "fmt" "github.com/hunterhug/marmot/miner" ) func postFile(filename string, targetUrl string) { worker, _ := miner.New(nil) result, err := worker.SetUrl(targetUrl).SetBData([]byte("dddd")).SetFileInfo(filename+".xxxxxx", "uploadfile").SetFormParam("xxxx", "xxx").PostFILE() if err != nil { fmt.Println(err.Error()) } else { fmt.Println(string(result)) } } // sample usage func main() { targetUrl := "http://127.0.0.1:1789/upload" filename := "./doc.go" postFile(filename, targetUrl) } ================================================ FILE: example/practice/upload/main.go ================================================ package main import ( "fmt" "io" "net/http" "os" ) func main() { fmt.Println("max upload size", 32<<20) fmt.Println("web url: 127.0.0.1:1789") http.HandleFunc("/", index) http.HandleFunc("/upload", upload) http.ListenAndServe(":1789", nil) } func upload(w http.ResponseWriter, r *http.Request) { r.ParseMultipartForm(32 << 20) file, handler, err := r.FormFile("uploadfile") if err != nil { fmt.Println(err) return } defer file.Close() f, err := os.OpenFile(handler.Filename, os.O_WRONLY|os.O_CREATE, 0666) if err != nil { fmt.Println(err) return } defer f.Close() io.Copy(f, file) fmt.Fprintln(w, "upload ok!") } func index(w http.ResponseWriter, r *http.Request) { w.Write([]byte(tpl)) } const tpl = ` 上传文件
` ================================================ FILE: expert/query.go ================================================ package expert // Package expert is use to parse content import ( "github.com/PuerkitoBio/goquery" "regexp" "strings" ) func QueryBytes(content []byte) (*goquery.Document, error) { doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(content))) return doc, err } func QueryString(content string) (*goquery.Document, error) { doc, err := goquery.NewDocumentFromReader(strings.NewReader(content)) return doc, err } // FindPicture Find All picture. Must prefix with http(s) func FindPicture(s string) []string { picList := make([]string, 0) re, _ := regexp.Compile(`src\s*=\s*["'](http[s]?:\/\/.*?\.(jpg|jpeg|png|gif))["']`) output := re.FindAllStringSubmatch(s, -1) for _, o := range output { picList = append(picList, o[1]) } return picList } ================================================ FILE: expert/query_test.go ================================================ package expert import ( "fmt" "testing" ) func TestFindPicture(t *testing.T) { data := ` https://imgsa.baidu.com/forum/w%3D580/sign=294db374d462853592e0d229a0ee76f2/e732c895d143ad4b630e8f4683025aafa40f0611.jpg https://imgsa.baidu.com/forum/whttps:/4d462853592e0d229a0ee76f2/e732c895d143ad4b630e8f4683025aafa40f0611.jpg https://img1.jpg http://im62853592e0d229a0ee76f2/e732c895d143ad4b630e8f4683025aafa40f0611.jpgsfsadfsda httpdb374://aafa40f0611.jpg src="http://s.jpg" src="https://s.jpg" src = "http://s.jpg" src="https:s.jpg" "https://img1.jpg" "https://img1.jpgsss","https://img1.jpgss","https://img1.jpgss" src="https://up.enterdesk.com/edpic_360_360/bf/53/52/bf5352f28a839e01dd3fcb811669102c.jpg" ` result := FindPicture(data) fmt.Printf("%#v", result) } ================================================ FILE: go.mod ================================================ module github.com/hunterhug/marmot go 1.25.5 require ( github.com/PuerkitoBio/goquery v1.8.0 github.com/hunterhug/golog/v2 v2.0.0 golang.org/x/net v0.52.0 ) require ( github.com/andybalholm/cascadia v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect ) ================================================ FILE: go.sum ================================================ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/hunterhug/golog/v2 v2.0.0 h1:atgT5kBKBxosf/9DfiAmbz7FWzO6VFKMJV3cEg0drB4= github.com/hunterhug/golog/v2 v2.0.0/go.mod h1:oxkZi9qle15EMD6MBjSGGYcepW7YGWUfrrZx44oIoKU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: miner/client.go ================================================ package miner import ( "crypto/tls" "errors" "golang.org/x/net/proxy" "net/http" "net/http/cookiejar" "net/url" "strings" "github.com/hunterhug/marmot/util" ) // NewJar Cookie record Jar func NewJar() *cookiejar.Jar { cookieJar, _ := cookiejar.New(nil) return cookieJar } // Default Client var ( // Client Save Cookie Client = &http.Client{ CheckRedirect: func(req *http.Request, via []*http.Request) error { Logger.Debugf("[GoWorker] Redirect: %v", req.URL) return nil }, Jar: NewJar(), Timeout: util.Second(DefaultTimeOut), Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } // NoCookieClient Not Save Cookie NoCookieClient = &http.Client{ CheckRedirect: func(req *http.Request, via []*http.Request) error { Logger.Debugf("[GoWorker] Redirect: %v", req.URL) return nil }, Timeout: util.Second(DefaultTimeOut), Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } ) // NewProxyClient New a Proxy client, Default save cookie, Can timeout // We should support some proxy way such as http(s) or socks func NewProxyClient(proxyString string) (*http.Client, error) { proxyUrl, err := url.Parse(proxyString) if err != nil { return nil, err } prefix := strings.Split(proxyString, ":")[0] // setup a http transport httpTransport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} // http:// // https:// // socks5:// switch prefix { case "http", "https": httpTransport.Proxy = http.ProxyURL(proxyUrl) case "socks5": // create a socks5 dialer dialer, err := proxy.FromURL(proxyUrl, proxy.Direct) if err != nil { return nil, err } httpTransport.Dial = dialer.Dial default: return nil, errors.New("this proxy way not allow:" + prefix) } // This alone client, diff from global client. client := &http.Client{ // Allow redirect CheckRedirect: func(req *http.Request, via []*http.Request) error { Logger.Debugf("[GoWorker] Redirect: %v", req.URL) return nil }, // Allow proxy: http, https, socks5 Transport: httpTransport, // Allow keep cookie Jar: NewJar(), // Allow Timeout Timeout: util.Second(DefaultTimeOut), } return client, nil } // NewClient New a client, diff from proxy client func NewClient() *http.Client { client := &http.Client{ // Allow redirect CheckRedirect: func(req *http.Request, via []*http.Request) error { Logger.Debugf("[GoWorker] Redirect: %v", req.URL) return nil }, Jar: NewJar(), Timeout: util.Second(DefaultTimeOut), Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, }, } return client } ================================================ FILE: miner/config/ua.txt ================================================ Mozilla/5.0 (SymbianOS/9.1; U; [en]; SymbianOS/91 Series60/3.0) AppleWebkit/413 (KHTML, like Gecko) Safari/413 Mozilla/5.0 (SymbianOS/9.1; U; [en]; Series60/3.0 NokiaE60/4.06.0) AppleWebKit/413 (KHTML, like Gecko) Safari/413 Mozilla/5.0 (SymbianOS/9.2; U; Series60/3.1 NokiaN95/10.0.018; Profile/MIDP-2.0 Configuration/CLDC-1.1) AppleWebKit/413 (KHTML, like Gecko) Safari/413 Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1/110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413 Mozilla/5.0 (SymbianOS/9.4; U; Series60/5.0 Nokia5800d-1/21.0.025; Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413 Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/12.0.024; Profile/MIDP-2.1 Configuration/CLDC-1.1; en-us) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.12344 Mozilla/4.0 (compatible; MSIE 4.0; MSN 2.5; Windows 95) Mozilla/4.0 (compatible; MSIE 4.0; Windows 95) Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; MSN 2.5; Windows 98) Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; Windows 95) Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; Windows 98) Mozilla/4.0 (compatible; MSIE 4.01; Windows 95) Mozilla/4.0 (compatible; MSIE 4.01; Windows 95; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.) Mozilla/4.0 (compatible; MSIE 4.01; Windows 98) Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; BIGLOBE) Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; canoncopyer) Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; Compaq) Mozilla/4.0 (compatible; MSIE 5.0; Windows 95) Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt) Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt; i-CABLE) Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt; ocnie5-1) Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.) Mozilla/4.0 (compatible; MSIE 5.0; Windows 98) Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; CNETHomeBuild051099) Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt) Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt; ocnie5-1) Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; wn_ie5_en_v1) Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.; DigExt) Mozilla/4.0 (compatible; MSIE 5.0; Windows NT) Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt) Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt) Mozilla/4.0 (compatible; MSIE 5.01; MSN 2.5; Windows 98) Mozilla/4.0 (compatible; MSIE 5.01; Windows 95) Mozilla/4.0 (compatible; MSIE 5.01; Windows 98) Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0) Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; DigExt) Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; TUCOWS) Mozilla/4.0 (compatible; MSIE 5.01; Windows NT) Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC) Mozilla/4.0 (compatible; MSIE 5.16; Mac_PowerPC) Mozilla/4.0 (compatible; MSIE 5.17; Mac_PowerPC) Mozilla/4.0 (compatible; MSIE 5.22; Mac_PowerPC) Mozilla/4.0 (compatible; MSIE 5.23; Mac_PowerPC) Mozilla/4.0 (compatible; MSIE 5.5; Windows 95) Mozilla/4.0 (compatible; MSIE 5.5; Windows 98) Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0) Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705) Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; by TSG) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 1.0.3705) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.40607) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.40607; .NET CLR 1.0.3705) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; NOKTURNAL KICKS ASS) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FDM; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.41115) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1 Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1 Mozilla/4.0 (compatible; MSIE 6.0; Windows 98) Mozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows NT 5.1; iebar; .NET CLR 1.0.3705) Mozilla/4.0 (compatible; MSIE 6.0; Win32); Mozilla/4.0 (compatible; MSIE 6.0; Win32); .NET CLR 1.0.3705) Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Hotbar 4.4.6.0) Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x 4.90) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0; BVG Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; FDM) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; {FF0C8E09-3C86-44CB-834A-B8CEEC80A1D7}; iOpus-I-M) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; i-Nav 3.0.1.0F; .NET CLR 1.0.3705; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; MathPlayer 2.0; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Maxthon; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1); Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 1.0.3705) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 2.0.40607) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Alexa Toolbar) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; BrowserBob) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; DFO-MPO Internet Explorer 6.0) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ENGINE; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT)) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ESB{F40811EE-DF17-4BC9-8785-B362ABF34098}; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FDM) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FTDv3 Browser; .NET CLR 1.0.3705; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; .NET CLR 1.1.4322; .NET CLR 2.0.40607) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; AtHome033) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; HCI0449; .NET CLR 1.0.3705) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; i-NavFourF; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2; Maxthon; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461; FunWebProducts; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Woningstichting Den Helder; .NET CLR 1.0.3705) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.41115) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; MyIE2; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; MyIE2; Maxthon; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 6.0; Windows XP) Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1) Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 2.0.50727; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322) Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1) Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1) Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30) Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506) Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; InfoPath.1) Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727) Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648) Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; InfoPath.1 Mozilla/4.0 (compatible; GoogleToolbar 5.0.2124.2070; Windows 6.0; MSIE 8.0.6001.18241) Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; EasyBits Go v1.0; InfoPath.1; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729) Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0) Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; Sleipnir/2.9.8) Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0) Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0) Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0) Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0) Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0) Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; ARM; Trident/6.0) Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.48 Safari/525.19 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.33 Safari/530.5 Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.38 Safari/532.0 Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.55 Safari/533.4 Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.55 Safari/534.3 Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.43 Safari/534.7 Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10 ChromePlus/1.5.2.0 Mozilla/5.0 (en-us) AppleWebKit/534.14 (KHTML, like Gecko; Google Wireless Transcoder) Chrome/9.0.597 Safari/534.14 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.151 Safari/534.16 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.71 Safari/534.24 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Iron/11.0.700.2 Chrome/11.0.700.2 Safari/534.24 Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.24 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30 ChromePlus/1.6.3.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.112 Safari/534.30 Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.107 Safari/535.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) RockMelt/0.9.64.361 Chrome/13.0.782.218 Safari/535.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.220 Safari/535.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2 Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.2 (KHTML, like Gecko) Ubuntu/10.04 Chromium/15.0.874.106 Chrome/15.0.874.106 Safari/535.2 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.106 Safari/535.2 Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7 Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30618; Lunascape 4.7.3) Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; Lunascape 5.0 alpha2) Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; Lunascape 5.0 alpha2) Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; Tablet PC 2.0; Lunascape 5.0 alpha2) Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MAGW; .NET4.0C; Lunascape 6.5.8.24780) Mozilla/2.02 (Macintosh; I; PPC) Mozilla/3.01 (Macintosh; I; PPC) Mozilla/4.01 (Macintosh; I; PPC) Mozilla/4.03 [en]C-IMS (Win95; I) Mozilla/4.04 [en] (Win95; I ;Nav) Mozilla/4.04 [en] (Win95; I) Mozilla/4.04 [en] (WinNT; I ;Nav) Mozilla/4.04 [en] (WinNT; I) Mozilla/4.04 [en] (Macintosh; I; PPC Nav) Mozilla/4.04 [en] (X11; I; SunOS 5.5 sun4u) Mozilla/4.05 [en] (Win95; I) Mozilla/4.05 (Macintosh; I; PPC) Mozilla/4.06 [en] (Win98; I) Mozilla/4.06 [en] (Macintosh; I; PPC) Mozilla/4.08 (Macintosh; I; PPC) Mozilla/4.5 [en] (Win95; I) Mozilla/4.5 [en] (Win98; I) Mozilla/4.5 [en] (WinNT; I) Mozilla/4.5 (Macintosh; I; PPC) Mozilla/4.51 [en] (Win95; I) Mozilla/4.51 [en] (Win98; I) Mozilla/4.51 [en] (WinNT; I) Mozilla/4.51 [en] (X11; I; SunOS 5.8 sun4u) Mozilla/4.6 [en] (Win95; I) Mozilla/4.6 [en] (Win98; I) Mozilla/4.6 [en] (WinNT; I) Mozilla/4.6 [en] (WinNT; I) Mozilla/4.7 [en] (WinNT; I) Mozilla/4.7 [en] (Win95; I) Mozilla/4.7 [en] (Win98; I) Mozilla/4.7 [en] (WinNT; I) Mozilla/4.7 [en] (WinNT; I) Mozilla/4.7 [en] (WinNT; U) Mozilla/4.7 [en] (Macintosh; I; PPC) Mozilla/4.76 [en_jp] (X11; U; SunOS 5.8 sun4u) Mozilla/4.76 [en] (X11; U; SunOS 5.8 sun4u) Mozilla/4.78 [en] (X11; U; SunOS 5.9 sun4u) Mozilla/4.8 [en] (X11; U; SunOS 5.7 sun4u) Mozilla/5.0 (Windows; U; Win98; en-JP; m18) Gecko/20001108 Netscape6/6.0 Mozilla/5.0 (Windows; U; Windows NT 5.0; en-JP; m18) Gecko/20010131 Netscape6/6.01 Mozilla/5.0 (Windows; U; Win 9x 4.90; en-JP; rv:0.9.4) Gecko/20011128 Netscape6/6.2.1 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:0.9.4.1) Gecko/20020508 Netscape6/6.2.3 Mozilla/5.0 (Macintosh; N; PPC; en-JP; macen-pub12) Gecko/20001108 Netscape6/6.0 Mozilla/5.0 (Macintosh; U; PPC; en-JP; rv:0.9.2) Gecko/20010726 Netscape6/6.1 Mozilla/5.0 (Macintosh; U; PPC; en-JP; rv:0.9.4) Gecko/20011022 Netscape6/6.2 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US; rv:0.9.4.1) Gecko/20020315 Netscape6/6.2.2 Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.0.2) Gecko/20030208 Netscape/7.02 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.4) Gecko/20030624 Netscape/7.1 (ax) Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.4) Gecko/20030624 Netscape/7.1 (ax) Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.2) Gecko/20040804 Netscape/7.2 (ax) Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.2) Gecko/20040805 Netscape/7.2 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-JP; rv:1.0.2) Gecko/20021120 Netscape/7.01 Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-JP; rv:1.4) Gecko/20030624 Netscape/7.1 Mozilla/5.0 (X11; U; SunOS sun4u; en-JP; rv:1.0.1) Gecko/20020921 Netscape/7.0 Mozilla/5.0 (Windows; U; Windows NT 5.0; en-JP; rv:1.5) Gecko/20031007 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113 Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040616 Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.6) Gecko/20040113 Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.2.1) Gecko/20030225 Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.4.1) Gecko/20031030 Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.1) Gecko/20040805 Mozilla/5.0 (X11; U; SunOS sun4u; en-JP; rv:1.4) Gecko/20040414 Mozilla/5.0 (X11; U; Linux i686; rv:1.7.3) Gecko/20040913 Firefox/0.10 Mozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) Gecko/20040913 Firefox/0.10 Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20040913 Firefox/0.10 Mozilla/5.0 (X11; U; Linux i686; rv:1.7.3) Gecko/20041001 Firefox/0.10.1 Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1 Mozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) Gecko/20041001 Firefox/0.10.1 Mozilla/5.0 (Windows; U; Windows NT 5.2; rv:1.7.3) Gecko/20041001 Firefox/0.10.1 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.6) Gecko/20040206 Firefox/0.8 Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.6) Gecko/20040206 Firefox/0.8 Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.6) Gecko/20040207 Firefox/0.8 Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.6) Gecko/20040206 Firefox/0.8 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040614 Firefox/0.9 Mozilla/5.0 (X11; U; Linux i686; es-ES; rv:1.7) Gecko/20040708 Firefox/0.9 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040707 Firefox/0.9.2 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/0.9.2 StumbleUpon/1.994 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040803 Firefox/0.9.3 Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3 Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3 Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3 Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3 Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20040803 Firefox/0.9.3 Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7) Gecko/20041013 Firefox/0.9.3 (Ubuntu) Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7) Gecko/20041013 Firefox/0.9.3 (Ubuntu) Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 Mozilla/5.0 (Windows; U; Win 9x 4.90; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0 Mozilla/5.0 (Windows; U; Win 9x 4.90; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0 Mozilla/5.0 (Windows; U; Win98; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.7.5) Gecko/20041107 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.7.5) Gecko/20041110 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.0; fr-FR; rv:1.7.5) Gecko/20041108 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.0; it-IT; rv:1.7.5) Gecko/20041110 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041107 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.7.5) Gecko/20041110 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 StumbleUpon/1.999 Mozilla/5.0 (Windows; U; Windows NT 5.1; es-ES; rv:1.7.5) Gecko/20041210 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.1; fr-FR; rv:1.7.5) Gecko/20041108 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.1; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.1; sv-SE; rv:1.7.5) Gecko/20041108 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.8b) Gecko/20050212 Firefox/1.0+ (MOOX M3) Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8b) Gecko/20050118 Firefox/1.0+ Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8b) Gecko/20050118 Firefox/1.0+ Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.5) Gecko/20050103 Firefox/1.0 Mozilla/5.0 (X11; U; Linux i386; en-US; rv:1.7.5) Gecko/20041109 Firefox/1.0 Mozilla/5.0 (X11; U; Linux i686; de-DE; rv:1.7.5) Gecko/20041128 Firefox/1.0 (Debian package 1.0-4) Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0 Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041111 Firefox/1.0 Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041111 Firefox/1.0 (Debian package 1.0-2) Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041116 Firefox/1.0 (Ubuntu) (Ubuntu package 1.0-2ubuntu3) Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041119 Firefox/1.0 Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041123 Firefox/1.0 Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041128 Firefox/1.0 (Debian package 1.0-4) Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041130 Firefox/1.0 Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041214 Firefox/1.0 StumbleUpon/1.999 Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041219 Firefox/1.0 (Debian package 1.0+dfsg.1-1) Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20050110 Firefox/1.0 (Debian package 1.0+dfsg.1-2) Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 Mozilla/5.0 (X11; U; Linux i686; nl-NL; rv:1.7.5) Gecko/20050221 Firefox/1.0 (Ubuntu) (Ubuntu package 1.0+dfsg.1-6ubuntu1) Mozilla/5.0 (X11; U; Linux i686; rv:1.8b) Gecko/20050124 Firefox/1.0+ Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.6) Gecko/20050306 Firefox/1.0.1 (Debian package 1.0.1-2) Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6) Gecko/20050223 Firefox/1.0.1 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6) Gecko/20050225 Firefox/1.0.1 Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.6) Gecko/20050226 Firefox/1.0.1 Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-JP-mac; rv:1.8) Gecko/20051111 Firefox/1.5 Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8) Gecko/20051111 Firefox/1.5 Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8) Gecko/20051111 Firefox/1.5 Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.0.1) Gecko/20060111 Firefox/1.5.0.1 Mozilla/5.0 (X11; U; Linux i686; en; rv:1.8.0.2) Gecko/20060308 Firefox/1.5.0.2 Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3 Mozilla/5.0 (X11; U; Linux i686; en; rv:1.8.0.4) Gecko/20060508 Firefox/1.5.0.4 Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20 Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.12) Gecko/20080201 Firefox/2.0.0.12 Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-JP-mac; rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20 Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6 Mozilla/5.0 (Windows; U; Windows NT 6.0; en; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6 (.NET CLR 3.5.30729) Mozilla/5.0 (Windows; U; Windows NT 6.0; en; rv:1.9.0.17) Gecko/2009122116 Firefox/3.0.17 GTB6 (.NET CLR 3.5.30729) Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-JP-mac; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6 Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-JP-mac; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6 GTB5 Mozilla/5.0 (Windows NT 6.1; rv:2.0) Gecko/20100101 Firefox/4.0 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:2.0) Gecko/20100101 Firefox/4.0 Mozilla/5.0 (Windows NT 6.1; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:5.0.1) Gecko/20100101 Firefox/5.0.1 Mozilla/5.0 (Windows NT 5.1; rv:6.0) Gecko/20100101 Firefox/6.0 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:6.0.2) Gecko/20100101 Firefox/6.0.2 Mozilla/5.0 (Windows NT 6.0; rv:7.0.1) Gecko/20100101 Firefox/7.0.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:7.0.1) Gecko/20100101 Firefox/7.0.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:8.0.1) Gecko/20100101 Firefox/8.0.1 Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:9.0.1) Gecko/20100101 Firefox/9.0.1 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.6 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.7 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/85.8.2 (KHTML, like Gecko) Safari/85.8 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/85.8.2 (KHTML, like Gecko) Safari/85.8.1 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/85.8.5 (KHTML, like Gecko) Safari/85.8 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/103u (KHTML, like Gecko) Safari/100.1 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/124 (KHTML, like Gecko) Safari/125.1 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/125.2 (KHTML, like Gecko) Safari/125.8 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/125.4 (KHTML, like Gecko) Safari/125.9 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.11 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; nl-nl) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/125.5.6 (KHTML, like Gecko) Safari/125.12 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.1 (KHTML, like Gecko) Safari/312 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.1.1 (KHTML, like Gecko) Safari/312 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.5 (KHTML, like Gecko) Safari/312.3 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.5 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.6 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412 (KHTML, like Gecko) Safari/412 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.6.2 (KHTML, like Gecko) Safari/125.11 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.6.2 (KHTML, like Gecko) Safari/412.2.2 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.7 (KHTML, like Gecko) Safari/412.5 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/416.11 (KHTML, like Gecko) Safari/416.12 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.2 Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-jp) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.3 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/522.11.1 (KHTML, like Gecko) Version/3.0.3 Safari/522.12.1 Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-jp) AppleWebKit/523.10.3 (KHTML, like Gecko) Version/3.0.4 Safari/523.10 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.10.6 (KHTML, like Gecko) Version/3.0.4 Safari/523.10.6 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.12 (KHTML, like Gecko) Version/3.0.4 Safari/523.12 Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.12.2 (KHTML, like Gecko) Version/3.0.4 Safari/523.12.2 Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en-jp) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13 Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_3; en-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.1 Safari/525.20 Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.22 Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; en-jp) AppleWebKit/525.26.2 (KHTML, like Gecko) Version/3.2 Safari/525.26.12 Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-jp) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1 Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-jp) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16 Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-jp) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10 Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-jp) AppleWebKit/531.21.11 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10 Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-jp) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.52.7 (KHTML, like Gecko) Version/5.1.2 Safari/534.52.7 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8) AppleWebKit/536.25 (KHTML, like Gecko) Version/6.0 Safari/536.25 Mozilla/4.0 (Windows 95;US) Opera 3.62 [en] Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC) Opera 6.0 [en] Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.01 [en] Mozilla/4.0 (compatible; MSIE 5.0; Windows ME) Opera 6.03 [en] Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.05 [en] Mozilla/4.0 (compatible; MSIE 5.0; Windows XP) Opera 6.06 [en] Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.06 [en] Opera 7.11 Opera/7.23 (Windows NT 5.0; U) [en] Opera/7.52 (Windows NT 5.1; U) [en] Opera/7.53 (Windows NT 5.0; U) [en] Opera/7.54 (Windows NT 5.0; U) [en] Opera/7.54 (Windows NT 5.1; U) [en] Opera/7.54 (Windows NT 5.1; U) Opera/7.54 (X11; Linux i686; U) [en] Opera/7.54 (X11; Linux i686; U) [sv] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.11 [en] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.21 [pt-BR] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.22 [en] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.23 [en] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.23 [en] Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686) Opera 7.23 [en] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.23 [fr] Mozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC) Opera 7.50 [en] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.53 [en] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.53 [en] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.54 [en] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.54 [en] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.54 [en] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.54u1 [en] Opera/7.54 (Windows 98; U) [en] Mozilla/4.78 (Windows NT 5.1; U) Opera 7.23 [en] Mozilla/5.0 (Windows NT 5.0; U) Opera 7.54 [en] Mozilla/4.0 (compatible; MSIE 6.0; X11; OpenBSD i386) Opera 7.54 [en] Opera/8.0 (X11; Linux i686; U; en) Opera/8.01 (Windows ME; U; en) Opera/8.01 (Windows NT 5.1; U; en) Mozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC Mac OS X; en) Opera 8.01 Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.01 Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; tr) Opera 8.02 Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.02 Opera/8.5 (Windows NT 5.0; U; en) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.5 Opera/8.51 (Windows NT 5.1; U; en) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.51 Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; de) Opera 8.52 Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; pl) Opera 8.52 Mozilla/5.0 (Windows NT 5.1; U; en) Opera 8.52 Opera/8.53 (Windows NT 5.1; U; en) Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.53 Opera/8.54 (Windows NT 5.1; U; en) Opera/8.54 (Windows NT 5.0; U; en) Mozilla/5.0 (X11; Linux i686; U; cs) Opera 8.54 Mozilla/4.0 (compatible; MSIE 6.0; KDDI-SA39) Opera 8.60 [en] Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 9.00 Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.00 Opera/9.00 (Windows NT 5.1; U; en) Opera/9.0 (Windows NT 5.1; U; en) Opera/9.00 (Macintosh; PPC Mac OS X; U; en) Opera/9.02 (Macintosh; PPC Mac OS X; U; en) Opera/9.02 (Windows NT 5.1; U; zh-tw) Opera/9.10 (Windows NT 6.0; U; en) Opera/9.21 (Windows NT 6.0; U; en) Opera/9.22 (Windows NT 5.1; U; en) Opera/9.23 (Windows NT 5.1; U; en) Opera/9.23 (Windows ME; U; en) Opera/9.26 (Windows NT 5.1; U; en) Opera/9.51 (Windows NT 5.1; U; en) Opera/9.52 (Macintosh; Intel Mac OS X; U; en) Opera/9.52 (Windows NT 5.1; U; en) Opera/9.60 (Macintosh; Intel Mac OS X; U; en) Presto/2.1.1 Opera/9.60 (Windows NT 5.1; U; en) Presto/2.1.1 Opera/9.61 (Windows NT 5.1; U; en) Presto/2.1.1 Opera/9.62 (Windows NT 5.1; U; en) Presto/2.1.1 Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686; en) Opera 10.10 Opera/9.80 (Windows NT 6.1; U; en) Presto/2.9.168 Version/11.50 Opera/9.80 (Windows NT 6.1; U; en) Presto/2.10.229 Version/11.60 Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.10.289 Version/12.00 ================================================ FILE: miner/config.go ================================================ package miner import ( "context" "net/http" "net/url" "strings" "sync" "github.com/hunterhug/marmot/util" ) // Worker is the main object to sent http request and return result of response type Worker struct { // In order fast chain func call I put the basic config below *Request *Response // Which url we want Url string // Get,Post method Method string // Our Client Client *http.Client // Wait Sleep Time Wait int // Worker proxy ip, just for user to record their proxy ip, default: localhost Ip string // AOP like Java Ctx context.Context BeforeAction func(context.Context, *Worker) AfterAction func(context.Context, *Worker) // Http header Header http.Header // DownloadFileReadMB is the size of the file read in MB DownloadFileReadMB int64 // Mux lock mux sync.RWMutex } type Request struct { Data url.Values // Sent by form data FileName string // FileName which sent to remote FileFormName string // File Form Name which sent to remote BData []byte // Sent by binary data, can together with File Request *http.Request // Debug } type Response struct { Response *http.Response // Debug Raw []byte // Raw data we get ResponseStatusCode int // The last url response code, such as 404 } // SetHeader Java Bean Chain pattern func (worker *Worker) SetHeader(header http.Header) *Worker { worker.Header = header return worker } // SetHeader Default Worker SetHeader! func SetHeader(header http.Header) *Worker { return DefaultWorker.SetHeader(header) } func (worker *Worker) SetHeaderParam(k, v string) *Worker { worker.Header.Set(k, v) return worker } func SetHeaderParam(k, v string) *Worker { return DefaultWorker.SetHeaderParam(k, v) } func (worker *Worker) SetCookie(v string) *Worker { worker.SetHeaderParam("Cookie", v) return worker } func SetCookie(v string) *Worker { return DefaultWorker.SetCookie(v) } // SetCookieByFile Set Cookie by file. func (worker *Worker) SetCookieByFile(file string) (*Worker, error) { haha, err := util.ReadFromFile(file) if err != nil { return nil, err } cookie := string(haha) cookie = strings.Replace(cookie, " ", "", -1) cookie = strings.Replace(cookie, "\n", "", -1) cookie = strings.Replace(cookie, "\r", "", -1) return worker.SetCookie(cookie), nil } func SetCookieByFile(file string) (*Worker, error) { return DefaultWorker.SetCookieByFile(file) } func (worker *Worker) SetUa(ua string) *Worker { worker.Header.Set("User-Agent", ua) return worker } func SetUa(ua string) *Worker { return DefaultWorker.SetUa(ua) } func (worker *Worker) SetRefer(refer string) *Worker { worker.Header.Set("Referer", refer) return worker } func SetRefer(refer string) *Worker { return DefaultWorker.SetRefer(refer) } func (worker *Worker) SetHost(host string) *Worker { worker.Header.Set("Host", host) return worker } // SetUrl at the same time SetHost func (worker *Worker) SetUrl(url string) *Worker { worker.Url = url temp := strings.Split(url, "//") if len(temp) >= 2 { worker.SetHost(strings.Split(temp[1], "/")[0]) } return worker } func SetUrl(url string) *Worker { return DefaultWorker.SetUrl(url) } func (worker *Worker) SetFileInfo(fileName, fileFormName string) *Worker { worker.FileName = fileName worker.FileFormName = fileFormName return worker } func SetFileInfo(fileName, fileFormName string) *Worker { return DefaultWorker.SetFileInfo(fileName, fileFormName) } func (worker *Worker) SetMethod(method string) *Worker { temp := GET switch strings.ToUpper(method) { case GET: temp = GET case POST: temp = POST case POSTFILE: temp = POSTFILE case POSTJSON: temp = POSTJSON case POSTXML: temp = POSTXML case PUT: temp = PUT case PUTFILE: temp = PUTFILE case PUTJSON: temp = PUTJSON case PUTXML: temp = PUTXML case DELETE: temp = DELETE default: temp = OTHER } worker.Method = temp return worker } func SetMethod(method string) *Worker { return DefaultWorker.SetMethod(method) } func (worker *Worker) SetWaitTime(num int) *Worker { if num <= 0 { num = 1 } worker.Wait = num return worker } func SetWaitTime(num int) *Worker { return DefaultWorker.SetWaitTime(num) } func (worker *Worker) SetDownloadFileReadMB(downloadFileReadMB int64) *Worker { if downloadFileReadMB <= 0 { downloadFileReadMB = 5 } worker.DownloadFileReadMB = downloadFileReadMB return worker } func SetDownloadFileReadMB(downloadFileReadMB int64) *Worker { return DefaultWorker.SetDownloadFileReadMB(downloadFileReadMB) } func (worker *Worker) SetBData(data []byte) *Worker { worker.BData = data return worker } func SetBData(data []byte) *Worker { return DefaultWorker.SetBData(data) } func (worker *Worker) SetForm(form url.Values) *Worker { worker.Data = form return worker } func SetForm(form url.Values) *Worker { return DefaultWorker.SetForm(form) } func (worker *Worker) SetFormParam(k, v string) *Worker { worker.Data.Set(k, v) return worker } func SetFormParam(k, v string) *Worker { return DefaultWorker.SetFormParam(k, v) } // SetContext Set Context so Action can soft func (worker *Worker) SetContext(ctx context.Context) *Worker { worker.Ctx = ctx return worker } func SetContext(ctx context.Context) *Worker { return DefaultWorker.SetContext(ctx) } func (worker *Worker) SetBeforeAction(fc func(context.Context, *Worker)) *Worker { worker.BeforeAction = fc return worker } func SetBeforeAction(fc func(context.Context, *Worker)) *Worker { return DefaultWorker.SetBeforeAction(fc) } func (worker *Worker) SetAfterAction(fc func(context.Context, *Worker)) *Worker { worker.AfterAction = fc return worker } func SetAfterAction(fc func(context.Context, *Worker)) *Worker { return DefaultWorker.SetAfterAction(fc) } // Clear data we sent // I suggest use Clone() to avoid clear func (worker *Worker) Clear() *Worker { worker.Request = newRequest() worker.Response = new(Response) return worker } func Clear() *Worker { return DefaultWorker.Clear() } // ClearAll All clear include header func (worker *Worker) ClearAll() *Worker { worker.Clear() worker.Header = http.Header{} return worker } func ClearAll() *Worker { return DefaultWorker.ClearAll() } // ClearCookie Clear Cookie func (worker *Worker) ClearCookie() *Worker { worker.Header.Del("Cookie") return worker } func ClearCookie() *Worker { return DefaultWorker.ClearCookie() } // GetCookies Get Cookies func (worker *Worker) GetCookies() []*http.Cookie { if worker.Response != nil && worker.Response.Response != nil { return worker.Response.Response.Cookies() } else { return []*http.Cookie{} } } // GetResponseStatusCode Get ResponseStatusCode func (worker *Worker) GetResponseStatusCode() int { if worker.Response != nil && worker.Response.Response != nil { return worker.Response.ResponseStatusCode } else { return 0 } } func GetCookies() []*http.Cookie { return DefaultWorker.GetCookies() } func GetResponseStatusCode() int { return DefaultWorker.GetResponseStatusCode() } ================================================ FILE: miner/cons.go ================================================ package miner import "net/http" const ( VERSION = "1.0.13" // GET HTTP method GET = "GET" POST = "POST" POSTJSON = "POSTJSON" POSTXML = "POSTXML" POSTFILE = "POSTFILE" PUT = "PUT" PUTJSON = "PUTJSON" PUTXML = "PUTXML" PUTFILE = "PUTFILE" DELETE = "DELETE" OTHER = "OTHER" // this stand for you can use other method this lib not own. // HTTPFORMContentType HTTP content type HTTPFORMContentType = "application/x-www-form-urlencoded" HTTPJSONContentType = "application/json" HTTPXMLContentType = "text/xml" HTTPFILEContentType = "multipart/form-data" ) var ( // Browser User-Agent, Our default Http ua header! ourLoveUa = "Marmot+" + VERSION + "+github:hunterhug" DefaultHeader = map[string][]string{ "User-Agent": { ourLoveUa, }, } // DefaultTimeOut http get and post No timeout DefaultTimeOut = 0 ) // SetGlobalTimeout Set global timeout, it can only by this way! func SetGlobalTimeout(num int) { DefaultTimeOut = num } func SetDefaultTimeOut(num int) { DefaultTimeOut = num } // MergeCookie Merge Cookie, not use func MergeCookie(before []*http.Cookie, after []*http.Cookie) []*http.Cookie { cs := make(map[string]*http.Cookie) for _, b := range before { cs[b.Name] = b } for _, a := range after { if a.Value != "" { cs[a.Name] = a } } res := make([]*http.Cookie, 0, len(cs)) for _, q := range cs { res = append(res, q) } return res } // CloneHeader Clone a header, If not exist Ua, Set our Ua! func CloneHeader(h map[string][]string) map[string][]string { if h == nil || len(h) == 0 { h = DefaultHeader return h } if len(h["User-Agent"]) == 0 { h["User-Agent"] = []string{ourLoveUa} } return CopyM(h) } ================================================ FILE: miner/default.go ================================================ package miner import "net/http" // DefaultWorker Global Worker var DefaultWorker *Worker func init() { UaInit() // New a Worker worker := new(Worker) worker.Request = newRequest() worker.Header = http.Header{} worker.Response = new(Response) worker.Client = Client // Global Worker! DefaultWorker = worker } // ToString This make effect only your Worker exec serial! Attention! // Change Your Raw data To string func ToString() string { return DefaultWorker.ToString() } // JsonToString This make effect only your Worker exec serial! Attention! // Change Your JSON like Raw data to string func JsonToString() (string, error) { return DefaultWorker.JsonToString() } func Clone() *Worker { return DefaultWorker.Clone() } func Get() (body []byte, e error) { return DefaultWorker.Get() } func Delete() (body []byte, e error) { return DefaultWorker.Delete() } func Go() (body []byte, e error) { return DefaultWorker.Go() } func GoByMethod(method string) (body []byte, e error) { return DefaultWorker.SetMethod(method).Go() } func OtherGo(method, contentType string) (body []byte, e error) { return DefaultWorker.OtherGo(method, contentType) } func Post() (body []byte, e error) { return DefaultWorker.Post() } func PostJSON() (body []byte, e error) { return DefaultWorker.PostJSON() } func PostFILE() (body []byte, e error) { return DefaultWorker.PostFILE() } func PostXML() (body []byte, e error) { return DefaultWorker.PostXML() } func Put() (body []byte, e error) { return DefaultWorker.Put() } func PutJSON() (body []byte, e error) { return DefaultWorker.PutJSON() } func PutFILE() (body []byte, e error) { return DefaultWorker.PutFILE() } func PutXML() (body []byte, e error) { return DefaultWorker.PutXML() } ================================================ FILE: miner/doc.go ================================================ /* All right reserved https://github.com/hunterhug/marmot at 2016-2022 Attribution-NonCommercial-NoDerivatives 4.0 International Notice: The following code's copyright by hunterhug, Please do not spread and modify. You can use it for education only but can't make profits for any companies and individuals! */ /* Package miner is the core of this project, use to request for http api. Example: package main import ( "fmt" "github.com/hunterhug/marmot/miner" ) func main() { miner.SetLogLevel(miner.DEBUG) // Use Default Worker, You can Also New One: //worker, _ := miner.New(nil) //worker = miner.NewWorkerWithNoProxy() //worker = miner.NewAPI() //worker, _ = miner.NewWorkerWithProxy("socks5://127.0.0.1:1080") worker := miner.Clone() _, err := worker.SetUrl("https://www.bing.com").Go() if err != nil { fmt.Println(err.Error()) } else { fmt.Println(worker.ToString()) } } */ package miner ================================================ FILE: miner/log.go ================================================ package miner import ( "github.com/hunterhug/golog/v2" ) type Level = golog.Level const ( DEBUG Level = golog.DebugLevel INFO Level = golog.InfoLevel WARN Level = golog.WarnLevel ERROR Level = golog.ErrorLevel ) // Logger Global logger config for debug var ( Logger = golog.New() ) func init() { Logger.InitLogger() } // SetLogLevel Set log level func SetLogLevel(level Level) { Logger.SetLevel(level) Logger.InitLogger() } // Log Return global log func Log() golog.LoggerInterface { return Logger } ================================================ FILE: miner/multworker.go ================================================ package miner import ( "sync" ) // Pool for many Worker, every Worker can only serial execution var Pool = &_Workers{ws: make(map[string]*Worker)} type _Workers struct { mux sync.RWMutex ws map[string]*Worker } func (pool *_Workers) Get(name string) (b *Worker, ok bool) { pool.mux.RLock() b, ok = pool.ws[name] pool.mux.RUnlock() return } func (pool *_Workers) Set(name string, b *Worker) { pool.mux.Lock() pool.ws[name] = b pool.mux.Unlock() return } func (pool *_Workers) Delete(name string) { pool.mux.Lock() delete(pool.ws, name) pool.mux.Unlock() return } ================================================ FILE: miner/ua.go ================================================ package miner import ( "github.com/hunterhug/marmot/util" "math/rand" "strings" ) // Ua Global User-Agent provide var Ua = map[int]string{} // UaInit User-Agent init func UaInit() { Ua = map[int]string{ 0: "Mozilla/5.0 (SymbianOS/9.1; U; [en]; SymbianOS/91 Series60/3.0) AppleWebkit/413 (KHTML, like Gecko) Safari/413", 1: "Mozilla/5.0 (SymbianOS/9.1; U; [en]; Series60/3.0 NokiaE60/4.06.0) AppleWebKit/413 (KHTML, like Gecko) Safari/413", 2: "Mozilla/5.0 (SymbianOS/9.2; U; Series60/3.1 NokiaN95/10.0.018; Profile/MIDP-2.0 Configuration/CLDC-1.1) AppleWebKit/413 (KHTML, like Gecko) Safari/413", 3: "Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1/110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413", 4: "Mozilla/5.0 (SymbianOS/9.4; U; Series60/5.0 Nokia5800d-1/21.0.025; Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413", 5: "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/12.0.024; Profile/MIDP-2.1 Configuration/CLDC-1.1; en-us) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.12344", 6: "Mozilla/4.0 (compatible; MSIE 4.0; MSN 2.5; Windows 95)", 7: "Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)", 8: "Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; MSN 2.5; Windows 98)", 9: "Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; Windows 95)", 10: "Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; Windows 98)", 11: "Mozilla/4.0 (compatible; MSIE 4.01; Windows 95)", 12: "Mozilla/4.0 (compatible; MSIE 4.01; Windows 95; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.)", 13: "Mozilla/4.0 (compatible; MSIE 4.01; Windows 98)", 14: "Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; BIGLOBE)", 15: "Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; canoncopyer)", 16: "Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; Compaq)", 17: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95)", 18: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt)", 19: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt; i-CABLE)", 20: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt; ocnie5-1)", 21: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.)", 22: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98)", 23: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; CNETHomeBuild051099)", 24: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt)", 25: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt; ocnie5-1)", 26: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; wn_ie5_en_v1)", 27: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.; DigExt)", 28: "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT)", 29: "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)", 30: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt)", 31: "Mozilla/4.0 (compatible; MSIE 5.01; MSN 2.5; Windows 98)", 32: "Mozilla/4.0 (compatible; MSIE 5.01; Windows 95)", 33: "Mozilla/4.0 (compatible; MSIE 5.01; Windows 98)", 34: "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)", 35: "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; DigExt)", 36: "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; TUCOWS)", 37: "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT)", 38: "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; .NET CLR 1.1.4322)", 39: "Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC)", 40: "Mozilla/4.0 (compatible; MSIE 5.16; Mac_PowerPC)", 41: "Mozilla/4.0 (compatible; MSIE 5.17; Mac_PowerPC)", 42: "Mozilla/4.0 (compatible; MSIE 5.22; Mac_PowerPC)", 43: "Mozilla/4.0 (compatible; MSIE 5.23; Mac_PowerPC)", 44: "Mozilla/4.0 (compatible; MSIE 5.5; Windows 95)", 45: "Mozilla/4.0 (compatible; MSIE 5.5; Windows 98)", 46: "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)", 47: "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705)", 48: "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.1.4322)", 49: "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; by TSG)", 50: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)", 51: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", 52: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)", 53: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)", 54: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.40607)", 55: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.40607; .NET CLR 1.0.3705)", 56: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; NOKTURNAL KICKS ASS)", 57: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FDM;", 58: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.41115)", 59: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", 60: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1", 61: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)", 62: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)", 63: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1", 64: "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)", 65: "Mozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows NT 5.1; iebar; .NET CLR 1.0.3705)", 66: "Mozilla/4.0 (compatible; MSIE 6.0; Win32);", 67: "Mozilla/4.0 (compatible; MSIE 6.0; Win32); .NET CLR 1.0.3705)", 68: "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Hotbar 4.4.6.0)", 69: "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x 4.90)", 70: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0)", 71: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0; BVG", 72: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)", 73: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", 74: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322)", 75: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; FDM)", 76: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; {FF0C8E09-3C86-44CB-834A-B8CEEC80A1D7}; iOpus-I-M)", 77: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; i-Nav 3.0.1.0F; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", 78: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; MathPlayer 2.0; .NET CLR 1.1.4322)", 79: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Maxthon; .NET CLR 1.1.4322)", 80: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461)", 81: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;)", 82: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)", 83: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1);", 84: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705)", 85: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", 86: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", 87: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)", 88: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 2.0.40607)", 89: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Alexa Toolbar)", 90: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; BrowserBob)", 91: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; DFO-MPO Internet Explorer 6.0)", 92: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ENGINE; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT))", 93: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ESB{F40811EE-DF17-4BC9-8785-B362ABF34098}; .NET CLR 1.1.4322)", 94: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FDM)", 95: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FTDv3 Browser; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", 96: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; .NET CLR 1.1.4322; .NET CLR 2.0.40607)", 97: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; AtHome033)", 98: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; HCI0449; .NET CLR 1.0.3705)", 99: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; i-NavFourF; .NET CLR 1.1.4322)", 100: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon;", 101: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon; .NET CLR 1.1.4322)", 102: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2; .NET CLR 1.1.4322)", 103: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2; Maxthon; .NET CLR 1.1.4322)", 104: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461; FunWebProducts; .NET CLR 1.1.4322)", 105: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Woningstichting Den Helder; .NET CLR 1.0.3705)", 106: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322)", 107: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.41115)", 108: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; MyIE2; .NET CLR 1.1.4322)", 109: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; MyIE2; Maxthon; .NET CLR 1.1.4322)", 110: "Mozilla/4.0 (compatible; MSIE 6.0; Windows XP)", 111: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1)", 112: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 2.0.50727; .NET CLR 1.1.4322)", 113: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)", 114: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1)", 115: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", 116: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", 117: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)", 118: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; InfoPath.1)", 119: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", 120: "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648)", 121: "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; InfoPath.1", 122: "Mozilla/4.0 (compatible; GoogleToolbar 5.0.2124.2070; Windows 6.0; MSIE 8.0.6001.18241)", 123: "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; EasyBits Go v1.0; InfoPath.1; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)", 124: "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)", 125: "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; Sleipnir/2.9.8)", 126: "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)", 127: "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", 128: "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)", 129: "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)", 130: "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)", 131: "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)", 132: "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; ARM; Trident/6.0)", 133: "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13", 134: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.48 Safari/525.19", 135: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.33 Safari/530.5", 136: "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.38 Safari/532.0", 137: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.55 Safari/533.4", 138: "Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3", 139: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.55 Safari/534.3", 140: "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.43 Safari/534.7", 141: "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10 ChromePlus/1.5.2.0", 142: "Mozilla/5.0 (en-us) AppleWebKit/534.14 (KHTML, like Gecko; Google Wireless Transcoder) Chrome/9.0.597 Safari/534.14", 143: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.151 Safari/534.16", 144: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.71 Safari/534.24", 145: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Iron/11.0.700.2 Chrome/11.0.700.2 Safari/534.24", 146: "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.24", 147: "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30 ChromePlus/1.6.3.1", 148: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.112 Safari/534.30", 149: "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.107 Safari/535.1", 150: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) RockMelt/0.9.64.361 Chrome/13.0.782.218 Safari/535.1", 151: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1", 152: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.220 Safari/535.1", 153: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1", 154: "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1", 155: "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2", 156: "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.2 (KHTML, like Gecko) Ubuntu/10.04 Chromium/15.0.874.106 Chrome/15.0.874.106 Safari/535.2", 157: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.106 Safari/535.2", 158: "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7", 159: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7", 160: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30618; Lunascape 4.7.3)", 161: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; Lunascape 5.0 alpha2)", 162: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; Lunascape 5.0 alpha2)", 163: "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; Tablet PC 2.0; Lunascape 5.0 alpha2)", 164: "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MAGW; .NET4.0C; Lunascape 6.5.8.24780)", 165: "Mozilla/2.02 (Macintosh; I; PPC)", 166: "Mozilla/3.01 (Macintosh; I; PPC)", 167: "Mozilla/4.01 (Macintosh; I; PPC)", 168: "Mozilla/4.03 [en]C-IMS (Win95; I)", 169: "Mozilla/4.04 [en] (Win95; I ;Nav)", 170: "Mozilla/4.04 [en] (Win95; I)", 171: "Mozilla/4.04 [en] (WinNT; I ;Nav)", 172: "Mozilla/4.04 [en] (WinNT; I)", 173: "Mozilla/4.04 [en] (Macintosh; I; PPC Nav)", 174: "Mozilla/4.04 [en] (X11; I; SunOS 5.5 sun4u)", 175: "Mozilla/4.05 [en] (Win95; I)", 176: "Mozilla/4.05 (Macintosh; I; PPC)", 177: "Mozilla/4.06 [en] (Win98; I)", 178: "Mozilla/4.06 [en] (Macintosh; I; PPC)", 179: "Mozilla/4.08 (Macintosh; I; PPC)", 180: "Mozilla/4.5 [en] (Win95; I)", 181: "Mozilla/4.5 [en] (Win98; I)", 182: "Mozilla/4.5 [en] (WinNT; I)", 183: "Mozilla/4.5 (Macintosh; I; PPC)", 184: "Mozilla/4.51 [en] (Win95; I)", 185: "Mozilla/4.51 [en] (Win98; I)", 186: "Mozilla/4.51 [en] (WinNT; I)", 187: "Mozilla/4.51 [en] (X11; I; SunOS 5.8 sun4u)", 188: "Mozilla/4.6 [en] (Win95; I)", 189: "Mozilla/4.6 [en] (Win98; I)", 190: "Mozilla/4.6 [en] (WinNT; I)", 191: "Mozilla/4.6 [en] (WinNT; I)", 192: "Mozilla/4.7 [en] (WinNT; I)", 193: "Mozilla/4.7 [en] (Win95; I)", 194: "Mozilla/4.7 [en] (Win98; I)", 195: "Mozilla/4.7 [en] (WinNT; I)", 196: "Mozilla/4.7 [en] (WinNT; I)", 197: "Mozilla/4.7 [en] (WinNT; U)", 198: "Mozilla/4.7 [en] (Macintosh; I; PPC)", 199: "Mozilla/4.76 [en_jp] (X11; U; SunOS 5.8 sun4u)", 200: "Mozilla/4.76 [en] (X11; U; SunOS 5.8 sun4u)", 201: "Mozilla/4.78 [en] (X11; U; SunOS 5.9 sun4u)", 202: "Mozilla/4.8 [en] (X11; U; SunOS 5.7 sun4u)", 203: "Mozilla/5.0 (Windows; U; Win98; en-JP; m18) Gecko/20001108 Netscape6/6.0", 204: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-JP; m18) Gecko/20010131 Netscape6/6.01", 205: "Mozilla/5.0 (Windows; U; Win 9x 4.90; en-JP; rv:0.9.4) Gecko/20011128 Netscape6/6.2.1", 206: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:0.9.4.1) Gecko/20020508 Netscape6/6.2.3", 207: "Mozilla/5.0 (Macintosh; N; PPC; en-JP; macen-pub12) Gecko/20001108 Netscape6/6.0", 208: "Mozilla/5.0 (Macintosh; U; PPC; en-JP; rv:0.9.2) Gecko/20010726 Netscape6/6.1", 209: "Mozilla/5.0 (Macintosh; U; PPC; en-JP; rv:0.9.4) Gecko/20011022 Netscape6/6.2", 210: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US; rv:0.9.4.1) Gecko/20020315 Netscape6/6.2.2", 211: "Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.0.2) Gecko/20030208 Netscape/7.02", 212: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)", 213: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)", 214: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.2) Gecko/20040804 Netscape/7.2 (ax)", 215: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.2) Gecko/20040805 Netscape/7.2", 216: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-JP; rv:1.0.2) Gecko/20021120 Netscape/7.01", 217: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-JP; rv:1.4) Gecko/20030624 Netscape/7.1", 218: "Mozilla/5.0 (X11; U; SunOS sun4u; en-JP; rv:1.0.1) Gecko/20020921 Netscape/7.0", 219: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-JP; rv:1.5) Gecko/20031007", 220: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113", 221: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040616", 222: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.6) Gecko/20040113", 223: "Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.2.1) Gecko/20030225", 224: "Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.4.1) Gecko/20031030", 225: "Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.1) Gecko/20040805", 226: "Mozilla/5.0 (X11; U; SunOS sun4u; en-JP; rv:1.4) Gecko/20040414", 227: "Mozilla/5.0 (X11; U; Linux i686; rv:1.7.3) Gecko/20040913 Firefox/0.10", 228: "Mozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) Gecko/20040913 Firefox/0.10", 229: "Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20040913 Firefox/0.10", 230: "Mozilla/5.0 (X11; U; Linux i686; rv:1.7.3) Gecko/20041001 Firefox/0.10.1", 231: "Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1", 232: "Mozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) Gecko/20041001 Firefox/0.10.1", 233: "Mozilla/5.0 (Windows; U; Windows NT 5.2; rv:1.7.3) Gecko/20041001 Firefox/0.10.1", 234: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.6) Gecko/20040206 Firefox/0.8", 235: "Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.6) Gecko/20040206 Firefox/0.8", 236: "Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.6) Gecko/20040207 Firefox/0.8", 237: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.6) Gecko/20040206 Firefox/0.8", 238: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040614 Firefox/0.9", 239: "Mozilla/5.0 (X11; U; Linux i686; es-ES; rv:1.7) Gecko/20040708 Firefox/0.9", 240: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040707 Firefox/0.9.2", 241: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/0.9.2 StumbleUpon/1.994", 242: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040803 Firefox/0.9.3", 243: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3", 244: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3", 245: "Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3", 246: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3", 247: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20040803 Firefox/0.9.3", 248: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7) Gecko/20041013 Firefox/0.9.3 (Ubuntu)", 249: "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7) Gecko/20041013 Firefox/0.9.3 (Ubuntu)", 250: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0", 251: "Mozilla/5.0 (Windows; U; Win 9x 4.90; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0", 252: "Mozilla/5.0 (Windows; U; Win 9x 4.90; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0", 253: "Mozilla/5.0 (Windows; U; Win98; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0", 254: "Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0", 255: "Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0", 256: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.7.5) Gecko/20041107 Firefox/1.0", 257: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.7.5) Gecko/20041110 Firefox/1.0", 258: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0", 259: "Mozilla/5.0 (Windows; U; Windows NT 5.0; fr-FR; rv:1.7.5) Gecko/20041108 Firefox/1.0", 260: "Mozilla/5.0 (Windows; U; Windows NT 5.0; it-IT; rv:1.7.5) Gecko/20041110 Firefox/1.0", 261: "Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041107 Firefox/1.0", 262: "Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0", 263: "Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0", 264: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.7.5) Gecko/20041110 Firefox/1.0", 265: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0", 266: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 StumbleUpon/1.999", 267: "Mozilla/5.0 (Windows; U; Windows NT 5.1; es-ES; rv:1.7.5) Gecko/20041210 Firefox/1.0", 268: "Mozilla/5.0 (Windows; U; Windows NT 5.1; fr-FR; rv:1.7.5) Gecko/20041108 Firefox/1.0", 269: "Mozilla/5.0 (Windows; U; Windows NT 5.1; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0", 270: "Mozilla/5.0 (Windows; U; Windows NT 5.1; sv-SE; rv:1.7.5) Gecko/20041108 Firefox/1.0", 271: "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0", 272: "Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.8b) Gecko/20050212 Firefox/1.0+ (MOOX M3)", 273: "Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0", 274: "Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8b) Gecko/20050118 Firefox/1.0+", 275: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8b) Gecko/20050118 Firefox/1.0+", 276: "Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.5) Gecko/20050103 Firefox/1.0", 277: "Mozilla/5.0 (X11; U; Linux i386; en-US; rv:1.7.5) Gecko/20041109 Firefox/1.0", 278: "Mozilla/5.0 (X11; U; Linux i686; de-DE; rv:1.7.5) Gecko/20041128 Firefox/1.0 (Debian package 1.0-4)", 279: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0", 280: "Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0", 281: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041111 Firefox/1.0", 282: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041111 Firefox/1.0 (Debian package 1.0-2)", 283: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041116 Firefox/1.0 (Ubuntu) (Ubuntu package 1.0-2ubuntu3)", 284: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041119 Firefox/1.0", 285: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041123 Firefox/1.0", 286: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041128 Firefox/1.0 (Debian package 1.0-4)", 287: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041130 Firefox/1.0", 288: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041214 Firefox/1.0 StumbleUpon/1.999", 289: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041219 Firefox/1.0 (Debian package 1.0+dfsg.1-1)", 290: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20050110 Firefox/1.0 (Debian package 1.0+dfsg.1-2)", 291: "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0", 292: "Mozilla/5.0 (X11; U; Linux i686; nl-NL; rv:1.7.5) Gecko/20050221 Firefox/1.0 (Ubuntu) (Ubuntu package 1.0+dfsg.1-6ubuntu1)", 293: "Mozilla/5.0 (X11; U; Linux i686; rv:1.8b) Gecko/20050124 Firefox/1.0+", 294: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.6) Gecko/20050306 Firefox/1.0.1 (Debian package 1.0.1-2)", 295: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6) Gecko/20050223 Firefox/1.0.1", 296: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6) Gecko/20050225 Firefox/1.0.1", 297: "Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.6) Gecko/20050226 Firefox/1.0.1", 298: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-JP-mac; rv:1.8) Gecko/20051111 Firefox/1.5", 299: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8) Gecko/20051111 Firefox/1.5", 300: "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8) Gecko/20051111 Firefox/1.5", 301: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.0.1) Gecko/20060111 Firefox/1.5.0.1", 302: "Mozilla/5.0 (X11; U; Linux i686; en; rv:1.8.0.2) Gecko/20060308 Firefox/1.5.0.2", 303: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3", 304: "Mozilla/5.0 (X11; U; Linux i686; en; rv:1.8.0.4) Gecko/20060508 Firefox/1.5.0.4", 305: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20", 306: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.12) Gecko/20080201 Firefox/2.0.0.12", 307: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-JP-mac; rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20", 308: "Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6", 309: "Mozilla/5.0 (Windows; U; Windows NT 6.0; en; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6 (.NET CLR 3.5.30729)", 310: "Mozilla/5.0 (Windows; U; Windows NT 6.0; en; rv:1.9.0.17) Gecko/2009122116 Firefox/3.0.17 GTB6 (.NET CLR 3.5.30729)", 311: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-JP-mac; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6", 312: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-JP-mac; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6 GTB5", 313: "Mozilla/5.0 (Windows NT 6.1; rv:2.0) Gecko/20100101 Firefox/4.0", 314: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:2.0) Gecko/20100101 Firefox/4.0", 315: "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0", 316: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:5.0.1) Gecko/20100101 Firefox/5.0.1", 317: "Mozilla/5.0 (Windows NT 5.1; rv:6.0) Gecko/20100101 Firefox/6.0", 318: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:6.0.2) Gecko/20100101 Firefox/6.0.2", 319: "Mozilla/5.0 (Windows NT 6.0; rv:7.0.1) Gecko/20100101 Firefox/7.0.1", 320: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:7.0.1) Gecko/20100101 Firefox/7.0.1", 321: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:8.0.1) Gecko/20100101 Firefox/8.0.1", 322: "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1", 323: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:9.0.1) Gecko/20100101 Firefox/9.0.1", 324: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.6", 325: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.7", 326: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/85.8.2 (KHTML, like Gecko) Safari/85.8", 327: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/85.8.2 (KHTML, like Gecko) Safari/85.8.1", 328: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/85.8.5 (KHTML, like Gecko) Safari/85.8", 329: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/103u (KHTML, like Gecko) Safari/100.1", 330: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/124 (KHTML, like Gecko) Safari/125.1", 331: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/125.2 (KHTML, like Gecko) Safari/125.8", 332: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/125.4 (KHTML, like Gecko) Safari/125.9", 333: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.11", 334: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12", 335: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12", 336: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; nl-nl) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12", 337: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/125.5.6 (KHTML, like Gecko) Safari/125.12", 338: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.1 (KHTML, like Gecko) Safari/312", 339: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.1.1 (KHTML, like Gecko) Safari/312", 340: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.5 (KHTML, like Gecko) Safari/312.3", 341: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.5", 342: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.6", 343: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412 (KHTML, like Gecko) Safari/412", 344: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.6.2 (KHTML, like Gecko) Safari/125.11", 345: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.6.2 (KHTML, like Gecko) Safari/412.2.2", 346: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.7 (KHTML, like Gecko) Safari/412.5", 347: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/416.11 (KHTML, like Gecko) Safari/416.12", 348: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.2", 349: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-jp) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.3", 350: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/522.11.1 (KHTML, like Gecko) Version/3.0.3 Safari/522.12.1", 351: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-jp) AppleWebKit/523.10.3 (KHTML, like Gecko) Version/3.0.4 Safari/523.10", 352: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.10.6 (KHTML, like Gecko) Version/3.0.4 Safari/523.10.6", 353: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.12 (KHTML, like Gecko) Version/3.0.4 Safari/523.12", 354: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.12.2 (KHTML, like Gecko) Version/3.0.4 Safari/523.12.2", 355: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en-jp) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13", 356: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_3; en-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.1 Safari/525.20", 357: "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.22", 358: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; en-jp) AppleWebKit/525.26.2 (KHTML, like Gecko) Version/3.2 Safari/525.26.12", 359: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-jp) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1", 360: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-jp) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16", 361: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-jp) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10", 362: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-jp) AppleWebKit/531.21.11 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10", 363: "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-jp) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16", 364: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.52.7 (KHTML, like Gecko) Version/5.1.2 Safari/534.52.7", 365: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2", 366: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8) AppleWebKit/536.25 (KHTML, like Gecko) Version/6.0 Safari/536.25", 367: "Mozilla/4.0 (Windows 95;US) Opera 3.62 [en]", 368: "Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC) Opera 6.0 [en]", 369: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.01 [en]", 370: "Mozilla/4.0 (compatible; MSIE 5.0; Windows ME) Opera 6.03 [en]", 371: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.05 [en]", 372: "Mozilla/4.0 (compatible; MSIE 5.0; Windows XP) Opera 6.06 [en]", 373: "Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.06 [en]", 374: "Opera 7.11", 375: "Opera/7.23 (Windows NT 5.0; U) [en]", 376: "Opera/7.52 (Windows NT 5.1; U) [en]", 377: "Opera/7.53 (Windows NT 5.0; U) [en]", 378: "Opera/7.54 (Windows NT 5.0; U) [en]", 379: "Opera/7.54 (Windows NT 5.1; U) [en]", 380: "Opera/7.54 (Windows NT 5.1; U)", 381: "Opera/7.54 (X11; Linux i686; U) [en]", 382: "Opera/7.54 (X11; Linux i686; U) [sv]", 383: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.11 [en]", 384: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.21 [pt-BR]", 385: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.22 [en]", 386: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.23 [en]", 387: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.23 [en]", 388: "Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686) Opera 7.23 [en]", 389: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.23 [fr]", 390: "Mozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC) Opera 7.50 [en]", 391: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.53 [en]", 392: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.53 [en]", 393: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.54 [en]", 394: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.54 [en]", 395: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.54 [en]", 396: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.54u1 [en]", 397: "Opera/7.54 (Windows 98; U) [en]", 398: "Mozilla/4.78 (Windows NT 5.1; U) Opera 7.23 [en]", 399: "Mozilla/5.0 (Windows NT 5.0; U) Opera 7.54 [en]", 400: "Mozilla/4.0 (compatible; MSIE 6.0; X11; OpenBSD i386) Opera 7.54 [en]", 401: "Opera/8.0 (X11; Linux i686; U; en)", 402: "Opera/8.01 (Windows ME; U; en)", 403: "Opera/8.01 (Windows NT 5.1; U; en)", 404: "Mozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC Mac OS X; en) Opera 8.01", 405: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.01", 406: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; tr) Opera 8.02", 407: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.02", 408: "Opera/8.5 (Windows NT 5.0; U; en)", 409: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.5", 410: "Opera/8.51 (Windows NT 5.1; U; en)", 411: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.51", 412: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; de) Opera 8.52", 413: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; pl) Opera 8.52", 414: "Mozilla/5.0 (Windows NT 5.1; U; en) Opera 8.52", 415: "Opera/8.53 (Windows NT 5.1; U; en)", 416: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.53", 417: "Opera/8.54 (Windows NT 5.1; U; en)", 418: "Opera/8.54 (Windows NT 5.0; U; en)", 419: "Mozilla/5.0 (X11; Linux i686; U; cs) Opera 8.54", 420: "Mozilla/4.0 (compatible; MSIE 6.0; KDDI-SA39) Opera 8.60 [en]", 421: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 9.00", 422: "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.00", 423: "Opera/9.00 (Windows NT 5.1; U; en)", 424: "Opera/9.0 (Windows NT 5.1; U; en)", 425: "Opera/9.00 (Macintosh; PPC Mac OS X; U; en)", 426: "Opera/9.02 (Macintosh; PPC Mac OS X; U; en)", 427: "Opera/9.02 (Windows NT 5.1; U; zh-tw)", 428: "Opera/9.10 (Windows NT 6.0; U; en)", 429: "Opera/9.21 (Windows NT 6.0; U; en)", 430: "Opera/9.22 (Windows NT 5.1; U; en)", 431: "Opera/9.23 (Windows NT 5.1; U; en)", 432: "Opera/9.23 (Windows ME; U; en)", 433: "Opera/9.26 (Windows NT 5.1; U; en)", 434: "Opera/9.51 (Windows NT 5.1; U; en)", 435: "Opera/9.52 (Macintosh; Intel Mac OS X; U; en)", 436: "Opera/9.52 (Windows NT 5.1; U; en)", 437: "Opera/9.60 (Macintosh; Intel Mac OS X; U; en) Presto/2.1.1", 438: "Opera/9.60 (Windows NT 5.1; U; en) Presto/2.1.1", 439: "Opera/9.61 (Windows NT 5.1; U; en) Presto/2.1.1", 440: "Opera/9.62 (Windows NT 5.1; U; en) Presto/2.1.1", 441: "Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686; en) Opera 10.10", 442: "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.9.168 Version/11.50", 443: "Opera/9.80 (Windows NT 6.1; U; en) Presto/2.10.229 Version/11.60", 444: "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.10.289 Version/12.00", } // this *.txt maybe not found if you exec binary, so we just fill several ua temp, err := util.ReadFromFile(util.CurDir() + "/config/ua.txt") if err == nil { uas := strings.Split(string(temp), "\n") for i, ua := range uas { Ua[i] = strings.TrimSpace(strings.Replace(ua, "\r", "", -1)) } } } // RandomUa back random User-Agent func RandomUa() string { length := len(Ua) if length == 0 { return "" } return Ua[rand.Intn(length-1)] } ================================================ FILE: miner/util.go ================================================ package miner import ( "errors" "fmt" "net/http" "strings" "github.com/hunterhug/marmot/util" ) // Wait some second func Wait(waitTime int) { if waitTime <= 0 { return } else { Logger.Debugf("[GoWorker] Wait %d Second.", waitTime) util.Sleep(waitTime) } } // CopyM Header map[string][]string ,can use to copy a http header, so that they are not effect each other func CopyM(h http.Header) http.Header { if h == nil || len(h) == 0 { return h } h2 := make(http.Header, len(h)) for k, vv := range h { vv2 := make([]string, len(vv)) copy(vv2, vv) h2[k] = vv2 } return h2 } // TooSortSizes if a file size smaller than sizes(KB), it will be thrown an error func TooSortSizes(data []byte, sizes float64) error { if float64(len(data))/1000 < sizes { return errors.New(fmt.Sprintf("FileSize:%d bytes,%d kb < %f kb dead too sort", len(data), len(data)/1000, sizes)) } return nil } // OutputMaps Just debug a map func OutputMaps(uuid string, info string, args map[string][]string) { s := "\n" for k, v := range args { s = s + fmt.Sprintf("%-25s | %-6s\n", k, strings.Join(v, " |||| ")) } Logger.Debugf("%s %s-%s", uuid, info, s) } ================================================ FILE: miner/worker.go ================================================ package miner import ( "bytes" "errors" "io" "net/http" "net/url" "os" "strings" "mime/multipart" "github.com/hunterhug/marmot/util" ) // NewWorkerByClient New Worker by Your Client func NewWorkerByClient(client *http.Client) *Worker { worker := new(Worker) worker.Request = newRequest() worker.Header = http.Header{} worker.Response = new(Response) worker.Client = client return worker } // NewAPI New API Worker, No Cookie Keep, share the same http client func NewAPI() *Worker { return NewWorkerByClient(NoCookieClient) } // NewWorker New a worker, if ipString is a proxy address, New a proxy client. evey time gen a new http client! // Proxy address such as: // // http://[user]:[password@]ip:port, [] stand it can choose or not. case: socks5://127.0.0.1:1080 func NewWorker(proxyIpString interface{}) (*Worker, error) { if proxyIpString != nil { client, err := NewProxyClient(strings.ToLower(proxyIpString.(string))) if err != nil { return nil, err } worker := NewWorkerByClient(client) worker.Ip = proxyIpString.(string) return worker, err } else { client := NewClient() worker := NewWorkerByClient(client) worker.Ip = "localhost" return worker, nil } } // NewWorkerWithProxy Alias func func NewWorkerWithProxy(proxyIpString interface{}) (*Worker, error) { return NewWorker(proxyIpString) } // NewWorkerWithNoProxy Alias func func NewWorkerWithNoProxy() *Worker { w, _ := NewWorker(nil) return w } // New Alias Name for NewWorker func New(ipString interface{}) (*Worker, error) { return NewWorker(ipString) } // fast new request func newRequest() *Request { req := new(Request) req.Data = url.Values{} req.BData = []byte{} return req } // Clone Should Clone a new worker if you want to use repeat func (worker *Worker) Clone() *Worker { cloneWorker := NewWorkerByClient(worker.Client) cloneWorker.Ip = worker.Ip return cloneWorker } // Go Auto decide which method, Default Get. func (worker *Worker) Go() (body []byte, e error) { switch strings.ToUpper(worker.Method) { case POST: return worker.Post() case POSTJSON: return worker.PostJSON() case POSTXML: return worker.PostXML() case POSTFILE: return worker.PostFILE() case PUT: return worker.Put() case PUTJSON: return worker.PutJSON() case PUTXML: return worker.PutXML() case PUTFILE: return worker.PutFILE() case DELETE: return worker.Delete() case OTHER: return []byte(""), errors.New("please use method OtherGo(method, contentType string) or OtherGoBinary(method, contentType string)") default: return worker.Get() } } func (worker *Worker) GoByMethod(method string) (body []byte, e error) { return worker.SetMethod(method).Go() } // ToString This make effect only your worker exec serial! Attention! // Change Your Raw data To string func (worker *Worker) ToString() string { if worker.Raw == nil { return "" } return string(worker.Raw) } // JsonToString This make effect only your worker exec serial! Attention! // Change Your JSON' like Raw data to string func (worker *Worker) JsonToString() (string, error) { if worker.Raw == nil { return "", nil } temp, err := util.JsonBack(worker.Raw) if err != nil { return "", err } return string(temp), nil } // Main method I make! func (worker *Worker) sent(method, contentType string, binary bool) (body []byte, e error) { // Lock it for save worker.mux.Lock() defer worker.mux.Unlock() // Before Action, we can change or add something before Go() if worker.BeforeAction != nil { worker.BeforeAction(worker.Ctx, worker) } // Wait if must if worker.Wait > 0 { Wait(worker.Wait) } // For debug uuid := "[GoWorker]" Logger.Debugf("%s %s %s", uuid, method, worker.Url) // New a Request var request *http.Request var err error // If binary value is true and BData is not empty // suit for POSTJSON(), POSTFILE() if len(worker.BData) != 0 && binary { pr := bytes.NewReader(worker.BData) request, err = http.NewRequest(method, worker.Url, pr) } else if len(worker.Data) != 0 { // such POST() from table form pr := strings.NewReader(worker.Data.Encode()) request, err = http.NewRequest(method, worker.Url, pr) } else { request, err = http.NewRequest(method, worker.Url, nil) } if err != nil { return nil, err } // Close avoid EOF // For client requests, setting this field prevents re-use of // TCP connections between requests to the same hosts, as if // Transport.DisableKeepAlives were set. // maybe you want long connection //request.Close = true // Clone Header, I add some HTTP header! request.Header = CloneHeader(worker.Header) // In fact content type must not empty if contentType != "" { request.Header.Set("Content-Type", contentType) } worker.Request.Request = request // Debug for RequestHeader OutputMaps(uuid, "Request Header", request.Header) // Tolerate abnormal way to create a Worker if worker.Client == nil { worker.Client = Client } // Do it response, err := worker.Client.Do(request) if err != nil { return nil, err } // Close it attention response may be nil if response != nil { //response.Close = true defer response.Body.Close() } // Debug OutputMaps(uuid, "Response Header", response.Header) Logger.Debugf("%s %s %s", uuid, response.Proto, response.Status) // Read output body, e = io.ReadAll(response.Body) worker.Raw = body worker.ResponseStatusCode = response.StatusCode worker.Response.Response = response // After action if worker.AfterAction != nil { worker.AfterAction(worker.Ctx, worker) } return } func (worker *Worker) sentByFile(method, contentType string, binary bool, filePath string) (e error) { // Lock it for save worker.mux.Lock() defer worker.mux.Unlock() // Before Action, we can change or add something before Go() if worker.BeforeAction != nil { worker.BeforeAction(worker.Ctx, worker) } // Wait if must if worker.Wait > 0 { Wait(worker.Wait) } // For debug uuid := "[GoWorker]" Logger.Debugf("%s %s %s", uuid, method, worker.Url) // New a Request var request *http.Request var err error // If binary value is true and BData is not empty // suit for POSTJSON(), POSTFILE() if len(worker.BData) != 0 && binary { pr := bytes.NewReader(worker.BData) request, err = http.NewRequest(method, worker.Url, pr) } else if len(worker.Data) != 0 { // such POST() from table form pr := strings.NewReader(worker.Data.Encode()) request, err = http.NewRequest(method, worker.Url, pr) } else { request, err = http.NewRequest(method, worker.Url, nil) } if err != nil { return err } // Close avoid EOF // For client requests, setting this field prevents re-use of // TCP connections between requests to the same hosts, as if // Transport.DisableKeepAlives were set. // maybe you want long connection //request.Close = true // Clone Header, I add some HTTP header! request.Header = CloneHeader(worker.Header) // In fact content type must not empty if contentType != "" { request.Header.Set("Content-Type", contentType) } worker.Request.Request = request // Debug for RequestHeader OutputMaps(uuid, "Request Header", request.Header) // Tolerate abnormal way to create a Worker if worker.Client == nil { worker.Client = Client } // Do it response, err := worker.Client.Do(request) if err != nil { return err } // Close it attention response may be nil if response != nil { //response.Close = true defer response.Body.Close() } // Debug OutputMaps(uuid, "Response Header", response.Header) Logger.Debugf("%s %s %s", uuid, response.Proto, response.Status) totalSize := response.ContentLength Logger.Debugf("Download Size: %.2f MB\n", float64(totalSize)/(1024*1024)) file, err := os.Create(filePath) if err != nil { return err } defer file.Close() var downloaded int64 sizeMB := worker.DownloadFileReadMB * 1024 * 1024 if sizeMB <= 0 { sizeMB = 5 * 1024 * 1024 // Default 5MB } buf := make([]byte, sizeMB) for { n, err := response.Body.Read(buf) if n > 0 { if _, writeErr := file.Write(buf[:n]); writeErr != nil { os.Remove(filePath) return writeErr } downloaded += int64(n) if totalSize > 0 { percent := float64(downloaded) / float64(totalSize) * 100 Logger.Debugf("\rDownload: %.1f%% (%.2f/%.2f MB)", percent, float64(downloaded)/(1024*1024), float64(totalSize)/(1024*1024)) } } if err == io.EOF { Logger.Debugf("\nDownload Done") break } if err != nil { os.Remove(filePath) return err } } worker.ResponseStatusCode = response.StatusCode worker.Response.Response = response // After action if worker.AfterAction != nil { worker.AfterAction(worker.Ctx, worker) } return } // Get method func (worker *Worker) Get() (body []byte, e error) { worker.Clear() return worker.sent(GET, "", false) } func (worker *Worker) Delete() (body []byte, e error) { worker.Clear() return worker.sent(DELETE, "", false) } // Post Almost include bellow: /* "application/x-www-form-urlencoded" "application/json" "text/xml" "multipart/form-data" */ func (worker *Worker) Post() (body []byte, e error) { return worker.sent(POST, HTTPFORMContentType, false) } func (worker *Worker) PostJSON() (body []byte, e error) { return worker.sent(POST, HTTPJSONContentType, true) } func (worker *Worker) PostXML() (body []byte, e error) { return worker.sent(POST, HTTPXMLContentType, true) } func (worker *Worker) PostFILE() (body []byte, e error) { return worker.sentFile(POST) } var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") func escapeQuotes(s string) string { return quoteEscaper.Replace(s) } func (worker *Worker) sentFile(method string) ([]byte, error) { if worker.FileName == "" || worker.FileFormName == "" { return nil, errors.New("fileName or fileFormName must not empty") } if len(worker.BData) == 0 { return nil, errors.New("BData must not empty") } bodyBuf := &bytes.Buffer{} bodyWriter := multipart.NewWriter(bodyBuf) fileWriter, err := bodyWriter.CreateFormFile(worker.FileFormName, worker.FileName) if err != nil { return nil, err } _, err = fileWriter.Write(worker.BData) if err != nil { return nil, err } for k, v := range worker.Data { for _, vv := range v { if vv != "" { bodyWriter.WriteField(k, vv) } } } contentType := bodyWriter.FormDataContentType() err = bodyWriter.Close() if err != nil { return nil, err } worker.SetBData(bodyBuf.Bytes()) return worker.sent(method, contentType, true) } func (worker *Worker) Put() (body []byte, e error) { return worker.sent(PUT, HTTPFORMContentType, false) } func (worker *Worker) PutJSON() (body []byte, e error) { return worker.sent(PUT, HTTPJSONContentType, true) } func (worker *Worker) PutXML() (body []byte, e error) { return worker.sent(PUT, HTTPXMLContentType, true) } func (worker *Worker) PutFILE() (body []byte, e error) { return worker.sentFile(PUT) } /* OtherGo Method Method = "OPTIONS" ; Section 9.2 | "GET" ; Section 9.3 | "HEAD" ; Section 9.4 | "POST" ; Section 9.5 | "PUT" ; Section 9.6 | "DELETE" ; Section 9.7 | "TRACE" ; Section 9.8 | "CONNECT" ; Section 9.9 | extension-method extension-method = token token = 1* Content Type "application/x-www-form-urlencoded" "application/json" "text/xml" "multipart/form-data" */ func (worker *Worker) OtherGo(method, contentType string) (body []byte, e error) { return worker.sent(method, contentType, false) } func (worker *Worker) OtherGoBinary(method, contentType string) (body []byte, e error) { return worker.sent(method, contentType, true) } func (worker *Worker) OtherDownload(method, contentType string, filePath string) (e error) { return worker.sentByFile(method, contentType, true, filePath) } ================================================ FILE: util/crypto.go ================================================ package util import ( "crypto/hmac" "crypto/md5" "crypto/sha256" "encoding/base64" "encoding/hex" "fmt" "io" "net/url" "strings" ) // ComputeHmac256 HMAC with the SHA256 func ComputeHmac256(message string, secret string) string { key := []byte(secret) h := hmac.New(sha256.New, key) h.Write([]byte(message)) return base64.StdEncoding.EncodeToString(h.Sum(nil)) } // StrToMd5 create md5 string func StrToMd5(s string) string { h := md5.New() h.Write([]byte(s)) rs := hex.EncodeToString(h.Sum(nil)) return rs } func Md5(str string) string { return StrToMd5(str) } func Base64E(body string) string { str := []byte(body) data := base64.StdEncoding.EncodeToString(str) return data } func Base64D(body string) string { data, err := base64.StdEncoding.DecodeString(body) if err != nil { return "" } s := fmt.Sprintf("%q", data) s = strings.Replace(s, "\"", "", -1) return s } func UrlE(s string) string { return url.QueryEscape(s) } func UrlD(s string) string { s, e := url.QueryUnescape(s) if e != nil { return e.Error() } else { return s } } func Md5FS(src io.Reader) string { h := md5.New() if err := CopyFF(src, h); err != nil { fmt.Println(err.Error()) return "" } return fmt.Sprintf("%x", h.Sum([]byte("hunterhug"))) } ================================================ FILE: util/file.go ================================================ package util import ( "errors" "io" "os" "os/exec" "path/filepath" "runtime" "strings" ) func CurDir() string { _, filename, _, _ := runtime.Caller(1) return filepath.Dir(filename) } func GetBinaryCurrentPath() (string, error) { file, err := exec.LookPath(os.Args[0]) if err != nil { return "", err } path, err := filepath.Abs(file) if err != nil { return "", err } if strings.Contains(path, "command-line-arguments") { return GetCurrentPath() } i := strings.LastIndex(path, "/") if i < 0 { i = strings.LastIndex(path, "\\") } if i < 0 { return "", errors.New(`error: Can't find "/" or "\"`) } return path[0 : i+1], nil } func GetCurrentPath() (string, error) { return os.Getwd() } func SaveToFile(filePath string, content []byte) error { err := os.WriteFile(filePath, content, 0777) return err } func ReadFromFile(filepath string) ([]byte, error) { return os.ReadFile(filepath) } func GetFilenameInfo(filepath string) (os.FileInfo, error) { info, err := os.Stat(filepath) return info, err } func Rename(oldFilename string, newFilename string) error { return os.Rename(oldFilename, newFilename) } func MakeDir(fileDir string) error { return os.MkdirAll(fileDir, 0777) } func MakeDirByFile(filepath string) error { temp := strings.Split(filepath, "/") if len(temp) <= 2 { return errors.New("please input complete file name like ./dir/filename or /home/dir/filename") } dirPath := strings.Join(temp[0:len(temp)-1], "/") return MakeDir(dirPath) } func FileExist(filename string) bool { fi, err := os.Stat(filename) if err != nil { return false } if fi.IsDir() { return false } else { return true } } func WalkDir(dirPth, suffix string) (files []string, err error) { files = make([]string, 0, 30) suffix = strings.ToUpper(suffix) err = filepath.Walk(dirPth, func(filename string, fi os.FileInfo, err error) error { if fi.IsDir() { return nil } if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) { files = append(files, filename) } return nil }) return files, err } func ListDir(dirPth string, suffix string) (files []string, err error) { files = make([]string, 0, 10) dir, err := os.ReadDir(dirPth) if err != nil { return nil, err } suffix = strings.ToUpper(suffix) for _, fi := range dir { if fi.IsDir() { continue } if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) { files = append(files, dirPth+"/"+fi.Name()) } } return files, nil } func ListDirOnlyName(dirPth string, suffix string) (files []string, err error) { files = make([]string, 0, 10) dir, err := os.ReadDir(dirPth) if err != nil { return nil, err } suffix = strings.ToUpper(suffix) for _, fi := range dir { if fi.IsDir() { continue } if strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) { files = append(files, fi.Name()) } } return files, nil } func HasFile(s string) bool { f, err := os.Open(s) if err != nil && os.IsNotExist(err) { return false } f.Close() return true } func CopyFF(src io.Reader, dst io.Writer) error { _, err := io.Copy(dst, src) return err } func CopyFS(src io.Reader, dst string) error { f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) if err != nil { return err } defer f.Close() _, err = io.Copy(f, src) return err } func IsFile(filepath string) bool { info, err := os.Stat(filepath) if err != nil { return false } else { if info.IsDir() { return false } else { return true } } } func IsDir(filepath string) bool { info, err := os.Stat(filepath) if err != nil { return false } else { if info.IsDir() { return true } else { return false } } } func SizeofDir(dirPth string) int { if IsDir(dirPth) { files, _ := os.ReadDir(dirPth) return len(files) } return 0 } func GetFileSuffix(f string) string { fa := strings.Split(f, ".") if len(fa) == 0 { return "" } else { return fa[len(fa)-1] } } func ValidFileName(filename string) string { patterns := []string{ " ", "#01#", "\\", "#02#", "/", "#03#", ":", "#04#", "\"", "#05#", "?", "#06#", "<", "#07#", ">", "#08#", "|", "#09#", } r := strings.NewReplacer(patterns...) dudu := r.Replace(filename) return dudu } func ValidBackFileName(filename string) string { patterns := []string{ "#01#", " ", "#02#", "\\", "#03#", "/", "#04#", ":", "#05#", "\"", "#06#", "?", "#07#", "<", "#08#", ">", "#09#", "|", } r := strings.NewReplacer(patterns...) dudu := r.Replace(filename) return dudu } ================================================ FILE: util/io.go ================================================ package util import ( "fmt" "strings" ) func Input(say, defaults string) string { fmt.Println(say) var str string _, _ = fmt.Scanln(&str) if strings.TrimSpace(str) == "" { if strings.TrimSpace(defaults) != "" { return defaults } else { fmt.Println("can not empty") return Input(say, defaults) } } return str } ================================================ FILE: util/json.go ================================================ package util import ( "encoding/json" ) func JsonBack(s []byte) ([]byte, error) { temp := new(interface{}) err := json.Unmarshal(s, temp) if err != nil { return nil, err } result, err := json.Marshal(temp) return result, err } ================================================ FILE: util/time.go ================================================ package util import ( "time" ) func Sleep(waitTime int) { time.Sleep(time.Duration(waitTime) * time.Second) } func Second(times int) time.Duration { return time.Duration(times) * time.Second } func TodayString(level int) string { formats := "20060102150405" switch level { case 1: formats = "2006" case 2: formats = "200601" case 3: formats = "20060102" case 4: formats = "2006010215" case 5: formats = "200601021504" default: } return time.Now().Format(formats) } ================================================ FILE: util/util.go ================================================ package util import ( "errors" "strconv" "strings" ) func SI(s string) (i int, e error) { i, e = strconv.Atoi(s) return } func IS(i int) string { return strconv.Itoa(i) } func ToLower(s string) string { return strings.ToLower(s) } func DivideStringList(files []string, num int) (map[int][]string, error) { length := len(files) split := map[int][]string{} if num <= 0 { return split, errors.New("num must not negative") } if num > length { num = length } process := length / num for i := 0; i < num; i++ { split[i] = append(split[i], files[i*process:(i+1)*process]...) } remain := files[num*process:] for i := 0; i < len(remain); i++ { split[i%num] = append(split[i%num], remain[i]) } return split, nil } func InArray(sa []string, a string) bool { for _, v := range sa { if a == v { return true } } return false } func Substr(str string, start, length int) string { rs := []rune(str) rl := len(rs) end := 0 if start < 0 { start = rl - 1 + start } end = start + length if start > end { start, end = end, start } if start < 0 { start = 0 } if start > rl { start = rl } if end < 0 { end = 0 } if end > rl { end = rl } return string(rs[start:end]) }