[ { "path": ".gitignore", "content": ".idea/\n/vendor\n/picture\n.DS_Store\ndoc.go.xxxxxx" }, { "path": "LICENSE", "content": "Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"{}\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright {yyyy} {name of copyright owner}\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License." }, { "path": "Makefile", "content": ".PHONY: clean fmt\n\ndefault: fmt\n\nclean:\n\trm -rf bin/*\n\nfmt:\n\tgo fmt ./" }, { "path": "README.md", "content": "# Project: Marmot | HTTP Download\n\n[![GitHub forks](https://img.shields.io/github/forks/hunterhug/marmot.svg?style=social&label=Forks)](https://github.com/hunterhug/marmot/network)\n[![GitHub stars](https://img.shields.io/github/stars/hunterhug/marmot.svg?style=social&label=Stars)](https://github.com/hunterhug/marmot/stargazers)\n[![GitHub last commit](https://img.shields.io/github/last-commit/hunterhug/marmot.svg)](https://github.com/hunterhug/marmot)\n[![Go Report Card](https://goreportcard.com/badge/github.com/hunterhug/marmot)](https://goreportcard.com/report/github.com/hunterhug/marmot)\n[![GitHub issues](https://img.shields.io/github/issues/hunterhug/marmot.svg)](https://github.com/hunterhug/marmot/issues)\n[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)\n\n[中文介绍](/README_ZH.md)\n\nHTTP Download Helper, Supports Many Features such as Cookie Persistence, HTTP(S) and SOCKS5 Proxy....\n\n![Marmot](logo.png)\n\n## 1. Introduction\n\nWorld-Wide-Web robot, also known as spiders and crawlers. The principle is to falsify network data by constructing appointed HTTP protocol data packet, then request resource to the specified host, goal is to access the data returned. \nThere are a large number of web information, human's hand movement such as `copy-paste data` from web page is `time-consuming` and `laborious`, thus inspired the data acquisition industry.\n\nBatch access to public network data does not break the law, but because there is no difference, no control, very violent means will lead to other services is not stable, therefore, most of the resources provider will filtering some data packets(falsify), \nin this context, batch small data acquisition has become a problem. Integrated with various requirements, such as various API development, automated software testing(all this have similar technical principle). So this project come into the world(very simple).\n\nThe `Marmot` is very easy to understand, just like Python's library `requests`(Not yet Smile~ --| ). By enhancing native Golang HTTP library, help you deal with some trivial logic (such as collecting information, checking parameters), and add some fault-tolerant mechanisms (such as add lock, close time flow, ensure the high concurrent run without accident).\nIt provides a human friendly API interface, you can reuse it often. Very convenient to support `Cookie Persistence`, `Crawler Proxy Settings`, as well as others general settings, such as `HTTP request header settings, timeout/pause settings, data upload/post settings`.\nIt supports all the HTTP methods `POST/PUT/GET/DELETE/...` and has built-in spider pool and browser UA pool, easy to develop UA+Cookie persistence distributed spider.\n\nThe library is simple and practical, just a few lines of code to replace the previous `Spaghetti code`, has been applied in some large projects.\n\nThe main uses: `WeChat development`/ `API docking` / `Automated test` / `Rush Ticket Scripting` / `Vote Plug-in` / `Data Crawling`\n\nNow We support Default Worker, You can easy use:\n\n`lesson1.go`\n\n```go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"github.com/hunterhug/marmot/miner\"\n)\n\nfunc main() {\n\tminer.SetLogLevel(miner.DEBUG)\n\n\t// Use Default Worker, You can Also New One:\n\t//worker, _ := miner.New(nil)\n\t//worker = miner.NewWorkerWithNoProxy()\n\t//worker = miner.NewAPI()\n\t//worker, _ = miner.NewWorkerWithProxy(\"socks5://127.0.0.1:1080\")\n\tworker := miner.Clone()\n\t_, err := worker.SetUrl(\"https://www.bing.com\").Go()\n\tif err != nil {\n\t\tfmt.Println(err.Error())\n\t} else {\n\t\tfmt.Println(worker.ToString())\n\t}\n}\n```\n\nSee the [example](example) dir. such as lesson or practice.\n\n## 2. How To Use\n\nYou can get it by:\n\n```\ngo get -v github.com/hunterhug/marmot/miner\n```\n\n### 2.1 The First Step\n\nThere are four kinds of worker:\n\n1. `worker, err := miner.NewWorker(\"http://xx:xxxx@103.118.111.26:808\") ` // proxy worker, format: `protocol://user(optional):password(optional)@ip:port`, alias to`New()`, support http(s), socks5\n2. `worker, err := miner.NewWorker(nil)` // normal worker, default keep Cookie, alias to `New()`\n3. `worker := miner.NewAPI()` // API worker, will not keep Cookie\n4. `worker, err := miner.NewWorkerByClient(&http.Client{})` // You can also pass a `http.Client` if you want\n\nif you want to use worker twice, you can call `Clone()` method to clone a new worker, it can isolate the request and response of http, otherwise, you should deal concurrent program carefully.\n\n### 2.2 The Second Step\n\nCamouflage our worker:\n\n1. `worker.SetUrl(\"https://www.bing.com\")` // required: set url you want to\n2. `worker.SetMethod(miner.GET)` // optional: set http method `POST/GET/PUT/POSTJSON` and so on\n3. `worker.SetWaitTime(2)` // optional: set timeout of http request\n4. `worker.SetUa(miner.RandomUa())` // optional: set http browser user agent, you can see miner/config/ua.txt\n5. `worker.SetRefer(\"https://www.bing.com\")` // optional: set http request Refer\n6. `worker.SetHeaderParam(\"diyheader\", \"diy\")` // optional: set http diy header\n7. `worker.SetBData([]byte(\"file data\"))` // optional: set binary data for post or put\n8. `worker.SetFormParam(\"username\",\"hunterhug\")` // optional: set form data for post or put \n9. `worker.SetCookie(\"xx=dddd\")` // optional: you can set a init cookie, some website you can login and F12 copy the cookie\n10. `worker.SetCookieByFile(\"/root/cookie.txt\")` // optional: set cookie which store in a file\n\n### 2.3 The Third Step\n\nRun our worker:\n\n1. `body, err := worker.Go()` // if you use SetMethod(), auto use following ways, otherwise use Get()\n2. `body, err := worker.Get()` // default\n3. `body, err := worker.Post()` // post form request, data fill by SetFormParam()\n4. `body, err := worker.PostJSON()` // post JSON request, data fill by SetBData()\n5. `body, err := worker.PostXML()` // post XML request, data fill by SetBData()\n6. `body, err := worker.PostFILE()` // upload file, data fill by SetBData(), and should set SetFileInfo(fileName, fileFormName string)\n7. `body, err := worker.Delete()` // you know!\n8. `body, err := worker.Put()` // ones http method...\n9. `body, err := worker.PutJSON()` // put JSON request\n10. `body, err := worker.PutXML()`\n11. `body, err := worker.PutFILE()`\n12. `body, err := worker.OtherGo(\"OPTIONS\", \"application/x-www-form-urlencoded\")` // Other http method, Such as OPTIONS etc., can not sent binary.\n13. `body, err := worker.OtherGoBinary(\"OPTIONS\", \"application/x-www-form-urlencoded\")` // Other http method, Such as OPTIONS etc., just sent binary.\n14. `body, err := worker.GoByMethod(\"POST\")` // you can override SetMethod() By this, equal SetMethod() then Go()\n\n### 2.4 The Fourth Step\n\nDeal the return data, all data will be return as binary, You can immediately store it into a new variable:\n\n1. `fmt.Println(string(html))` // type change directly\n2. `fmt.Println(worker.ToString())` // use spider method, after http response, data will keep in the field `Raw`, just use ToString\n3. `fmt.Println(worker.JsonToString())` // some json data will include chinese and other multibyte character, such as `我爱你,我的小绵羊`,`사랑해`\n\nAttention: \n\nAfter every request for an url, the next request you should cover your http request header, otherwise http header you set still exist,\nif just want clear post data, use `Clear()`, and want clear HTTP header too please use `ClearAll()`, but I suggest use `Clone()` to avoid this.\n\n### 2.5 Other\n\nHook:\n\n1. `SetBeforeAction(fc func(context.Context, *Worker))`\n2. `SetAfterAction(fc func(context.Context, *Worker))`\n\n# License\n\n```\nCopyright [2016-2022] [github.com/hunterhug]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n https://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n```" }, { "path": "README_ZH.md", "content": "# 项目代号：土拨鼠\n\n[![GitHub forks](https://img.shields.io/github/forks/hunterhug/marmot.svg?style=social&label=Forks)](https://github.com/hunterhug/marmot/network)\n[![GitHub stars](https://img.shields.io/github/stars/hunterhug/marmot.svg?style=social&label=Stars)](https://github.com/hunterhug/marmot/stargazers)\n[![GitHub last commit](https://img.shields.io/github/last-commit/hunterhug/marmot.svg)](https://github.com/hunterhug/marmot)\n[![Go Report Card](https://goreportcard.com/badge/github.com/hunterhug/marmot)](https://goreportcard.com/report/github.com/hunterhug/marmot)\n[![GitHub issues](https://img.shields.io/github/issues/hunterhug/marmot.svg)](https://github.com/hunterhug/marmot/issues)\n[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)\n\n[English README](/README.md)\n\n![Marmot](logo.png)\n\n>万维网网络机器人，又称蜘蛛，爬虫，原理主要是通过构造符合HTTP协议的网络数据包，向指定主机请求资源，获取返回的数据.万维网有大量的公开信息，人力采集数据费时费力，故激发了爬虫的产业化.\n批量获取公开网络数据并不违反，但由于无差别性，无节制，十分暴力的手段会导致对方服务的不稳定，因此，大部分资源提供商对数据包进行了某些过滤，在此背景下，小批量数据获取成为了难题.\n综合各种需求，如各种API对接，自动化测试等原理均一样，故开发了此爬虫库.\n\n>土拨鼠项目是一个人类友好姿势的代码库，开发采用面向对象的方式，易于理解.通过对Golang原生HTTP库的封装，帮用户处理了一些琐碎逻辑(如收集信息，检测参数)，并加入了一些容错机制(如加锁，及时关闭流)，保证了爬虫高并发的安全.此库提供了大量优美的API接口，复用率高，十分方便地支持Cookie接力，爬虫代理设置，以及一般的HTTP请求设置如头部设置，超时，暂停设置，数据设置等，支持全部的HTTP方法如POST/PUT/GET/DELETE等，内置爬虫池和浏览器UA池，易于开发多UA多Cookie分布式爬虫.\n\n>该库简单实用，短短几行代码即可取代以往杂乱无章的面包条代码片段，已经应用在某些大项目中。\n\n>该库主要用途：微信开发/API对接/自动化测试/抢票脚本/网站监控/点赞插件/数据爬取\n\n## 一. 下载\n\n自己封装的 `Golang` 爬虫下载库，支持各种代理模式和伪装功能，你只需通过该方式获取库：\n\n```\ngo get -v github.com/hunterhug/marmot/miner\n```\n\n代码结构：\n\n```\n├── miner 核心库（HTTP请求封装）\n├── expert 解析库（HTML解析封装）\n├── example Example示例库\n ├── lession 示例\n ├── practice 练习 \n```\n\n以下是几个实战例子：\n\n1. [多线程批量抓图片](/example/practice/pictures/README.md)。\n2. [模拟上传文件](/example/practice/upload/README.md)。\n\n## 二. 使用\n\n此库可模拟上传文件，模拟表单提交，模拟各种各样的操作。\n\n`lesson1.go`\n\n```go\npackage main\n\nimport (\n\t\"fmt\"\n\t\"github.com/hunterhug/marmot/miner\"\n)\n\nfunc main() {\n\tminer.SetLogLevel(miner.DEBUG)\n\n\t// 使用默认的全局矿工，当然你可以自己建一个\n\t//worker, _ := miner.New(nil)\n\t//worker = miner.NewWorkerWithNoProxy()\n\t//worker = miner.NewAPI()\n\t//worker, _ = miner.NewWorkerWithProxy(\"socks5://127.0.0.1:1080\")\n\tworker := miner.Clone()\n\t_, err := worker.SetUrl(\"https://www.bing.com\").Go()\n\tif err != nil {\n\t\tfmt.Println(err.Error())\n\t} else {\n\t\tfmt.Println(worker.ToString())\n\t}\n}\n```\n\n官方部分示例已经合进本库，参见 [example](example) 文件夹。\n\n### 第一步\n\n矿工有四种类型：\n\n1. `miner.NewWorker(\"http://user:password@103.118.111.26:808\") ` // 代理矿工，默认自动化Cookie接力 格式:`协议://代理帐号(可选):代理密码(可选)@ip:port`，支持http(s)，socks5，别名函数 `New()`\n2. `miner.NewWorker(nil)` // 正常矿工，默认自动化Cookie接力，别名函数`New()`\n3. `miner.NewAPI()` // API矿工，默认Cookie不接力，主要用来对接服务端 API\n4. `miner.NewWorkerByClient(&http.Client{})` // 可自定义客户端\n\n如果你想使用一个矿工多次，使用前请先调用 `Clone()` 方法，它会创建新的矿工来进行 HTTP 请求的数据隔离，避免你并发调用多次导致的混乱。\n\n### 第二步\n\n模拟矿工设置头部:\n\n1. `worker.SetUrl(\"https://www.bing.com\")` // 设置HTTP请求要抓取的网址，**必须**\n2. `worker.SetMethod(miner.GET)` // 设置HTTP请求的方法:`POST/GET/PUT/POSTJSON`等\n3. `worker.SetWaitTime(2)` // 设置HTTP请求超时时间\n4. `worker.SetUa(miner.RandomUa())` // 设置HTTP请求浏览器标志，本项目提供445个浏览器标志，可选设置\n5. `worker.SetRefer(\"http://www.baidu.com\")` // 设置HTTP请求Refer头\n6. `worker.SetHeaderParam(\"diyheader\", \"diy\")` // 设置HTTP请求自定义头部\n7. `worker.SetBData([]byte(\"file data\"))` // HTTP请求需要上传数据\n8. `worker.SetFormParam(\"username\",\"hunterhug\")` // HTTP请求需要提交表单\n9. `worker.SetCookie(\"xx=dddd\")` // HTTP请求设置cookie，某些网站需要登录后F12复制cookie\n\n### 第三步\n\n矿工启动方式有：\n\n1. `body, err := worker.Go()` // 如果设置SetMethod()，会调用下方对应的方法，否则使用Get()\n2. `body, err := worker.Get()` // 默认\n3. `body, err := worker.Post()` // POST表单请求，数据在SetFormParam()\n4. `body, err := worker.PostJSON()` // 提交JSON请求，数据在SetBData()\n5. `body, err := worker.PostXML()` // 提交XML请求，数据在SetBData()\n6. `body, err := worker.PostFILE()` // 提交文件上传请求，文件二进制数据通过SetBData()设置，然后设置SetFileInfo(fileName，fileFormName string) 表明文件名和表单field Name\n7. `body, err := worker.Delete()` \n8. `body, err := worker.Put()`\n9. `body, err := worker.PutJSON()`\n10. `body, err := worker.PutXML()`\n11. `body, err := worker.PutFILE()`\n12. `body, err := worker.OtherGo(\"OPTIONS\", \"application/x-www-form-urlencoded\")` // 其他自定义的HTTP方法，不能模拟二进制\n13. `body, err := worker.OtherGoBinary(\"OPTIONS\", \"application/x-www-form-urlencoded\")` // 其他自定义的HTTP方法，模拟二进制\n14. `body, err := worker.GoByMethod(\"POST\")` // 等同于 SetMethod() 然后 Go()\n\n### 第四步\n\n每次下载会返回 `[]byte`，请自行解析，调试时可以使用以下方法：\n\n1. `fmt.Println(string(html))` // 每次抓取后会返回二进制数据，直接类型转化\n2. `fmt.Println(worker.ToString())` // http响应后二进制数据也会保存在矿工对象的Raw字段中，使用ToString可取出来\n3. `fmt.Println(worker.JsonToString())` // 如果获取到的是JSON数据，请采用此方法转义回来，不然字符串会乱码\n\n注意：每次下载后，需要使用以下方法将表单等数据重置（我推荐每次使用 `Clone()` 来避免这种做法，也就是克隆一个新的矿工来进行下一次操作）：\n\n1. `Clear()` // 清除表单和二进制数据\n2. `ClearAll()` // 清除表单和二进制数据，还清除全部HTTP头部\n3. `ClearCookie()` // 可清除Cookie\n\n### 其他\n\n勾子:\n\n1. `SetBeforeAction(fc func(context.Context, *Worker))` 爬虫动作前可AOP注入。\n2. `SetAfterAction(fc func(context.Context, *Worker))` 爬虫动作完成后。\n\n# License\n\n```\nCopyright [2016-2022] [github.com/hunterhug]\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n https://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n```" }, { "path": "example/lesson/lesson1/lesson1.go", "content": "package main\n\nimport (\n\t\"fmt\"\n\t\"github.com/hunterhug/marmot/miner\"\n)\n\n/*\nMost Simple: Use Default Worker!\n*/\nfunc main() {\n\tminer.SetLogLevel(miner.DEBUG)\n\n\t// Use Default Worker, You can Also New One:\n\t//worker, _ := miner.New(nil)\n\t//worker = miner.NewWorkerWithNoProxy()\n\t//worker = miner.NewAPI()\n\t//worker, _ = miner.NewWorkerWithProxy(\"socks5://127.0.0.1:1080\")\n\tworker := miner.Clone()\n\t_, err := worker.SetUrl(\"https://www.bing.com\").Go()\n\tif err != nil {\n\t\tfmt.Println(err.Error())\n\t} else {\n\t\tfmt.Println(worker.ToString())\n\t}\n}\n" }, { "path": "example/lesson/lesson2/lesson2.go", "content": "package main\n\nimport (\n\t\"fmt\"\n\t\"github.com/hunterhug/marmot/miner\"\n)\n\n/*\nNew A Worker!\n*/\nfunc main() {\n\t// 1. New a worker\n\tworker, _ := miner.New(nil)\n\n\t// 2. Set a URL And Fetch\n\thtml, err := worker.SetUrl(\"https://www.bing.com\").SetUa(miner.RandomUa()).SetMethod(miner.GET).Go()\n\tif err != nil {\n\t\tfmt.Println(err.Error())\n\t}\n\n\t// 3.Print content equal to fmt.Println(worker.ToString())\n\tfmt.Println(string(html))\n}\n" }, { "path": "example/lesson/lesson3/lesson3.go", "content": "package main\n\n/*\n\tMore detail Example\n*/\nimport (\n\t// 1:import package\n\t\"github.com/hunterhug/marmot/miner\"\n\t\"github.com/hunterhug/marmot/util\"\n)\n\nfunc init() {\n\t// 2:Optional global setting\n\tminer.SetLogLevel(miner.DEBUG) // optional, set log to debug\n\tminer.SetGlobalTimeout(3) // optional, http request timeout time\n\n}\n\nfunc main() {\n\tlog := miner.Log() // optional, miner log you can choose to use\n\n\t// 3: Must new a Worker object, three ways\n\t//worker, err := miner.NewWorker(\"http://xx:xxxx@103.118.111.26:808\") // proxy format: protocol://user(optional):password(optional)@ip:port\n\t//worker, err := miner.NewWorker(nil) // normal worker, default keep Cookie\n\t//worker := miner.NewAPI() // API worker, not keep Cookie\n\tworker, err := miner.New(nil) // NewWorker alias\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\t// 4: Set the request Method/URL and some others, can chain set, only SetUrl is required.\n\t// SetUrl: required, the Url\n\t// SetMethod: optional, HTTP method: POST/GET/..., default GET\n\t// SetWaitTime: optional, HTTP request wait/pause time\n\tworker.SetUrl(\"https://www.bing.com\").SetMethod(miner.GET).SetWaitTime(2)\n\tworker.SetUa(miner.RandomUa()) // optional, browser user agent: IE/Firefox...\n\tworker.SetRefer(\"https://www.bing.com\") // optional, url refer\n\tworker.SetHeaderParam(\"diy_header\", \"diy\") // optional, some other diy http header\n\t//worker.SetBData([]byte(\"file data\")) // optional, if you want post JSON data or upload file\n\t//worker.SetFormParam(\"username\",\"root\") // optional: if you want post form\n\t//worker.SetFormParam(\"password\",\"123\")\n\n\t// 5: Start Run\n\t//worker.Get() // default GET\n\t//worker.Post() // POST form request data, data can fill by SetFormParam()\n\t//worker.PostJSON() // POST JSON dara, use SetBData()\n\t//worker.PostXML() // POST XML, use SetBData()\n\t//worker.PostFILE() // POST to Upload File, data in SetBData() too\n\t//worker.OtherGo(\"OPTIONS\", \"application/x-www-form-urlencoded\") // Other http method, Such as OPTIONS etcd\n\tbody, err := worker.Go() // if you use SetMethod(), otherwise equal to Get()\n\tif err != nil {\n\t\tlog.Error(err.Error())\n\t} else {\n\t\tlog.Infof(\"%s\", string(body)) // Print return data\n\t}\n\n\tlog.Debugf(\"%#v\", worker.GetCookies) // if you not set log as debug, it will not appear\n\n\t// You must Clear it! If you want to POST Data by SetFormParam()/SetBData() again\n\t// After get the return data by post data, you can clear the data you fill\n\t// I suggest use Clone()\n\tworker.Clear()\n\t//worker.ClearAll() // you can also want to clear all, include http header you set\n\n\t// Worker pool for concurrent, every Worker Object is serial as the browser. if you want collateral execution, use this.\n\tminer.Pool.Set(\"myfirstworker\", worker)\n\tif w, ok := miner.Pool.Get(\"myfirstworker\"); ok {\n\t\tgo func() {\n\t\t\tdata, _ := w.Clone().SetUrl(\"https://www.bing.com\").Get()\n\t\t\tlog.Info(string(data))\n\t\t}()\n\t\tutil.Sleep(10)\n\t}\n}\n" }, { "path": "example/lesson/lesson4/lesson4.go", "content": "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"github.com/hunterhug/marmot/util\"\n\t\"strings\"\n\n\t\"github.com/hunterhug/marmot/expert\"\n\t\"github.com/hunterhug/marmot/miner\"\n)\n\nfunc main() {\n\t// We can debug, to see whether SetBeforeAction make sense\n\tminer.SetLogLevel(miner.DEBUG)\n\n\t// The url we want\n\turl := \"https://www.bing.com\"\n\n\t// IAM we can NewAPI\n\tworker := miner.NewAPI()\n\n\t// We can aop by context\n\tctx, cancel := context.WithCancel(context.Background())\n\t//ctx := context.TODO()\n\tworker.SetContext(ctx)\n\n\t// we cancel it after 5 second\n\tgo func() {\n\t\tfmt.Println(\"I stop and sleep 5\")\n\t\tutil.Sleep(5)\n\t\tfmt.Println(\"I wake up after sleep 5\")\n\t\tcancel()\n\t}()\n\n\t// Before we make some change, And every GET Or POST it will action\n\tworker.SetBeforeAction(func(ctx context.Context, this *miner.Worker) {\n\t\tfmt.Println(\"Before Action, I will add a HTTP header, then sleep wait cancel\")\n\t\tthis.SetHeaderParam(\"Marmot\", \"v2\")\n\t\tthis.SetHeaderParam(\"DUDUDUU\", \"DUDU\")\n\t\tselect {\n\t\tcase <-ctx.Done(): // block in here util cancel()\n\t\t\t//fmt.Println(ctx.Err())\n\t\t\tfmt.Println(\"after sleep, i do action.\")\n\t\t}\n\t})\n\n\tworker.SetAfterAction(func(ctx context.Context, this *miner.Worker) {\n\t\tfmt.Println(\"After Action, I just print this sentence\")\n\t})\n\n\t// Let's Go\n\tbody, err := worker.SetUrl(url).GoByMethod(miner.GET)\n\tif err != nil {\n\t\tfmt.Println(err.Error())\n\t} else {\n\t\t// Parse We want\n\t\tfmt.Printf(\"Output:\\n %s\\n\", MyParse(body))\n\t}\n\n}\n\n// MyParse Parse HTML page\nfunc MyParse(data []byte) string {\n\tdoc, err := expert.QueryBytes(data)\n\tif err != nil {\n\t\tfmt.Println(err.Error())\n\t}\n\treturn strings.TrimSpace(doc.Find(\"title\").Text())\n}\n" }, { "path": "example/lesson/lesson5/lesson5.go", "content": "package main\n\n/*\n\tProxy Worker!\n\tYou first should own a remote machine, Then in your local tap:\n\t\t`ssh -ND 1080 ubuntu@remoteIp`\n\tIt will generate socks5 proxy client in your local, which port is 1080\n*/\nimport (\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/hunterhug/marmot/expert\"\n\t\"github.com/hunterhug/marmot/miner\"\n)\n\nfunc init() {\n\tminer.SetLogLevel(miner.DEBUG)\n}\n\nfunc main() {\n\t// You can use a lot of proxy ip such \"https/http/socks5\"\n\tproxyIp := \"socks5://127.0.0.1:1080\"\n\n\turl := \"https://www.google.com\"\n\n\tworker, err := miner.New(proxyIp)\n\tif err != nil {\n\t\tfmt.Println(err.Error())\n\t\tos.Exit(1)\n\t}\n\n\tbody, err := worker.SetUa(miner.RandomUa()).SetUrl(url).SetMethod(miner.GET).Go()\n\tif err != nil {\n\t\tfmt.Println(err.Error())\n\t} else {\n\t\tfmt.Println(parse(body))\n\t}\n}\n\n// Parse HTML page\nfunc parse(data []byte) string {\n\tdoc, err := expert.QueryBytes(data)\n\tif err != nil {\n\t\tfmt.Println(err.Error())\n\t\treturn \"\"\n\t}\n\treturn doc.Find(\"title\").Text()\n}\n" }, { "path": "example/practice/helloworld/README.md", "content": "# 简单示例\n\n1. simple.go 最简单的一个" }, { "path": "example/practice/helloworld/main.go", "content": "package main\n\nimport (\n\t// 第一步：引入库\n\t\"github.com/hunterhug/marmot/miner\"\n\t\"github.com/hunterhug/marmot/util\"\n)\n\nfunc init() {\n\t// 第二步：可选设置全局\n\tminer.SetLogLevel(miner.DEBUG) // 设置全局矿工日志,可不设置,设置debug可打印出http请求轨迹\n\tminer.SetGlobalTimeout(3) // 矿工超时时间,可不设置\n\n}\n\nfunc main() {\n\n\tlog := miner.Log() // 矿工为你提供的日志工具,可不用\n\n\t// 第三步： 必须新建一个矿工对象\n\t// 代理IP格式: 协议://代理帐号(可选):代理密码(可选)@ip:port\n\t//worker, err := miner.NewWorker(\"http://xx:xxxx@103.118.111.26:808\")\n\t//worker, err := miner.NewWorker(nil) // 正常矿工 默认带Cookie\n\t//worker := miner.NewAPI() // API矿工 默认不带Cookie\n\tworker, err := miner.New(nil) // NewWorker同名函数\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\t// 第四步：设置抓取方式和网站,可链式结构设置,只有SetUrl是必须的\n\t// SetUrl:Url必须设置\n\t// SetMethod:HTTP方法可以是POST或GET,可不设置,默认GET,传错值默认为GET\n\t// SetWaitTime:暂停时间,可不设置,默认不暂停\n\tworker.SetUrl(\"https://www.bing.com\").SetMethod(miner.GET).SetWaitTime(2)\n\tworker.SetUa(miner.RandomUa()) //设置随机浏览器标志\n\tworker.SetRefer(\"https://www.bing.com\") // 设置Refer头\n\tworker.SetHeaderParam(\"diyheader\", \"diy\") // 自定义头部\n\t//worker.SetBData([]byte(\"file data\")) // 如果你要提交JSON数据/上传文件\n\t//worker.SetFormParam(\"username\",\"hunterhug\") // 提交表单\n\t//worker.SetFormParam(\"password\",\"123\")\n\n\t// 第五步：开始爬\n\t//worker.Get() // 默认GET\n\t//worker.Post() // POST表单请求,数据在SetFormParam()\n\t//worker.PostJSON() // 提交JSON请求,数据在SetBData()\n\t//worker.PostXML() // 提交XML请求,数据在SetBData()\n\t//worker.PostFILE() // 提交文件上传请求,数据在SetBData()\n\tbody, err := worker.Go() // 如果设置SetMethod(),采用,否则Get()\n\tif err != nil {\n\t\tlog.Error(err.Error())\n\t} else {\n\t\tlog.Infof(\"%s\", string(body)) // 打印获取的数据\n\t}\n\n\tlog.Debugf(\"%#v\", worker.GetCookies()) // 不设置全局log为debug是不会出现这个东西的\n\n\t// 爬取完毕后可以清除POST的表单数据/文件数据/JSON数据\n\t// 我建议使用 Clone()，这样也可以不用清除\n\tworker.Clear()\n\n\t// 爬取完毕后可以清除设置的Http头部和POST的表单数据/文件数据/JSON数据\n\t//worker.ClearAll()\n\n\t// 矿工池子\n\tminer.Pool.Set(\"worker1\", worker)\n\tif w, ok := miner.Pool.Get(\"worker1\"); ok {\n\t\tgo func() {\n\t\t\tdata, _ := w.Clone().SetUrl(\"https://www.bing.com\").Get()\n\t\t\tlog.Info(string(data))\n\t\t}()\n\t\tutil.Sleep(10)\n\t}\n\n}\n" }, { "path": "example/practice/pictures/README.md", "content": "# 多线程批量抓图片\n\n直接运行程序即可。\n\n```go\ngo run main.go\n\nWelcome: Input \"url\" and picture keep \"dir\"\n---------------------------------------------\nURL(Like: \"https://www.bizhizj.com\")\n\nDIR(Default: \"./picture\")\n```" }, { "path": "example/practice/pictures/main.go", "content": "package main\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"github.com/hunterhug/marmot/expert\"\n\t\"github.com/hunterhug/marmot/miner\"\n\t\"github.com/hunterhug/marmot/util\"\n\t\"net/url\"\n\t\"strings\"\n)\n\n// MinerNum Num of miner, We can run it at the same time to crawl data fast\nvar MinerNum = 5\n\n// ProxyAddress You can update this decides whether to proxy\nvar ProxyAddress interface{}\n\nfunc main() {\n\t// You can Proxy!\n\t// ProxyAddress = \"socks5://127.0.0.1:1080\"\n\n\tfmt.Println(`Welcome: Input \"url\" and picture keep \"dir\"`)\n\tfmt.Println(\"---------------------------------------------\")\n\turlPicture := util.Input(`URL(Like: \"https://www.bizhizj.com\")`, \"https://www.bizhizj.com/dongman\")\n\tdir := util.Input(`DIR(Default: \"./picture\")`, \"./picture\")\n\tfmt.Printf(\"You will keep %s picture in dir %s\\n\", urlPicture, dir)\n\tfmt.Println(\"---------------------------------------------\")\n\n\t// Start Catch\n\terr := DownloadHTMLPictures(urlPicture, dir, MinerNum, ProxyAddress)\n\tif err != nil {\n\t\tfmt.Println(\"Error:\" + err.Error())\n\t}\n}\n\n// DownloadHTMLPictures Download one HTML page's all pictures\n// @URL: http://image.baidu.com\n// @SaveDir /home/images\n// @ProxyAddress : \"socks5://127.0.0.1:1080\"\nfunc DownloadHTMLPictures(URL string, SaveDir string, MinerNum int, ProxyAddress interface{}) error {\n\n\t// Check valid\n\t_, err := url.Parse(URL)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// New a worker to get url\n\tworker, err := miner.New(ProxyAddress)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tresult, err := worker.SetUrl(URL).SetUa(miner.RandomUa()).Get()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Find all picture\n\tpictures := expert.FindPicture(string(result))\n\n\treturn DownloadURLPictures(pictures, SaveDir, MinerNum, worker)\n}\n\n// DownloadURLPictures Download pictures faster!\nfunc DownloadURLPictures(PictureUrls []string, SaveDir string, MinerNum int, initWorker *miner.Worker) error {\n\t// Empty, What a pity!\n\tif len(PictureUrls) == 0 {\n\t\treturn errors.New(\"empty\")\n\t}\n\n\t// Make dir!\n\terr := util.MakeDir(SaveDir)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Divide pictures into several worker\n\txxx, _ := util.DivideStringList(PictureUrls, MinerNum)\n\n\t// Chanel to info exchange\n\tchs := make(chan int, len(PictureUrls))\n\n\t// Go at the same time\n\tfor num, pictureList := range xxx {\n\t\t// Clone new worker\n\t\tworkerPicture := initWorker.Clone()\n\t\tworkerPicture.SetUa(miner.RandomUa())\n\n\t\t// Go save picture!\n\t\tgo func(images []string, worker *miner.Worker, num int) {\n\t\t\tfor _, img := range images {\n\n\t\t\t\t// Check, May be Pass\n\t\t\t\t_, err := url.Parse(img)\n\t\t\t\tif err != nil {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\t// Change Name of our picture\n\t\t\t\tfilename := strings.Replace(util.ValidFileName(img), \"#\", \"_\", -1)\n\n\t\t\t\t// Exist?\n\t\t\t\tif util.FileExist(SaveDir + \"/\" + filename) {\n\t\t\t\t\tfmt.Println(\"File Exist：\" + SaveDir + \"/\" + filename)\n\t\t\t\t\tchs <- 0\n\t\t\t\t} else {\n\n\t\t\t\t\t// Not Exsit?\n\t\t\t\t\timgsrc, e := worker.SetUrl(img).Get()\n\t\t\t\t\tif e != nil {\n\t\t\t\t\t\tfmt.Println(\"Download \" + img + \" error:\" + e.Error())\n\t\t\t\t\t\tchs <- 0\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\n\t\t\t\t\t// Save it!\n\t\t\t\t\te = util.SaveToFile(SaveDir+\"/\"+filename, imgsrc)\n\t\t\t\t\tif e == nil {\n\t\t\t\t\t\tfmt.Printf(\"SP%d: Keep in %s/%s\\n\", num, SaveDir, filename)\n\t\t\t\t\t}\n\t\t\t\t\tchs <- 1\n\t\t\t\t}\n\t\t\t}\n\t\t}(pictureList, workerPicture, num)\n\t}\n\n\t// Every picture should return\n\tfor i := 0; i < len(PictureUrls); i++ {\n\t\t<-chs\n\t}\n\n\treturn nil\n}\n" }, { "path": "example/practice/upload/README.md", "content": "# 模拟上传文件\n\n## 启动服务端\n\n```\ngo run main.go\n```\n\n打开 127.0.0.1:1789 可以看到上传文件的界面。\n\n但我们使用程序模拟上传。\n\n## 启动模拟上传程序\n\n```\ncd client\ngo run main.go\n```\n\n将会看到出现一个 `doc.go.xxxx` 的文件出现在目录下。" }, { "path": "example/practice/upload/client/main.go", "content": "package main\n\nimport (\n\t\"fmt\"\n\t\"github.com/hunterhug/marmot/miner\"\n)\n\nfunc postFile(filename string, targetUrl string) {\n\tworker, _ := miner.New(nil)\n\tresult, err := worker.SetUrl(targetUrl).SetBData([]byte(\"dddd\")).SetFileInfo(filename+\".xxxxxx\", \"uploadfile\").SetFormParam(\"xxxx\", \"xxx\").PostFILE()\n\tif err != nil {\n\t\tfmt.Println(err.Error())\n\t} else {\n\t\tfmt.Println(string(result))\n\t}\n}\n\n// sample usage\nfunc main() {\n\ttargetUrl := \"http://127.0.0.1:1789/upload\"\n\tfilename := \"./doc.go\"\n\tpostFile(filename, targetUrl)\n}\n" }, { "path": "example/practice/upload/main.go", "content": "package main\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"net/http\"\n\t\"os\"\n)\n\nfunc main() {\n\tfmt.Println(\"max upload size\", 32<<20)\n\tfmt.Println(\"web url: 127.0.0.1:1789\")\n\thttp.HandleFunc(\"/\", index)\n\thttp.HandleFunc(\"/upload\", upload)\n\thttp.ListenAndServe(\":1789\", nil)\n}\n\nfunc upload(w http.ResponseWriter, r *http.Request) {\n\tr.ParseMultipartForm(32 << 20)\n\tfile, handler, err := r.FormFile(\"uploadfile\")\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\treturn\n\t}\n\tdefer file.Close()\n\tf, err := os.OpenFile(handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)\n\tif err != nil {\n\t\tfmt.Println(err)\n\t\treturn\n\t}\n\tdefer f.Close()\n\tio.Copy(f, file)\n\tfmt.Fprintln(w, \"upload ok!\")\n}\n\nfunc index(w http.ResponseWriter, r *http.Request) {\n\tw.Write([]byte(tpl))\n}\n\nconst tpl = ` \n \n上传文件 \n \n \n
\n \n \n \n
\n \n`\n" }, { "path": "expert/query.go", "content": "package expert\n\n// Package expert is use to parse content\nimport (\n\t\"github.com/PuerkitoBio/goquery\"\n\t\"regexp\"\n\t\"strings\"\n)\n\nfunc QueryBytes(content []byte) (*goquery.Document, error) {\n\tdoc, err := goquery.NewDocumentFromReader(strings.NewReader(string(content)))\n\treturn doc, err\n}\n\nfunc QueryString(content string) (*goquery.Document, error) {\n\tdoc, err := goquery.NewDocumentFromReader(strings.NewReader(content))\n\treturn doc, err\n}\n\n// FindPicture Find All picture. Must prefix with http(s)\nfunc FindPicture(s string) []string {\n\tpicList := make([]string, 0)\n\tre, _ := regexp.Compile(`src\\s*=\\s*[\"'](http[s]?:\\/\\/.*?\\.(jpg|jpeg|png|gif))[\"']`)\n\toutput := re.FindAllStringSubmatch(s, -1)\n\tfor _, o := range output {\n\t\tpicList = append(picList, o[1])\n\t}\n\treturn picList\n}\n" }, { "path": "expert/query_test.go", "content": "package expert\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n)\n\nfunc TestFindPicture(t *testing.T) {\n\tdata := `\n\t\t\n\t\thttps://imgsa.baidu.com/forum/w%3D580/sign=294db374d462853592e0d229a0ee76f2/e732c895d143ad4b630e8f4683025aafa40f0611.jpg\n\t\t\n\t\thttps://imgsa.baidu.com/forum/whttps:/4d462853592e0d229a0ee76f2/e732c895d143ad4b630e8f4683025aafa40f0611.jpg\n\t\thttps://img1.jpg\n\t\thttp://im62853592e0d229a0ee76f2/e732c895d143ad4b630e8f4683025aafa40f0611.jpgsfsadfsda\n\t\thttpdb374://aafa40f0611.jpg\n\n\t\tsrc=\"http://s.jpg\"\n\n\t\tsrc=\"https://s.jpg\"\n\n\t\tsrc = \"http://s.jpg\"\n\t\tsrc=\"https:s.jpg\"\n\t\t\"https://img1.jpg\" \"https://img1.jpgsss\",\"https://img1.jpgss\",\"https://img1.jpgss\"\n\nsrc=\"https://up.enterdesk.com/edpic_360_360/bf/53/52/bf5352f28a839e01dd3fcb811669102c.jpg\"\n\t\t`\n\tresult := FindPicture(data)\n\n\tfmt.Printf(\"%#v\", result)\n}\n" }, { "path": "go.mod", "content": "module github.com/hunterhug/marmot\n\ngo 1.25.5\n\nrequire (\n\tgithub.com/PuerkitoBio/goquery v1.8.0\n\tgithub.com/hunterhug/golog/v2 v2.0.0\n\tgolang.org/x/net v0.52.0\n)\n\nrequire (\n\tgithub.com/andybalholm/cascadia v1.3.1 // indirect\n\tgo.uber.org/multierr v1.11.0 // indirect\n\tgo.uber.org/zap v1.27.1 // indirect\n\tgopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect\n)\n" }, { "path": "go.sum", "content": "github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=\ngithub.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=\ngithub.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=\ngithub.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/hunterhug/golog/v2 v2.0.0 h1:atgT5kBKBxosf/9DfiAmbz7FWzO6VFKMJV3cEg0drB4=\ngithub.com/hunterhug/golog/v2 v2.0.0/go.mod h1:oxkZi9qle15EMD6MBjSGGYcepW7YGWUfrrZx44oIoKU=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=\ngithub.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=\ngo.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=\ngo.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=\ngo.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=\ngo.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=\ngo.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=\ngo.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=\ngolang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=\ngolang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=\ngopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n" }, { "path": "miner/client.go", "content": "package miner\n\nimport (\n\t\"crypto/tls\"\n\t\"errors\"\n\t\"golang.org/x/net/proxy\"\n\t\"net/http\"\n\t\"net/http/cookiejar\"\n\t\"net/url\"\n\t\"strings\"\n\n\t\"github.com/hunterhug/marmot/util\"\n)\n\n// NewJar Cookie record Jar\nfunc NewJar() *cookiejar.Jar {\n\tcookieJar, _ := cookiejar.New(nil)\n\treturn cookieJar\n}\n\n// Default Client\nvar (\n\t// Client Save Cookie\n\tClient = &http.Client{\n\t\tCheckRedirect: func(req *http.Request, via []*http.Request) error {\n\t\t\tLogger.Debugf(\"[GoWorker] Redirect: %v\", req.URL)\n\t\t\treturn nil\n\t\t},\n\t\tJar: NewJar(),\n\t\tTimeout: util.Second(DefaultTimeOut),\n\t\tTransport: &http.Transport{\n\t\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t\t},\n\t}\n\n\t// NoCookieClient Not Save Cookie\n\tNoCookieClient = &http.Client{\n\t\tCheckRedirect: func(req *http.Request, via []*http.Request) error {\n\t\t\tLogger.Debugf(\"[GoWorker] Redirect: %v\", req.URL)\n\t\t\treturn nil\n\t\t},\n\t\tTimeout: util.Second(DefaultTimeOut),\n\t\tTransport: &http.Transport{\n\t\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t\t},\n\t}\n)\n\n// NewProxyClient New a Proxy client, Default save cookie, Can timeout\n// We should support some proxy way such as http(s) or socks\nfunc NewProxyClient(proxyString string) (*http.Client, error) {\n\tproxyUrl, err := url.Parse(proxyString)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tprefix := strings.Split(proxyString, \":\")[0]\n\n\t// setup a http transport\n\thttpTransport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}\n\n\t// http://\n\t// https://\n\t// socks5://\n\tswitch prefix {\n\tcase \"http\", \"https\":\n\t\thttpTransport.Proxy = http.ProxyURL(proxyUrl)\n\tcase \"socks5\":\n\t\t// create a socks5 dialer\n\t\tdialer, err := proxy.FromURL(proxyUrl, proxy.Direct)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\thttpTransport.Dial = dialer.Dial\n\tdefault:\n\t\treturn nil, errors.New(\"this proxy way not allow:\" + prefix)\n\t}\n\n\t// This alone client, diff from global client.\n\tclient := &http.Client{\n\t\t// Allow redirect\n\t\tCheckRedirect: func(req *http.Request, via []*http.Request) error {\n\t\t\tLogger.Debugf(\"[GoWorker] Redirect: %v\", req.URL)\n\t\t\treturn nil\n\t\t},\n\t\t// Allow proxy: http, https, socks5\n\t\tTransport: httpTransport,\n\t\t// Allow keep cookie\n\t\tJar: NewJar(),\n\t\t// Allow Timeout\n\t\tTimeout: util.Second(DefaultTimeOut),\n\t}\n\treturn client, nil\n}\n\n// NewClient New a client, diff from proxy client\nfunc NewClient() *http.Client {\n\tclient := &http.Client{\n\t\t// Allow redirect\n\t\tCheckRedirect: func(req *http.Request, via []*http.Request) error {\n\t\t\tLogger.Debugf(\"[GoWorker] Redirect: %v\", req.URL)\n\t\t\treturn nil\n\t\t},\n\t\tJar: NewJar(),\n\t\tTimeout: util.Second(DefaultTimeOut),\n\t\tTransport: &http.Transport{\n\t\t\tTLSClientConfig: &tls.Config{InsecureSkipVerify: true},\n\t\t},\n\t}\n\n\treturn client\n}\n" }, { "path": "miner/config/ua.txt", "content": "Mozilla/5.0 (SymbianOS/9.1; U; [en]; SymbianOS/91 Series60/3.0) AppleWebkit/413 (KHTML, like Gecko) Safari/413\nMozilla/5.0 (SymbianOS/9.1; U; [en]; Series60/3.0 NokiaE60/4.06.0) AppleWebKit/413 (KHTML, like Gecko) Safari/413\nMozilla/5.0 (SymbianOS/9.2; U; Series60/3.1 NokiaN95/10.0.018; Profile/MIDP-2.0 Configuration/CLDC-1.1) AppleWebKit/413 (KHTML, like Gecko) Safari/413\nMozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1/110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413\nMozilla/5.0 (SymbianOS/9.4; U; Series60/5.0 Nokia5800d-1/21.0.025; Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413\nMozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/12.0.024; Profile/MIDP-2.1 Configuration/CLDC-1.1; en-us) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.12344\nMozilla/4.0 (compatible; MSIE 4.0; MSN 2.5; Windows 95)\nMozilla/4.0 (compatible; MSIE 4.0; Windows 95)\nMozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; MSN 2.5; Windows 98)\nMozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; Windows 95)\nMozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; Windows 98)\nMozilla/4.0 (compatible; MSIE 4.01; Windows 95)\nMozilla/4.0 (compatible; MSIE 4.01; Windows 95; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.)\nMozilla/4.0 (compatible; MSIE 4.01; Windows 98)\nMozilla/4.0 (compatible; MSIE 4.01; Windows 98; BIGLOBE)\nMozilla/4.0 (compatible; MSIE 4.01; Windows 98; canoncopyer)\nMozilla/4.0 (compatible; MSIE 4.01; Windows 98; Compaq)\nMozilla/4.0 (compatible; MSIE 5.0; Windows 95)\nMozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt)\nMozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt; i-CABLE)\nMozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt; ocnie5-1)\nMozilla/4.0 (compatible; MSIE 5.0; Windows 95; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.)\nMozilla/4.0 (compatible; MSIE 5.0; Windows 98)\nMozilla/4.0 (compatible; MSIE 5.0; Windows 98; CNETHomeBuild051099)\nMozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt)\nMozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt; ocnie5-1)\nMozilla/4.0 (compatible; MSIE 5.0; Windows 98; wn_ie5_en_v1)\nMozilla/4.0 (compatible; MSIE 5.0; Windows 98; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.; DigExt)\nMozilla/4.0 (compatible; MSIE 5.0; Windows NT)\nMozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)\nMozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt)\nMozilla/4.0 (compatible; MSIE 5.01; MSN 2.5; Windows 98)\nMozilla/4.0 (compatible; MSIE 5.01; Windows 95)\nMozilla/4.0 (compatible; MSIE 5.01; Windows 98)\nMozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)\nMozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; DigExt)\nMozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; TUCOWS)\nMozilla/4.0 (compatible; MSIE 5.01; Windows NT)\nMozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC)\nMozilla/4.0 (compatible; MSIE 5.16; Mac_PowerPC)\nMozilla/4.0 (compatible; MSIE 5.17; Mac_PowerPC)\nMozilla/4.0 (compatible; MSIE 5.22; Mac_PowerPC)\nMozilla/4.0 (compatible; MSIE 5.23; Mac_PowerPC)\nMozilla/4.0 (compatible; MSIE 5.5; Windows 95)\nMozilla/4.0 (compatible; MSIE 5.5; Windows 98)\nMozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)\nMozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705)\nMozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; by TSG)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.40607)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.40607; .NET CLR 1.0.3705)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; NOKTURNAL KICKS ASS)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FDM;\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.41115)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1\nMozilla/4.0 (compatible; MSIE 6.0; Windows 98)\nMozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows NT 5.1; iebar; .NET CLR 1.0.3705)\nMozilla/4.0 (compatible; MSIE 6.0; Win32);\nMozilla/4.0 (compatible; MSIE 6.0; Win32); .NET CLR 1.0.3705)\nMozilla/4.0 (compatible; MSIE 6.0; Windows 98; Hotbar 4.4.6.0)\nMozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x 4.90)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0; BVG\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; FDM)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; {FF0C8E09-3C86-44CB-834A-B8CEEC80A1D7}; iOpus-I-M)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; i-Nav 3.0.1.0F; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; MathPlayer 2.0; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Maxthon; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1);\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 2.0.40607)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Alexa Toolbar)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; BrowserBob)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; DFO-MPO Internet Explorer 6.0)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ENGINE; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT))\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ESB{F40811EE-DF17-4BC9-8785-B362ABF34098}; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FDM)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FTDv3 Browser; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; .NET CLR 1.1.4322; .NET CLR 2.0.40607)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; AtHome033)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; HCI0449; .NET CLR 1.0.3705)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; i-NavFourF; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon;\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2; Maxthon; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461; FunWebProducts; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Woningstichting Den Helder; .NET CLR 1.0.3705)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.41115)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; MyIE2; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; MyIE2; Maxthon; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 6.0; Windows XP)\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1)\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 2.0.50727; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1)\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; InfoPath.1)\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)\nMozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648)\nMozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; InfoPath.1\nMozilla/4.0 (compatible; GoogleToolbar 5.0.2124.2070; Windows 6.0; MSIE 8.0.6001.18241)\nMozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; EasyBits Go v1.0; InfoPath.1; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)\nMozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)\nMozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; Sleipnir/2.9.8)\nMozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)\nMozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)\nMozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)\nMozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)\nMozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)\nMozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)\nMozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; ARM; Trident/6.0)\nMozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.48 Safari/525.19\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.33 Safari/530.5\nMozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.38 Safari/532.0\nMozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.55 Safari/533.4\nMozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.55 Safari/534.3\nMozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.43 Safari/534.7\nMozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10 ChromePlus/1.5.2.0\nMozilla/5.0 (en-us) AppleWebKit/534.14 (KHTML, like Gecko; Google Wireless Transcoder) Chrome/9.0.597 Safari/534.14\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.151 Safari/534.16\nMozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.71 Safari/534.24\nMozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Iron/11.0.700.2 Chrome/11.0.700.2 Safari/534.24\nMozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.24\nMozilla/5.0 (Windows NT 5.1) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30 ChromePlus/1.6.3.1\nMozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.112 Safari/534.30\nMozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.107 Safari/535.1\nMozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) RockMelt/0.9.64.361 Chrome/13.0.782.218 Safari/535.1\nMozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1\nMozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.220 Safari/535.1\nMozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1\nMozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1\nMozilla/5.0 (Windows NT 5.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2\nMozilla/5.0 (X11; Linux i686) AppleWebKit/535.2 (KHTML, like Gecko) Ubuntu/10.04 Chromium/15.0.874.106 Chrome/15.0.874.106 Safari/535.2\nMozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.106 Safari/535.2\nMozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7\nMozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30618; Lunascape 4.7.3)\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; Lunascape 5.0 alpha2)\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; Lunascape 5.0 alpha2)\nMozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; Tablet PC 2.0; Lunascape 5.0 alpha2)\nMozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MAGW; .NET4.0C; Lunascape 6.5.8.24780)\nMozilla/2.02 (Macintosh; I; PPC)\nMozilla/3.01 (Macintosh; I; PPC)\nMozilla/4.01 (Macintosh; I; PPC)\nMozilla/4.03 [en]C-IMS (Win95; I)\nMozilla/4.04 [en] (Win95; I ;Nav)\nMozilla/4.04 [en] (Win95; I)\nMozilla/4.04 [en] (WinNT; I ;Nav)\nMozilla/4.04 [en] (WinNT; I)\nMozilla/4.04 [en] (Macintosh; I; PPC Nav)\nMozilla/4.04 [en] (X11; I; SunOS 5.5 sun4u)\nMozilla/4.05 [en] (Win95; I)\nMozilla/4.05 (Macintosh; I; PPC)\nMozilla/4.06 [en] (Win98; I)\nMozilla/4.06 [en] (Macintosh; I; PPC)\nMozilla/4.08 (Macintosh; I; PPC)\nMozilla/4.5 [en] (Win95; I)\nMozilla/4.5 [en] (Win98; I)\nMozilla/4.5 [en] (WinNT; I)\nMozilla/4.5 (Macintosh; I; PPC)\nMozilla/4.51 [en] (Win95; I)\nMozilla/4.51 [en] (Win98; I)\nMozilla/4.51 [en] (WinNT; I)\nMozilla/4.51 [en] (X11; I; SunOS 5.8 sun4u)\nMozilla/4.6 [en] (Win95; I)\nMozilla/4.6 [en] (Win98; I)\nMozilla/4.6 [en] (WinNT; I)\nMozilla/4.6 [en] (WinNT; I)\nMozilla/4.7 [en] (WinNT; I)\nMozilla/4.7 [en] (Win95; I)\nMozilla/4.7 [en] (Win98; I)\nMozilla/4.7 [en] (WinNT; I)\nMozilla/4.7 [en] (WinNT; I)\nMozilla/4.7 [en] (WinNT; U)\nMozilla/4.7 [en] (Macintosh; I; PPC)\nMozilla/4.76 [en_jp] (X11; U; SunOS 5.8 sun4u)\nMozilla/4.76 [en] (X11; U; SunOS 5.8 sun4u)\nMozilla/4.78 [en] (X11; U; SunOS 5.9 sun4u)\nMozilla/4.8 [en] (X11; U; SunOS 5.7 sun4u)\nMozilla/5.0 (Windows; U; Win98; en-JP; m18) Gecko/20001108 Netscape6/6.0\nMozilla/5.0 (Windows; U; Windows NT 5.0; en-JP; m18) Gecko/20010131 Netscape6/6.01\nMozilla/5.0 (Windows; U; Win 9x 4.90; en-JP; rv:0.9.4) Gecko/20011128 Netscape6/6.2.1\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:0.9.4.1) Gecko/20020508 Netscape6/6.2.3\nMozilla/5.0 (Macintosh; N; PPC; en-JP; macen-pub12) Gecko/20001108 Netscape6/6.0\nMozilla/5.0 (Macintosh; U; PPC; en-JP; rv:0.9.2) Gecko/20010726 Netscape6/6.1\nMozilla/5.0 (Macintosh; U; PPC; en-JP; rv:0.9.4) Gecko/20011022 Netscape6/6.2\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US; rv:0.9.4.1) Gecko/20020315 Netscape6/6.2.2\nMozilla/5.0 (Windows; U; Win98; en-US; rv:1.0.2) Gecko/20030208 Netscape/7.02\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.2) Gecko/20040804 Netscape/7.2 (ax)\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.2) Gecko/20040805 Netscape/7.2\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-JP; rv:1.0.2) Gecko/20021120 Netscape/7.01\nMozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-JP; rv:1.4) Gecko/20030624 Netscape/7.1\nMozilla/5.0 (X11; U; SunOS sun4u; en-JP; rv:1.0.1) Gecko/20020921 Netscape/7.0\nMozilla/5.0 (Windows; U; Windows NT 5.0; en-JP; rv:1.5) Gecko/20031007\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113\nMozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040616\nMozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.6) Gecko/20040113\nMozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.2.1) Gecko/20030225\nMozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.4.1) Gecko/20031030\nMozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.1) Gecko/20040805\nMozilla/5.0 (X11; U; SunOS sun4u; en-JP; rv:1.4) Gecko/20040414\nMozilla/5.0 (X11; U; Linux i686; rv:1.7.3) Gecko/20040913 Firefox/0.10\nMozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) Gecko/20040913 Firefox/0.10\nMozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20040913 Firefox/0.10\nMozilla/5.0 (X11; U; Linux i686; rv:1.7.3) Gecko/20041001 Firefox/0.10.1\nMozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1\nMozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) Gecko/20041001 Firefox/0.10.1\nMozilla/5.0 (Windows; U; Windows NT 5.2; rv:1.7.3) Gecko/20041001 Firefox/0.10.1\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.6) Gecko/20040206 Firefox/0.8\nMozilla/5.0 (Windows; U; Win98; en-US; rv:1.6) Gecko/20040206 Firefox/0.8\nMozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.6) Gecko/20040207 Firefox/0.8\nMozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.6) Gecko/20040206 Firefox/0.8\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040614 Firefox/0.9\nMozilla/5.0 (X11; U; Linux i686; es-ES; rv:1.7) Gecko/20040708 Firefox/0.9\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040707 Firefox/0.9.2\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/0.9.2 StumbleUpon/1.994\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040803 Firefox/0.9.3\nMozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3\nMozilla/5.0 (Windows; U; Win98; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3\nMozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20040803 Firefox/0.9.3\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7) Gecko/20041013 Firefox/0.9.3 (Ubuntu)\nMozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7) Gecko/20041013 Firefox/0.9.3 (Ubuntu)\nMozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\nMozilla/5.0 (Windows; U; Win 9x 4.90; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0\nMozilla/5.0 (Windows; U; Win 9x 4.90; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0\nMozilla/5.0 (Windows; U; Win98; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.7.5) Gecko/20041107 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.7.5) Gecko/20041110 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.0; fr-FR; rv:1.7.5) Gecko/20041108 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.0; it-IT; rv:1.7.5) Gecko/20041110 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041107 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.7.5) Gecko/20041110 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 StumbleUpon/1.999\nMozilla/5.0 (Windows; U; Windows NT 5.1; es-ES; rv:1.7.5) Gecko/20041210 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.1; fr-FR; rv:1.7.5) Gecko/20041108 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.1; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.1; sv-SE; rv:1.7.5) Gecko/20041108 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.8b) Gecko/20050212 Firefox/1.0+ (MOOX M3)\nMozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\nMozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8b) Gecko/20050118 Firefox/1.0+\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8b) Gecko/20050118 Firefox/1.0+\nMozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.5) Gecko/20050103 Firefox/1.0\nMozilla/5.0 (X11; U; Linux i386; en-US; rv:1.7.5) Gecko/20041109 Firefox/1.0\nMozilla/5.0 (X11; U; Linux i686; de-DE; rv:1.7.5) Gecko/20041128 Firefox/1.0 (Debian package 1.0-4)\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\nMozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041111 Firefox/1.0\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041111 Firefox/1.0 (Debian package 1.0-2)\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041116 Firefox/1.0 (Ubuntu) (Ubuntu package 1.0-2ubuntu3)\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041119 Firefox/1.0\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041123 Firefox/1.0\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041128 Firefox/1.0 (Debian package 1.0-4)\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041130 Firefox/1.0\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041214 Firefox/1.0 StumbleUpon/1.999\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041219 Firefox/1.0 (Debian package 1.0+dfsg.1-1)\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20050110 Firefox/1.0 (Debian package 1.0+dfsg.1-2)\nMozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\nMozilla/5.0 (X11; U; Linux i686; nl-NL; rv:1.7.5) Gecko/20050221 Firefox/1.0 (Ubuntu) (Ubuntu package 1.0+dfsg.1-6ubuntu1)\nMozilla/5.0 (X11; U; Linux i686; rv:1.8b) Gecko/20050124 Firefox/1.0+\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.6) Gecko/20050306 Firefox/1.0.1 (Debian package 1.0.1-2)\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6) Gecko/20050223 Firefox/1.0.1\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6) Gecko/20050225 Firefox/1.0.1\nMozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.6) Gecko/20050226 Firefox/1.0.1\nMozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-JP-mac; rv:1.8) Gecko/20051111 Firefox/1.5\nMozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8) Gecko/20051111 Firefox/1.5\nMozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8) Gecko/20051111 Firefox/1.5\nMozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.0.1) Gecko/20060111 Firefox/1.5.0.1\nMozilla/5.0 (X11; U; Linux i686; en; rv:1.8.0.2) Gecko/20060308 Firefox/1.5.0.2\nMozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3\nMozilla/5.0 (X11; U; Linux i686; en; rv:1.8.0.4) Gecko/20060508 Firefox/1.5.0.4\nMozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20\nMozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.12) Gecko/20080201 Firefox/2.0.0.12\nMozilla/5.0 (Macintosh; U; Intel Mac OS X; en-JP-mac; rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20\nMozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6\nMozilla/5.0 (Windows; U; Windows NT 6.0; en; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6 (.NET CLR 3.5.30729)\nMozilla/5.0 (Windows; U; Windows NT 6.0; en; rv:1.9.0.17) Gecko/2009122116 Firefox/3.0.17 GTB6 (.NET CLR 3.5.30729)\nMozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-JP-mac; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6\nMozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-JP-mac; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6 GTB5\nMozilla/5.0 (Windows NT 6.1; rv:2.0) Gecko/20100101 Firefox/4.0\nMozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:2.0) Gecko/20100101 Firefox/4.0\nMozilla/5.0 (Windows NT 6.1; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0\nMozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:5.0.1) Gecko/20100101 Firefox/5.0.1\nMozilla/5.0 (Windows NT 5.1; rv:6.0) Gecko/20100101 Firefox/6.0\nMozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:6.0.2) Gecko/20100101 Firefox/6.0.2\nMozilla/5.0 (Windows NT 6.0; rv:7.0.1) Gecko/20100101 Firefox/7.0.1\nMozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:7.0.1) Gecko/20100101 Firefox/7.0.1\nMozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:8.0.1) Gecko/20100101 Firefox/8.0.1\nMozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1\nMozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:9.0.1) Gecko/20100101 Firefox/9.0.1\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.6\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.7\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/85.8.2 (KHTML, like Gecko) Safari/85.8\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/85.8.2 (KHTML, like Gecko) Safari/85.8.1\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/85.8.5 (KHTML, like Gecko) Safari/85.8\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/103u (KHTML, like Gecko) Safari/100.1\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/124 (KHTML, like Gecko) Safari/125.1\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/125.2 (KHTML, like Gecko) Safari/125.8\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/125.4 (KHTML, like Gecko) Safari/125.9\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.11\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; nl-nl) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/125.5.6 (KHTML, like Gecko) Safari/125.12\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.1 (KHTML, like Gecko) Safari/312\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.1.1 (KHTML, like Gecko) Safari/312\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.5 (KHTML, like Gecko) Safari/312.3\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.5\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.6\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412 (KHTML, like Gecko) Safari/412\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.6.2 (KHTML, like Gecko) Safari/125.11\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.6.2 (KHTML, like Gecko) Safari/412.2.2\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.7 (KHTML, like Gecko) Safari/412.5\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/416.11 (KHTML, like Gecko) Safari/416.12\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.2\nMozilla/5.0 (Macintosh; U; Intel Mac OS X; en-jp) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.3\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/522.11.1 (KHTML, like Gecko) Version/3.0.3 Safari/522.12.1\nMozilla/5.0 (Macintosh; U; Intel Mac OS X; en-jp) AppleWebKit/523.10.3 (KHTML, like Gecko) Version/3.0.4 Safari/523.10\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.10.6 (KHTML, like Gecko) Version/3.0.4 Safari/523.10.6\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.12 (KHTML, like Gecko) Version/3.0.4 Safari/523.12\nMozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.12.2 (KHTML, like Gecko) Version/3.0.4 Safari/523.12.2\nMozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en-jp) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13\nMozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_3; en-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.1 Safari/525.20\nMozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.22\nMozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; en-jp) AppleWebKit/525.26.2 (KHTML, like Gecko) Version/3.2 Safari/525.26.12\nMozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-jp) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1\nMozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-jp) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16\nMozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-jp) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10\nMozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-jp) AppleWebKit/531.21.11 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10\nMozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-jp) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16\nMozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.52.7 (KHTML, like Gecko) Version/5.1.2 Safari/534.52.7\nMozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2\nMozilla/5.0 (Macintosh; Intel Mac OS X 10_8) AppleWebKit/536.25 (KHTML, like Gecko) Version/6.0 Safari/536.25\nMozilla/4.0 (Windows 95;US) Opera 3.62 [en]\nMozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC) Opera 6.0 [en]\nMozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.01 [en]\nMozilla/4.0 (compatible; MSIE 5.0; Windows ME) Opera 6.03 [en]\nMozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.05 [en]\nMozilla/4.0 (compatible; MSIE 5.0; Windows XP) Opera 6.06 [en]\nMozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.06 [en]\nOpera 7.11\nOpera/7.23 (Windows NT 5.0; U) [en]\nOpera/7.52 (Windows NT 5.1; U) [en]\nOpera/7.53 (Windows NT 5.0; U) [en]\nOpera/7.54 (Windows NT 5.0; U) [en]\nOpera/7.54 (Windows NT 5.1; U) [en]\nOpera/7.54 (Windows NT 5.1; U)\nOpera/7.54 (X11; Linux i686; U) [en]\nOpera/7.54 (X11; Linux i686; U) [sv]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.11 [en]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.21 [pt-BR]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.22 [en]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.23 [en]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.23 [en]\nMozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686) Opera 7.23 [en]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.23 [fr]\nMozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC) Opera 7.50 [en]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.53 [en]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.53 [en]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.54 [en]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.54 [en]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.54 [en]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.54u1 [en]\nOpera/7.54 (Windows 98; U) [en]\nMozilla/4.78 (Windows NT 5.1; U) Opera 7.23 [en]\nMozilla/5.0 (Windows NT 5.0; U) Opera 7.54 [en]\nMozilla/4.0 (compatible; MSIE 6.0; X11; OpenBSD i386) Opera 7.54 [en]\nOpera/8.0 (X11; Linux i686; U; en)\nOpera/8.01 (Windows ME; U; en)\nOpera/8.01 (Windows NT 5.1; U; en)\nMozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC Mac OS X; en) Opera 8.01\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.01\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; tr) Opera 8.02\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.02\nOpera/8.5 (Windows NT 5.0; U; en)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.5\nOpera/8.51 (Windows NT 5.1; U; en)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.51\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; de) Opera 8.52\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; pl) Opera 8.52\nMozilla/5.0 (Windows NT 5.1; U; en) Opera 8.52\nOpera/8.53 (Windows NT 5.1; U; en)\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.53\nOpera/8.54 (Windows NT 5.1; U; en)\nOpera/8.54 (Windows NT 5.0; U; en)\nMozilla/5.0 (X11; Linux i686; U; cs) Opera 8.54\nMozilla/4.0 (compatible; MSIE 6.0; KDDI-SA39) Opera 8.60 [en]\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 9.00\nMozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.00\nOpera/9.00 (Windows NT 5.1; U; en)\nOpera/9.0 (Windows NT 5.1; U; en)\nOpera/9.00 (Macintosh; PPC Mac OS X; U; en)\nOpera/9.02 (Macintosh; PPC Mac OS X; U; en)\nOpera/9.02 (Windows NT 5.1; U; zh-tw)\nOpera/9.10 (Windows NT 6.0; U; en)\nOpera/9.21 (Windows NT 6.0; U; en)\nOpera/9.22 (Windows NT 5.1; U; en)\nOpera/9.23 (Windows NT 5.1; U; en)\nOpera/9.23 (Windows ME; U; en)\nOpera/9.26 (Windows NT 5.1; U; en)\nOpera/9.51 (Windows NT 5.1; U; en)\nOpera/9.52 (Macintosh; Intel Mac OS X; U; en)\nOpera/9.52 (Windows NT 5.1; U; en)\nOpera/9.60 (Macintosh; Intel Mac OS X; U; en) Presto/2.1.1\nOpera/9.60 (Windows NT 5.1; U; en) Presto/2.1.1\nOpera/9.61 (Windows NT 5.1; U; en) Presto/2.1.1\nOpera/9.62 (Windows NT 5.1; U; en) Presto/2.1.1\nMozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686; en) Opera 10.10\nOpera/9.80 (Windows NT 6.1; U; en) Presto/2.9.168 Version/11.50\nOpera/9.80 (Windows NT 6.1; U; en) Presto/2.10.229 Version/11.60\nOpera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.10.289 Version/12.00" }, { "path": "miner/config.go", "content": "package miner\n\nimport (\n\t\"context\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"github.com/hunterhug/marmot/util\"\n)\n\n// Worker is the main object to sent http request and return result of response\ntype Worker struct {\n\t// In order fast chain func call I put the basic config below\n\t*Request\n\t*Response\n\n\t// Which url we want\n\tUrl string\n\n\t// Get,Post method\n\tMethod string\n\n\t// Our Client\n\tClient *http.Client\n\n\t// Wait Sleep Time\n\tWait int\n\n\t// Worker proxy ip, just for user to record their proxy ip, default: localhost\n\tIp string\n\n\t// AOP like Java\n\tCtx context.Context\n\tBeforeAction func(context.Context, *Worker)\n\tAfterAction func(context.Context, *Worker)\n\n\t// Http header\n\tHeader http.Header\n\n\t// DownloadFileReadMB is the size of the file read in MB\n\tDownloadFileReadMB int64\n\n\t// Mux lock\n\tmux sync.RWMutex\n}\n\ntype Request struct {\n\tData url.Values // Sent by form data\n\tFileName string // FileName which sent to remote\n\tFileFormName string // File Form Name which sent to remote\n\tBData []byte // Sent by binary data, can together with File\n\tRequest *http.Request // Debug\n}\n\ntype Response struct {\n\tResponse *http.Response // Debug\n\tRaw []byte // Raw data we get\n\tResponseStatusCode int // The last url response code, such as 404\n}\n\n// SetHeader Java Bean Chain pattern\nfunc (worker *Worker) SetHeader(header http.Header) *Worker {\n\tworker.Header = header\n\treturn worker\n}\n\n// SetHeader Default Worker SetHeader!\nfunc SetHeader(header http.Header) *Worker {\n\treturn DefaultWorker.SetHeader(header)\n}\n\nfunc (worker *Worker) SetHeaderParam(k, v string) *Worker {\n\tworker.Header.Set(k, v)\n\treturn worker\n}\n\nfunc SetHeaderParam(k, v string) *Worker {\n\treturn DefaultWorker.SetHeaderParam(k, v)\n}\n\nfunc (worker *Worker) SetCookie(v string) *Worker {\n\tworker.SetHeaderParam(\"Cookie\", v)\n\treturn worker\n}\n\nfunc SetCookie(v string) *Worker {\n\treturn DefaultWorker.SetCookie(v)\n}\n\n// SetCookieByFile Set Cookie by file.\nfunc (worker *Worker) SetCookieByFile(file string) (*Worker, error) {\n\thaha, err := util.ReadFromFile(file)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcookie := string(haha)\n\tcookie = strings.Replace(cookie, \" \", \"\", -1)\n\tcookie = strings.Replace(cookie, \"\\n\", \"\", -1)\n\tcookie = strings.Replace(cookie, \"\\r\", \"\", -1)\n\treturn worker.SetCookie(cookie), nil\n}\n\nfunc SetCookieByFile(file string) (*Worker, error) {\n\treturn DefaultWorker.SetCookieByFile(file)\n}\n\nfunc (worker *Worker) SetUa(ua string) *Worker {\n\tworker.Header.Set(\"User-Agent\", ua)\n\treturn worker\n}\n\nfunc SetUa(ua string) *Worker {\n\treturn DefaultWorker.SetUa(ua)\n}\n\nfunc (worker *Worker) SetRefer(refer string) *Worker {\n\tworker.Header.Set(\"Referer\", refer)\n\treturn worker\n}\n\nfunc SetRefer(refer string) *Worker {\n\treturn DefaultWorker.SetRefer(refer)\n}\n\nfunc (worker *Worker) SetHost(host string) *Worker {\n\tworker.Header.Set(\"Host\", host)\n\treturn worker\n}\n\n// SetUrl at the same time SetHost\nfunc (worker *Worker) SetUrl(url string) *Worker {\n\tworker.Url = url\n\ttemp := strings.Split(url, \"//\")\n\tif len(temp) >= 2 {\n\t\tworker.SetHost(strings.Split(temp[1], \"/\")[0])\n\t}\n\treturn worker\n}\n\nfunc SetUrl(url string) *Worker {\n\treturn DefaultWorker.SetUrl(url)\n}\n\nfunc (worker *Worker) SetFileInfo(fileName, fileFormName string) *Worker {\n\tworker.FileName = fileName\n\tworker.FileFormName = fileFormName\n\treturn worker\n}\n\nfunc SetFileInfo(fileName, fileFormName string) *Worker {\n\treturn DefaultWorker.SetFileInfo(fileName, fileFormName)\n}\n\nfunc (worker *Worker) SetMethod(method string) *Worker {\n\ttemp := GET\n\tswitch strings.ToUpper(method) {\n\tcase GET:\n\t\ttemp = GET\n\tcase POST:\n\t\ttemp = POST\n\tcase POSTFILE:\n\t\ttemp = POSTFILE\n\tcase POSTJSON:\n\t\ttemp = POSTJSON\n\tcase POSTXML:\n\t\ttemp = POSTXML\n\tcase PUT:\n\t\ttemp = PUT\n\tcase PUTFILE:\n\t\ttemp = PUTFILE\n\tcase PUTJSON:\n\t\ttemp = PUTJSON\n\tcase PUTXML:\n\t\ttemp = PUTXML\n\tcase DELETE:\n\t\ttemp = DELETE\n\tdefault:\n\t\ttemp = OTHER\n\t}\n\tworker.Method = temp\n\treturn worker\n}\n\nfunc SetMethod(method string) *Worker {\n\treturn DefaultWorker.SetMethod(method)\n}\n\nfunc (worker *Worker) SetWaitTime(num int) *Worker {\n\tif num <= 0 {\n\t\tnum = 1\n\t}\n\tworker.Wait = num\n\treturn worker\n}\n\nfunc SetWaitTime(num int) *Worker {\n\treturn DefaultWorker.SetWaitTime(num)\n}\n\nfunc (worker *Worker) SetDownloadFileReadMB(downloadFileReadMB int64) *Worker {\n\tif downloadFileReadMB <= 0 {\n\t\tdownloadFileReadMB = 5\n\t}\n\tworker.DownloadFileReadMB = downloadFileReadMB\n\treturn worker\n}\n\nfunc SetDownloadFileReadMB(downloadFileReadMB int64) *Worker {\n\treturn DefaultWorker.SetDownloadFileReadMB(downloadFileReadMB)\n}\n\nfunc (worker *Worker) SetBData(data []byte) *Worker {\n\tworker.BData = data\n\treturn worker\n}\n\nfunc SetBData(data []byte) *Worker {\n\treturn DefaultWorker.SetBData(data)\n}\n\nfunc (worker *Worker) SetForm(form url.Values) *Worker {\n\tworker.Data = form\n\treturn worker\n}\n\nfunc SetForm(form url.Values) *Worker {\n\treturn DefaultWorker.SetForm(form)\n}\n\nfunc (worker *Worker) SetFormParam(k, v string) *Worker {\n\tworker.Data.Set(k, v)\n\treturn worker\n}\n\nfunc SetFormParam(k, v string) *Worker {\n\treturn DefaultWorker.SetFormParam(k, v)\n}\n\n// SetContext Set Context so Action can soft\nfunc (worker *Worker) SetContext(ctx context.Context) *Worker {\n\tworker.Ctx = ctx\n\treturn worker\n}\n\nfunc SetContext(ctx context.Context) *Worker {\n\treturn DefaultWorker.SetContext(ctx)\n}\n\nfunc (worker *Worker) SetBeforeAction(fc func(context.Context, *Worker)) *Worker {\n\tworker.BeforeAction = fc\n\treturn worker\n}\n\nfunc SetBeforeAction(fc func(context.Context, *Worker)) *Worker {\n\treturn DefaultWorker.SetBeforeAction(fc)\n}\n\nfunc (worker *Worker) SetAfterAction(fc func(context.Context, *Worker)) *Worker {\n\tworker.AfterAction = fc\n\treturn worker\n}\n\nfunc SetAfterAction(fc func(context.Context, *Worker)) *Worker {\n\treturn DefaultWorker.SetAfterAction(fc)\n}\n\n// Clear data we sent\n// I suggest use Clone() to avoid clear\nfunc (worker *Worker) Clear() *Worker {\n\tworker.Request = newRequest()\n\tworker.Response = new(Response)\n\treturn worker\n}\n\nfunc Clear() *Worker {\n\treturn DefaultWorker.Clear()\n}\n\n// ClearAll All clear include header\nfunc (worker *Worker) ClearAll() *Worker {\n\tworker.Clear()\n\tworker.Header = http.Header{}\n\treturn worker\n}\n\nfunc ClearAll() *Worker {\n\treturn DefaultWorker.ClearAll()\n}\n\n// ClearCookie Clear Cookie\nfunc (worker *Worker) ClearCookie() *Worker {\n\tworker.Header.Del(\"Cookie\")\n\treturn worker\n}\n\nfunc ClearCookie() *Worker {\n\treturn DefaultWorker.ClearCookie()\n}\n\n// GetCookies Get Cookies\nfunc (worker *Worker) GetCookies() []*http.Cookie {\n\tif worker.Response != nil && worker.Response.Response != nil {\n\t\treturn worker.Response.Response.Cookies()\n\t} else {\n\t\treturn []*http.Cookie{}\n\t}\n}\n\n// GetResponseStatusCode Get ResponseStatusCode\nfunc (worker *Worker) GetResponseStatusCode() int {\n\tif worker.Response != nil && worker.Response.Response != nil {\n\t\treturn worker.Response.ResponseStatusCode\n\t} else {\n\t\treturn 0\n\t}\n}\n\nfunc GetCookies() []*http.Cookie {\n\treturn DefaultWorker.GetCookies()\n}\n\nfunc GetResponseStatusCode() int {\n\treturn DefaultWorker.GetResponseStatusCode()\n}\n" }, { "path": "miner/cons.go", "content": "package miner\n\nimport \"net/http\"\n\nconst (\n\tVERSION = \"1.0.13\"\n\n\t// GET HTTP method\n\tGET = \"GET\"\n\tPOST = \"POST\"\n\tPOSTJSON = \"POSTJSON\"\n\tPOSTXML = \"POSTXML\"\n\tPOSTFILE = \"POSTFILE\"\n\tPUT = \"PUT\"\n\tPUTJSON = \"PUTJSON\"\n\tPUTXML = \"PUTXML\"\n\tPUTFILE = \"PUTFILE\"\n\tDELETE = \"DELETE\"\n\tOTHER = \"OTHER\" // this stand for you can use other method this lib not own.\n\n\t// HTTPFORMContentType HTTP content type\n\tHTTPFORMContentType = \"application/x-www-form-urlencoded\"\n\tHTTPJSONContentType = \"application/json\"\n\tHTTPXMLContentType = \"text/xml\"\n\tHTTPFILEContentType = \"multipart/form-data\"\n)\n\nvar (\n\t// Browser User-Agent, Our default Http ua header!\n\tourLoveUa = \"Marmot+\" + VERSION + \"+github:hunterhug\"\n\n\tDefaultHeader = map[string][]string{\n\t\t\"User-Agent\": {\n\t\t\tourLoveUa,\n\t\t},\n\t}\n\n\t// DefaultTimeOut http get and post No timeout\n\tDefaultTimeOut = 0\n)\n\n// SetGlobalTimeout Set global timeout, it can only by this way!\nfunc SetGlobalTimeout(num int) {\n\tDefaultTimeOut = num\n}\n\nfunc SetDefaultTimeOut(num int) {\n\tDefaultTimeOut = num\n}\n\n// MergeCookie Merge Cookie, not use\nfunc MergeCookie(before []*http.Cookie, after []*http.Cookie) []*http.Cookie {\n\tcs := make(map[string]*http.Cookie)\n\n\tfor _, b := range before {\n\t\tcs[b.Name] = b\n\t}\n\n\tfor _, a := range after {\n\t\tif a.Value != \"\" {\n\t\t\tcs[a.Name] = a\n\t\t}\n\t}\n\n\tres := make([]*http.Cookie, 0, len(cs))\n\n\tfor _, q := range cs {\n\t\tres = append(res, q)\n\n\t}\n\n\treturn res\n\n}\n\n// CloneHeader Clone a header, If not exist Ua, Set our Ua!\nfunc CloneHeader(h map[string][]string) map[string][]string {\n\tif h == nil || len(h) == 0 {\n\t\th = DefaultHeader\n\t\treturn h\n\t}\n\n\tif len(h[\"User-Agent\"]) == 0 {\n\t\th[\"User-Agent\"] = []string{ourLoveUa}\n\t}\n\treturn CopyM(h)\n}\n" }, { "path": "miner/default.go", "content": "package miner\n\nimport \"net/http\"\n\n// DefaultWorker Global Worker\nvar DefaultWorker *Worker\n\nfunc init() {\n\tUaInit()\n\n\t// New a Worker\n\tworker := new(Worker)\n\tworker.Request = newRequest()\n\tworker.Header = http.Header{}\n\tworker.Response = new(Response)\n\tworker.Client = Client\n\n\t// Global Worker!\n\tDefaultWorker = worker\n}\n\n// ToString This make effect only your Worker exec serial! Attention!\n// Change Your Raw data To string\nfunc ToString() string {\n\treturn DefaultWorker.ToString()\n}\n\n// JsonToString This make effect only your Worker exec serial! Attention!\n// Change Your JSON like Raw data to string\nfunc JsonToString() (string, error) {\n\treturn DefaultWorker.JsonToString()\n}\n\nfunc Clone() *Worker {\n\treturn DefaultWorker.Clone()\n}\n\nfunc Get() (body []byte, e error) {\n\treturn DefaultWorker.Get()\n}\n\nfunc Delete() (body []byte, e error) {\n\treturn DefaultWorker.Delete()\n}\n\nfunc Go() (body []byte, e error) {\n\treturn DefaultWorker.Go()\n}\n\nfunc GoByMethod(method string) (body []byte, e error) {\n\treturn DefaultWorker.SetMethod(method).Go()\n}\n\nfunc OtherGo(method, contentType string) (body []byte, e error) {\n\treturn DefaultWorker.OtherGo(method, contentType)\n}\n\nfunc Post() (body []byte, e error) {\n\treturn DefaultWorker.Post()\n}\n\nfunc PostJSON() (body []byte, e error) {\n\treturn DefaultWorker.PostJSON()\n}\n\nfunc PostFILE() (body []byte, e error) {\n\treturn DefaultWorker.PostFILE()\n}\n\nfunc PostXML() (body []byte, e error) {\n\treturn DefaultWorker.PostXML()\n}\n\nfunc Put() (body []byte, e error) {\n\treturn DefaultWorker.Put()\n}\nfunc PutJSON() (body []byte, e error) {\n\treturn DefaultWorker.PutJSON()\n}\n\nfunc PutFILE() (body []byte, e error) {\n\treturn DefaultWorker.PutFILE()\n}\n\nfunc PutXML() (body []byte, e error) {\n\treturn DefaultWorker.PutXML()\n}\n" }, { "path": "miner/doc.go", "content": "/*\n\tAll right reserved https://github.com/hunterhug/marmot at 2016-2022\n\tAttribution-NonCommercial-NoDerivatives 4.0 International\n\tNotice: The following code's copyright by hunterhug, Please do not spread and modify.\n\tYou can use it for education only but can't make profits for any companies and individuals!\n*/\n\n/*\nPackage miner is the core of this project, use to request for http api.\n\nExample:\n\n\tpackage main\n\n\timport (\n\t\t\"fmt\"\n\t\t\"github.com/hunterhug/marmot/miner\"\n\t)\n\n\tfunc main() {\n\t\tminer.SetLogLevel(miner.DEBUG)\n\n\t\t// Use Default Worker, You can Also New One:\n\t\t//worker, _ := miner.New(nil)\n\t\t//worker = miner.NewWorkerWithNoProxy()\n\t\t//worker = miner.NewAPI()\n\t\t//worker, _ = miner.NewWorkerWithProxy(\"socks5://127.0.0.1:1080\")\n\t\tworker := miner.Clone()\n\t\t_, err := worker.SetUrl(\"https://www.bing.com\").Go()\n\t\tif err != nil {\n\t\t\tfmt.Println(err.Error())\n\t\t} else {\n\t\t\tfmt.Println(worker.ToString())\n\t\t}\n\t}\n*/\npackage miner\n" }, { "path": "miner/log.go", "content": "package miner\n\nimport (\n\t\"github.com/hunterhug/golog/v2\"\n)\n\ntype Level = golog.Level\n\nconst (\n\tDEBUG Level = golog.DebugLevel\n\tINFO Level = golog.InfoLevel\n\tWARN Level = golog.WarnLevel\n\tERROR Level = golog.ErrorLevel\n)\n\n// Logger Global logger config for debug\nvar (\n\tLogger = golog.New()\n)\n\nfunc init() {\n\tLogger.InitLogger()\n}\n\n// SetLogLevel Set log level\nfunc SetLogLevel(level Level) {\n\tLogger.SetLevel(level)\n\tLogger.InitLogger()\n}\n\n// Log Return global log\nfunc Log() golog.LoggerInterface {\n\treturn Logger\n}\n" }, { "path": "miner/multworker.go", "content": "package miner\n\nimport (\n\t\"sync\"\n)\n\n// Pool for many Worker, every Worker can only serial execution\nvar Pool = &_Workers{ws: make(map[string]*Worker)}\n\ntype _Workers struct {\n\tmux sync.RWMutex\n\tws map[string]*Worker\n}\n\nfunc (pool *_Workers) Get(name string) (b *Worker, ok bool) {\n\tpool.mux.RLock()\n\tb, ok = pool.ws[name]\n\tpool.mux.RUnlock()\n\treturn\n}\n\nfunc (pool *_Workers) Set(name string, b *Worker) {\n\tpool.mux.Lock()\n\tpool.ws[name] = b\n\tpool.mux.Unlock()\n\treturn\n}\n\nfunc (pool *_Workers) Delete(name string) {\n\tpool.mux.Lock()\n\tdelete(pool.ws, name)\n\tpool.mux.Unlock()\n\treturn\n}\n" }, { "path": "miner/ua.go", "content": "package miner\n\nimport (\n\t\"github.com/hunterhug/marmot/util\"\n\t\"math/rand\"\n\t\"strings\"\n)\n\n// Ua Global User-Agent provide\nvar Ua = map[int]string{}\n\n// UaInit User-Agent init\nfunc UaInit() {\n\tUa = map[int]string{\n\t\t0: \"Mozilla/5.0 (SymbianOS/9.1; U; [en]; SymbianOS/91 Series60/3.0) AppleWebkit/413 (KHTML, like Gecko) Safari/413\",\n\t\t1: \"Mozilla/5.0 (SymbianOS/9.1; U; [en]; Series60/3.0 NokiaE60/4.06.0) AppleWebKit/413 (KHTML, like Gecko) Safari/413\",\n\t\t2: \"Mozilla/5.0 (SymbianOS/9.2; U; Series60/3.1 NokiaN95/10.0.018; Profile/MIDP-2.0 Configuration/CLDC-1.1) AppleWebKit/413 (KHTML, like Gecko) Safari/413\",\n\t\t3: \"Mozilla/5.0 (SymbianOS/9.3; U; Series60/3.2 NokiaE75-1/110.48.125 Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413\",\n\t\t4: \"Mozilla/5.0 (SymbianOS/9.4; U; Series60/5.0 Nokia5800d-1/21.0.025; Profile/MIDP-2.1 Configuration/CLDC-1.1 ) AppleWebKit/413 (KHTML, like Gecko) Safari/413\",\n\t\t5: \"Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/12.0.024; Profile/MIDP-2.1 Configuration/CLDC-1.1; en-us) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.12344\",\n\t\t6: \"Mozilla/4.0 (compatible; MSIE 4.0; MSN 2.5; Windows 95)\",\n\t\t7: \"Mozilla/4.0 (compatible; MSIE 4.0; Windows 95)\",\n\t\t8: \"Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; MSN 2.5; Windows 98)\",\n\t\t9: \"Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; Windows 95)\",\n\t\t10: \"Mozilla/4.0 (compatible; MSIE 4.01; MSN 2.5; Windows 98)\",\n\t\t11: \"Mozilla/4.0 (compatible; MSIE 4.01; Windows 95)\",\n\t\t12: \"Mozilla/4.0 (compatible; MSIE 4.01; Windows 95; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.)\",\n\t\t13: \"Mozilla/4.0 (compatible; MSIE 4.01; Windows 98)\",\n\t\t14: \"Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; BIGLOBE)\",\n\t\t15: \"Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; canoncopyer)\",\n\t\t16: \"Mozilla/4.0 (compatible; MSIE 4.01; Windows 98; Compaq)\",\n\t\t17: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95)\",\n\t\t18: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt)\",\n\t\t19: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt; i-CABLE)\",\n\t\t20: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; DigExt; ocnie5-1)\",\n\t\t21: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 95; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.)\",\n\t\t22: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 98)\",\n\t\t23: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; CNETHomeBuild051099)\",\n\t\t24: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt)\",\n\t\t25: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt; ocnie5-1)\",\n\t\t26: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; wn_ie5_en_v1)\",\n\t\t27: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; Yahoo! enPAN Version Windows 95/NT CD-ROM Edition 1.0.; DigExt)\",\n\t\t28: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows NT)\",\n\t\t29: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)\",\n\t\t30: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 98; DigExt)\",\n\t\t31: \"Mozilla/4.0 (compatible; MSIE 5.01; MSN 2.5; Windows 98)\",\n\t\t32: \"Mozilla/4.0 (compatible; MSIE 5.01; Windows 95)\",\n\t\t33: \"Mozilla/4.0 (compatible; MSIE 5.01; Windows 98)\",\n\t\t34: \"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)\",\n\t\t35: \"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; DigExt)\",\n\t\t36: \"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; TUCOWS)\",\n\t\t37: \"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT)\",\n\t\t38: \"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0; .NET CLR 1.1.4322)\",\n\t\t39: \"Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC)\",\n\t\t40: \"Mozilla/4.0 (compatible; MSIE 5.16; Mac_PowerPC)\",\n\t\t41: \"Mozilla/4.0 (compatible; MSIE 5.17; Mac_PowerPC)\",\n\t\t42: \"Mozilla/4.0 (compatible; MSIE 5.22; Mac_PowerPC)\",\n\t\t43: \"Mozilla/4.0 (compatible; MSIE 5.23; Mac_PowerPC)\",\n\t\t44: \"Mozilla/4.0 (compatible; MSIE 5.5; Windows 95)\",\n\t\t45: \"Mozilla/4.0 (compatible; MSIE 5.5; Windows 98)\",\n\t\t46: \"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)\",\n\t\t47: \"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705)\",\n\t\t48: \"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.1.4322)\",\n\t\t49: \"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; by TSG)\",\n\t\t50: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)\",\n\t\t51: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\",\n\t\t52: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)\",\n\t\t53: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)\",\n\t\t54: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.40607)\",\n\t\t55: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.40607; .NET CLR 1.0.3705)\",\n\t\t56: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; NOKTURNAL KICKS ASS)\",\n\t\t57: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; FDM;\",\n\t\t58: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Maxthon; .NET CLR 1.1.4322; .NET CLR 2.0.41115)\",\n\t\t59: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)\",\n\t\t60: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1\",\n\t\t61: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)\",\n\t\t62: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)\",\n\t\t63: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1\",\n\t\t64: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)\",\n\t\t65: \"Mozilla/4.0 (compatible; MSIE 6.0; AOL 9.0; Windows NT 5.1; iebar; .NET CLR 1.0.3705)\",\n\t\t66: \"Mozilla/4.0 (compatible; MSIE 6.0; Win32);\",\n\t\t67: \"Mozilla/4.0 (compatible; MSIE 6.0; Win32); .NET CLR 1.0.3705)\",\n\t\t68: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Hotbar 4.4.6.0)\",\n\t\t69: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; Win 9x 4.90)\",\n\t\t70: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0)\",\n\t\t71: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 4.0; BVG\",\n\t\t72: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0)\",\n\t\t73: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\",\n\t\t74: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322)\",\n\t\t75: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; FDM)\",\n\t\t76: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; {FF0C8E09-3C86-44CB-834A-B8CEEC80A1D7}; iOpus-I-M)\",\n\t\t77: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; i-Nav 3.0.1.0F; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\",\n\t\t78: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; MathPlayer 2.0; .NET CLR 1.1.4322)\",\n\t\t79: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; Maxthon; .NET CLR 1.1.4322)\",\n\t\t80: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; T312461)\",\n\t\t81: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0;)\",\n\t\t82: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)\",\n\t\t83: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1);\",\n\t\t84: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705)\",\n\t\t85: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\",\n\t\t86: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)\",\n\t\t87: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 1.0.3705)\",\n\t\t88: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 2.0.40607)\",\n\t\t89: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Alexa Toolbar)\",\n\t\t90: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; BrowserBob)\",\n\t\t91: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; DFO-MPO Internet Explorer 6.0)\",\n\t\t92: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ENGINE; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT))\",\n\t\t93: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; ESB{F40811EE-DF17-4BC9-8785-B362ABF34098}; .NET CLR 1.1.4322)\",\n\t\t94: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FDM)\",\n\t\t95: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FTDv3 Browser; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\",\n\t\t96: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; .NET CLR 1.1.4322; .NET CLR 2.0.40607)\",\n\t\t97: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FunWebProducts; AtHome033)\",\n\t\t98: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; HCI0449; .NET CLR 1.0.3705)\",\n\t\t99: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; i-NavFourF; .NET CLR 1.1.4322)\",\n\t\t100: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon;\",\n\t\t101: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon; .NET CLR 1.1.4322)\",\n\t\t102: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2; .NET CLR 1.1.4322)\",\n\t\t103: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; MyIE2; Maxthon; .NET CLR 1.1.4322)\",\n\t\t104: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Q312461; FunWebProducts; .NET CLR 1.1.4322)\",\n\t\t105: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Woningstichting Den Helder; .NET CLR 1.0.3705)\",\n\t\t106: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322)\",\n\t\t107: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.41115)\",\n\t\t108: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; MyIE2; .NET CLR 1.1.4322)\",\n\t\t109: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; MyIE2; Maxthon; .NET CLR 1.1.4322)\",\n\t\t110: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows XP)\",\n\t\t111: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1)\",\n\t\t112: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.0.3705; .NET CLR 2.0.50727; .NET CLR 1.1.4322)\",\n\t\t113: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)\",\n\t\t114: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1)\",\n\t\t115: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)\",\n\t\t116: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)\",\n\t\t117: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)\",\n\t\t118: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; InfoPath.1)\",\n\t\t119: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)\",\n\t\t120: \"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.04506.648)\",\n\t\t121: \"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; InfoPath.1\",\n\t\t122: \"Mozilla/4.0 (compatible; GoogleToolbar 5.0.2124.2070; Windows 6.0; MSIE 8.0.6001.18241)\",\n\t\t123: \"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; EasyBits Go v1.0; InfoPath.1; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)\",\n\t\t124: \"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)\",\n\t\t125: \"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; Sleipnir/2.9.8)\",\n\t\t126: \"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)\",\n\t\t127: \"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)\",\n\t\t128: \"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)\",\n\t\t129: \"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)\",\n\t\t130: \"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)\",\n\t\t131: \"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)\",\n\t\t132: \"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; ARM; Trident/6.0)\",\n\t\t133: \"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13\",\n\t\t134: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/1.0.154.48 Safari/525.19\",\n\t\t135: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/530.5 (KHTML, like Gecko) Chrome/2.0.172.33 Safari/530.5\",\n\t\t136: \"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/532.0 (KHTML, like Gecko) Chrome/3.0.195.38 Safari/532.0\",\n\t\t137: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-US) AppleWebKit/533.4 (KHTML, like Gecko) Chrome/5.0.375.55 Safari/533.4\",\n\t\t138: \"Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3\",\n\t\t139: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.55 Safari/534.3\",\n\t\t140: \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.517.43 Safari/534.7\",\n\t\t141: \"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10 ChromePlus/1.5.2.0\",\n\t\t142: \"Mozilla/5.0 (en-us) AppleWebKit/534.14 (KHTML, like Gecko; Google Wireless Transcoder) Chrome/9.0.597 Safari/534.14\",\n\t\t143: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.151 Safari/534.16\",\n\t\t144: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.71 Safari/534.24\",\n\t\t145: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.24 (KHTML, like Gecko) Iron/11.0.700.2 Chrome/11.0.700.2 Safari/534.24\",\n\t\t146: \"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.65 Safari/534.24\",\n\t\t147: \"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30 ChromePlus/1.6.3.1\",\n\t\t148: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.112 Safari/534.30\",\n\t\t149: \"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.107 Safari/535.1\",\n\t\t150: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) RockMelt/0.9.64.361 Chrome/13.0.782.218 Safari/535.1\",\n\t\t151: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1\",\n\t\t152: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.220 Safari/535.1\",\n\t\t153: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1\",\n\t\t154: \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.202 Safari/535.1\",\n\t\t155: \"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2\",\n\t\t156: \"Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.2 (KHTML, like Gecko) Ubuntu/10.04 Chromium/15.0.874.106 Chrome/15.0.874.106 Safari/535.2\",\n\t\t157: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.106 Safari/535.2\",\n\t\t158: \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7\",\n\t\t159: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7\",\n\t\t160: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; SLCC1; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30618; Lunascape 4.7.3)\",\n\t\t161: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; Lunascape 5.0 alpha2)\",\n\t\t162: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; Lunascape 5.0 alpha2)\",\n\t\t163: \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506; Tablet PC 2.0; Lunascape 5.0 alpha2)\",\n\t\t164: \"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; MAGW; .NET4.0C; Lunascape 6.5.8.24780)\",\n\t\t165: \"Mozilla/2.02 (Macintosh; I; PPC)\",\n\t\t166: \"Mozilla/3.01 (Macintosh; I; PPC)\",\n\t\t167: \"Mozilla/4.01 (Macintosh; I; PPC)\",\n\t\t168: \"Mozilla/4.03 [en]C-IMS (Win95; I)\",\n\t\t169: \"Mozilla/4.04 [en] (Win95; I ;Nav)\",\n\t\t170: \"Mozilla/4.04 [en] (Win95; I)\",\n\t\t171: \"Mozilla/4.04 [en] (WinNT; I ;Nav)\",\n\t\t172: \"Mozilla/4.04 [en] (WinNT; I)\",\n\t\t173: \"Mozilla/4.04 [en] (Macintosh; I; PPC Nav)\",\n\t\t174: \"Mozilla/4.04 [en] (X11; I; SunOS 5.5 sun4u)\",\n\t\t175: \"Mozilla/4.05 [en] (Win95; I)\",\n\t\t176: \"Mozilla/4.05 (Macintosh; I; PPC)\",\n\t\t177: \"Mozilla/4.06 [en] (Win98; I)\",\n\t\t178: \"Mozilla/4.06 [en] (Macintosh; I; PPC)\",\n\t\t179: \"Mozilla/4.08 (Macintosh; I; PPC)\",\n\t\t180: \"Mozilla/4.5 [en] (Win95; I)\",\n\t\t181: \"Mozilla/4.5 [en] (Win98; I)\",\n\t\t182: \"Mozilla/4.5 [en] (WinNT; I)\",\n\t\t183: \"Mozilla/4.5 (Macintosh; I; PPC)\",\n\t\t184: \"Mozilla/4.51 [en] (Win95; I)\",\n\t\t185: \"Mozilla/4.51 [en] (Win98; I)\",\n\t\t186: \"Mozilla/4.51 [en] (WinNT; I)\",\n\t\t187: \"Mozilla/4.51 [en] (X11; I; SunOS 5.8 sun4u)\",\n\t\t188: \"Mozilla/4.6 [en] (Win95; I)\",\n\t\t189: \"Mozilla/4.6 [en] (Win98; I)\",\n\t\t190: \"Mozilla/4.6 [en] (WinNT; I)\",\n\t\t191: \"Mozilla/4.6 [en] (WinNT; I)\",\n\t\t192: \"Mozilla/4.7 [en] (WinNT; I)\",\n\t\t193: \"Mozilla/4.7 [en] (Win95; I)\",\n\t\t194: \"Mozilla/4.7 [en] (Win98; I)\",\n\t\t195: \"Mozilla/4.7 [en] (WinNT; I)\",\n\t\t196: \"Mozilla/4.7 [en] (WinNT; I)\",\n\t\t197: \"Mozilla/4.7 [en] (WinNT; U)\",\n\t\t198: \"Mozilla/4.7 [en] (Macintosh; I; PPC)\",\n\t\t199: \"Mozilla/4.76 [en_jp] (X11; U; SunOS 5.8 sun4u)\",\n\t\t200: \"Mozilla/4.76 [en] (X11; U; SunOS 5.8 sun4u)\",\n\t\t201: \"Mozilla/4.78 [en] (X11; U; SunOS 5.9 sun4u)\",\n\t\t202: \"Mozilla/4.8 [en] (X11; U; SunOS 5.7 sun4u)\",\n\t\t203: \"Mozilla/5.0 (Windows; U; Win98; en-JP; m18) Gecko/20001108 Netscape6/6.0\",\n\t\t204: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; en-JP; m18) Gecko/20010131 Netscape6/6.01\",\n\t\t205: \"Mozilla/5.0 (Windows; U; Win 9x 4.90; en-JP; rv:0.9.4) Gecko/20011128 Netscape6/6.2.1\",\n\t\t206: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:0.9.4.1) Gecko/20020508 Netscape6/6.2.3\",\n\t\t207: \"Mozilla/5.0 (Macintosh; N; PPC; en-JP; macen-pub12) Gecko/20001108 Netscape6/6.0\",\n\t\t208: \"Mozilla/5.0 (Macintosh; U; PPC; en-JP; rv:0.9.2) Gecko/20010726 Netscape6/6.1\",\n\t\t209: \"Mozilla/5.0 (Macintosh; U; PPC; en-JP; rv:0.9.4) Gecko/20011022 Netscape6/6.2\",\n\t\t210: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-US; rv:0.9.4.1) Gecko/20020315 Netscape6/6.2.2\",\n\t\t211: \"Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.0.2) Gecko/20030208 Netscape/7.02\",\n\t\t212: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)\",\n\t\t213: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.4) Gecko/20030624 Netscape/7.1 (ax)\",\n\t\t214: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.2) Gecko/20040804 Netscape/7.2 (ax)\",\n\t\t215: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.2) Gecko/20040805 Netscape/7.2\",\n\t\t216: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-JP; rv:1.0.2) Gecko/20021120 Netscape/7.01\",\n\t\t217: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-JP; rv:1.4) Gecko/20030624 Netscape/7.1\",\n\t\t218: \"Mozilla/5.0 (X11; U; SunOS sun4u; en-JP; rv:1.0.1) Gecko/20020921 Netscape/7.0\",\n\t\t219: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; en-JP; rv:1.5) Gecko/20031007\",\n\t\t220: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.6) Gecko/20040113\",\n\t\t221: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040616\",\n\t\t222: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.6) Gecko/20040113\",\n\t\t223: \"Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.2.1) Gecko/20030225\",\n\t\t224: \"Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.4.1) Gecko/20031030\",\n\t\t225: \"Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.1) Gecko/20040805\",\n\t\t226: \"Mozilla/5.0 (X11; U; SunOS sun4u; en-JP; rv:1.4) Gecko/20040414\",\n\t\t227: \"Mozilla/5.0 (X11; U; Linux i686; rv:1.7.3) Gecko/20040913 Firefox/0.10\",\n\t\t228: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) Gecko/20040913 Firefox/0.10\",\n\t\t229: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20040913 Firefox/0.10\",\n\t\t230: \"Mozilla/5.0 (X11; U; Linux i686; rv:1.7.3) Gecko/20041001 Firefox/0.10.1\",\n\t\t231: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; rv:1.7.3) Gecko/20041001 Firefox/0.10.1\",\n\t\t232: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; rv:1.7.3) Gecko/20041001 Firefox/0.10.1\",\n\t\t233: \"Mozilla/5.0 (Windows; U; Windows NT 5.2; rv:1.7.3) Gecko/20041001 Firefox/0.10.1\",\n\t\t234: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.6) Gecko/20040206 Firefox/0.8\",\n\t\t235: \"Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.6) Gecko/20040206 Firefox/0.8\",\n\t\t236: \"Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.6) Gecko/20040207 Firefox/0.8\",\n\t\t237: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.6) Gecko/20040206 Firefox/0.8\",\n\t\t238: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040614 Firefox/0.9\",\n\t\t239: \"Mozilla/5.0 (X11; U; Linux i686; es-ES; rv:1.7) Gecko/20040708 Firefox/0.9\",\n\t\t240: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040707 Firefox/0.9.2\",\n\t\t241: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/0.9.2 StumbleUpon/1.994\",\n\t\t242: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-JP; rv:1.7) Gecko/20040803 Firefox/0.9.3\",\n\t\t243: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3\",\n\t\t244: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3\",\n\t\t245: \"Mozilla/5.0 (Windows; U; Win98; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3\",\n\t\t246: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7) Gecko/20040803 Firefox/0.9.3\",\n\t\t247: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20040803 Firefox/0.9.3\",\n\t\t248: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7) Gecko/20041013 Firefox/0.9.3 (Ubuntu)\",\n\t\t249: \"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7) Gecko/20041013 Firefox/0.9.3 (Ubuntu)\",\n\t\t250: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\",\n\t\t251: \"Mozilla/5.0 (Windows; U; Win 9x 4.90; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0\",\n\t\t252: \"Mozilla/5.0 (Windows; U; Win 9x 4.90; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0\",\n\t\t253: \"Mozilla/5.0 (Windows; U; Win98; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0\",\n\t\t254: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0\",\n\t\t255: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0\",\n\t\t256: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.7.5) Gecko/20041107 Firefox/1.0\",\n\t\t257: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; en-GB; rv:1.7.5) Gecko/20041110 Firefox/1.0\",\n\t\t258: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\",\n\t\t259: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; fr-FR; rv:1.7.5) Gecko/20041108 Firefox/1.0\",\n\t\t260: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; it-IT; rv:1.7.5) Gecko/20041110 Firefox/1.0\",\n\t\t261: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041107 Firefox/1.0\",\n\t\t262: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041108 Firefox/1.0\",\n\t\t263: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.5) Gecko/20041122 Firefox/1.0\",\n\t\t264: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.7.5) Gecko/20041110 Firefox/1.0\",\n\t\t265: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\",\n\t\t266: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0 StumbleUpon/1.999\",\n\t\t267: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; es-ES; rv:1.7.5) Gecko/20041210 Firefox/1.0\",\n\t\t268: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; fr-FR; rv:1.7.5) Gecko/20041108 Firefox/1.0\",\n\t\t269: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; nl-NL; rv:1.7.5) Gecko/20041202 Firefox/1.0\",\n\t\t270: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; sv-SE; rv:1.7.5) Gecko/20041108 Firefox/1.0\",\n\t\t271: \"Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\",\n\t\t272: \"Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.8b) Gecko/20050212 Firefox/1.0+ (MOOX M3)\",\n\t\t273: \"Mozilla/5.0 (Windows; U; WinNT4.0; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\",\n\t\t274: \"Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.8b) Gecko/20050118 Firefox/1.0+\",\n\t\t275: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8b) Gecko/20050118 Firefox/1.0+\",\n\t\t276: \"Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.7.5) Gecko/20050103 Firefox/1.0\",\n\t\t277: \"Mozilla/5.0 (X11; U; Linux i386; en-US; rv:1.7.5) Gecko/20041109 Firefox/1.0\",\n\t\t278: \"Mozilla/5.0 (X11; U; Linux i686; de-DE; rv:1.7.5) Gecko/20041128 Firefox/1.0 (Debian package 1.0-4)\",\n\t\t279: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\",\n\t\t280: \"Mozilla/5.0 (X11; U; Linux i686; en-JP; rv:1.7.5) Gecko/20041108 Firefox/1.0\",\n\t\t281: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041111 Firefox/1.0\",\n\t\t282: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041111 Firefox/1.0 (Debian package 1.0-2)\",\n\t\t283: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041116 Firefox/1.0 (Ubuntu) (Ubuntu package 1.0-2ubuntu3)\",\n\t\t284: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041119 Firefox/1.0\",\n\t\t285: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041123 Firefox/1.0\",\n\t\t286: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041128 Firefox/1.0 (Debian package 1.0-4)\",\n\t\t287: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041130 Firefox/1.0\",\n\t\t288: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041214 Firefox/1.0 StumbleUpon/1.999\",\n\t\t289: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20041219 Firefox/1.0 (Debian package 1.0+dfsg.1-1)\",\n\t\t290: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.5) Gecko/20050110 Firefox/1.0 (Debian package 1.0+dfsg.1-2)\",\n\t\t291: \"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.7.5) Gecko/20041107 Firefox/1.0\",\n\t\t292: \"Mozilla/5.0 (X11; U; Linux i686; nl-NL; rv:1.7.5) Gecko/20050221 Firefox/1.0 (Ubuntu) (Ubuntu package 1.0+dfsg.1-6ubuntu1)\",\n\t\t293: \"Mozilla/5.0 (X11; U; Linux i686; rv:1.8b) Gecko/20050124 Firefox/1.0+\",\n\t\t294: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.7.6) Gecko/20050306 Firefox/1.0.1 (Debian package 1.0.1-2)\",\n\t\t295: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6) Gecko/20050223 Firefox/1.0.1\",\n\t\t296: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.6) Gecko/20050225 Firefox/1.0.1\",\n\t\t297: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; de-DE; rv:1.7.6) Gecko/20050226 Firefox/1.0.1\",\n\t\t298: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-JP-mac; rv:1.8) Gecko/20051111 Firefox/1.5\",\n\t\t299: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8) Gecko/20051111 Firefox/1.5\",\n\t\t300: \"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8) Gecko/20051111 Firefox/1.5\",\n\t\t301: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.0.1) Gecko/20060111 Firefox/1.5.0.1\",\n\t\t302: \"Mozilla/5.0 (X11; U; Linux i686; en; rv:1.8.0.2) Gecko/20060308 Firefox/1.5.0.2\",\n\t\t303: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3\",\n\t\t304: \"Mozilla/5.0 (X11; U; Linux i686; en; rv:1.8.0.4) Gecko/20060508 Firefox/1.5.0.4\",\n\t\t305: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20\",\n\t\t306: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.12) Gecko/20080201 Firefox/2.0.0.12\",\n\t\t307: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-JP-mac; rv:1.8.1.20) Gecko/20081217 Firefox/2.0.0.20\",\n\t\t308: \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6\",\n\t\t309: \"Mozilla/5.0 (Windows; U; Windows NT 6.0; en; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6 (.NET CLR 3.5.30729)\",\n\t\t310: \"Mozilla/5.0 (Windows; U; Windows NT 6.0; en; rv:1.9.0.17) Gecko/2009122116 Firefox/3.0.17 GTB6 (.NET CLR 3.5.30729)\",\n\t\t311: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-JP-mac; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6\",\n\t\t312: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en-JP-mac; rv:1.9.0.6) Gecko/2009011912 Firefox/3.0.6 GTB5\",\n\t\t313: \"Mozilla/5.0 (Windows NT 6.1; rv:2.0) Gecko/20100101 Firefox/4.0\",\n\t\t314: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:2.0) Gecko/20100101 Firefox/4.0\",\n\t\t315: \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:5.0) Gecko/20100101 Firefox/5.0\",\n\t\t316: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:5.0.1) Gecko/20100101 Firefox/5.0.1\",\n\t\t317: \"Mozilla/5.0 (Windows NT 5.1; rv:6.0) Gecko/20100101 Firefox/6.0\",\n\t\t318: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:6.0.2) Gecko/20100101 Firefox/6.0.2\",\n\t\t319: \"Mozilla/5.0 (Windows NT 6.0; rv:7.0.1) Gecko/20100101 Firefox/7.0.1\",\n\t\t320: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:7.0.1) Gecko/20100101 Firefox/7.0.1\",\n\t\t321: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:8.0.1) Gecko/20100101 Firefox/8.0.1\",\n\t\t322: \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:9.0.1) Gecko/20100101 Firefox/9.0.1\",\n\t\t323: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:9.0.1) Gecko/20100101 Firefox/9.0.1\",\n\t\t324: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.6\",\n\t\t325: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/85.7 (KHTML, like Gecko) Safari/85.7\",\n\t\t326: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/85.8.2 (KHTML, like Gecko) Safari/85.8\",\n\t\t327: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/85.8.2 (KHTML, like Gecko) Safari/85.8.1\",\n\t\t328: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/85.8.5 (KHTML, like Gecko) Safari/85.8\",\n\t\t329: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/103u (KHTML, like Gecko) Safari/100.1\",\n\t\t330: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/124 (KHTML, like Gecko) Safari/125.1\",\n\t\t331: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/125.2 (KHTML, like Gecko) Safari/125.8\",\n\t\t332: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/125.4 (KHTML, like Gecko) Safari/125.9\",\n\t\t333: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.11\",\n\t\t334: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; de-de) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12\",\n\t\t335: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12\",\n\t\t336: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; nl-nl) AppleWebKit/125.5.5 (KHTML, like Gecko) Safari/125.12\",\n\t\t337: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/125.5.6 (KHTML, like Gecko) Safari/125.12\",\n\t\t338: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.1 (KHTML, like Gecko) Safari/312\",\n\t\t339: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.1.1 (KHTML, like Gecko) Safari/312\",\n\t\t340: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.5 (KHTML, like Gecko) Safari/312.3\",\n\t\t341: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.5\",\n\t\t342: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/312.8 (KHTML, like Gecko) Safari/312.6\",\n\t\t343: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412 (KHTML, like Gecko) Safari/412\",\n\t\t344: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.6.2 (KHTML, like Gecko) Safari/125.11\",\n\t\t345: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.6.2 (KHTML, like Gecko) Safari/412.2.2\",\n\t\t346: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/412.7 (KHTML, like Gecko) Safari/412.5\",\n\t\t347: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/416.11 (KHTML, like Gecko) Safari/416.12\",\n\t\t348: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.2\",\n\t\t349: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-jp) AppleWebKit/418 (KHTML, like Gecko) Safari/417.9.3\",\n\t\t350: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/522.11.1 (KHTML, like Gecko) Version/3.0.3 Safari/522.12.1\",\n\t\t351: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X; en-jp) AppleWebKit/523.10.3 (KHTML, like Gecko) Version/3.0.4 Safari/523.10\",\n\t\t352: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.10.6 (KHTML, like Gecko) Version/3.0.4 Safari/523.10.6\",\n\t\t353: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.12 (KHTML, like Gecko) Version/3.0.4 Safari/523.12\",\n\t\t354: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-jp) AppleWebKit/523.12.2 (KHTML, like Gecko) Version/3.0.4 Safari/523.12.2\",\n\t\t355: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en-jp) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13\",\n\t\t356: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_3; en-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.1 Safari/525.20\",\n\t\t357: \"Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_4_11; en-jp) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.22\",\n\t\t358: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; en-jp) AppleWebKit/525.26.2 (KHTML, like Gecko) Version/3.2 Safari/525.26.12\",\n\t\t359: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-jp) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1\",\n\t\t360: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_6; en-jp) AppleWebKit/528.16 (KHTML, like Gecko) Version/4.0 Safari/528.16\",\n\t\t361: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; en-jp) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10\",\n\t\t362: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-jp) AppleWebKit/531.21.11 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10\",\n\t\t363: \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-jp) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16\",\n\t\t364: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.52.7 (KHTML, like Gecko) Version/5.1.2 Safari/534.52.7\",\n\t\t365: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2\",\n\t\t366: \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8) AppleWebKit/536.25 (KHTML, like Gecko) Version/6.0 Safari/536.25\",\n\t\t367: \"Mozilla/4.0 (Windows 95;US) Opera 3.62 [en]\",\n\t\t368: \"Mozilla/4.0 (compatible; MSIE 5.0; Mac_PowerPC) Opera 6.0 [en]\",\n\t\t369: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.01 [en]\",\n\t\t370: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows ME) Opera 6.03 [en]\",\n\t\t371: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.05 [en]\",\n\t\t372: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows XP) Opera 6.06 [en]\",\n\t\t373: \"Mozilla/4.0 (compatible; MSIE 5.0; Windows 2000) Opera 6.06 [en]\",\n\t\t374: \"Opera 7.11\",\n\t\t375: \"Opera/7.23 (Windows NT 5.0; U) [en]\",\n\t\t376: \"Opera/7.52 (Windows NT 5.1; U) [en]\",\n\t\t377: \"Opera/7.53 (Windows NT 5.0; U) [en]\",\n\t\t378: \"Opera/7.54 (Windows NT 5.0; U) [en]\",\n\t\t379: \"Opera/7.54 (Windows NT 5.1; U) [en]\",\n\t\t380: \"Opera/7.54 (Windows NT 5.1; U)\",\n\t\t381: \"Opera/7.54 (X11; Linux i686; U) [en]\",\n\t\t382: \"Opera/7.54 (X11; Linux i686; U) [sv]\",\n\t\t383: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.11 [en]\",\n\t\t384: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.21 [pt-BR]\",\n\t\t385: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.22 [en]\",\n\t\t386: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.23 [en]\",\n\t\t387: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.23 [en]\",\n\t\t388: \"Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686) Opera 7.23 [en]\",\n\t\t389: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.23 [fr]\",\n\t\t390: \"Mozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC) Opera 7.50 [en]\",\n\t\t391: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.53 [en]\",\n\t\t392: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.53 [en]\",\n\t\t393: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.54 [en]\",\n\t\t394: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.54 [en]\",\n\t\t395: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1) Opera 7.54 [en]\",\n\t\t396: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0) Opera 7.54u1 [en]\",\n\t\t397: \"Opera/7.54 (Windows 98; U) [en]\",\n\t\t398: \"Mozilla/4.78 (Windows NT 5.1; U) Opera 7.23 [en]\",\n\t\t399: \"Mozilla/5.0 (Windows NT 5.0; U) Opera 7.54 [en]\",\n\t\t400: \"Mozilla/4.0 (compatible; MSIE 6.0; X11; OpenBSD i386) Opera 7.54 [en]\",\n\t\t401: \"Opera/8.0 (X11; Linux i686; U; en)\",\n\t\t402: \"Opera/8.01 (Windows ME; U; en)\",\n\t\t403: \"Opera/8.01 (Windows NT 5.1; U; en)\",\n\t\t404: \"Mozilla/4.0 (compatible; MSIE 6.0; Mac_PowerPC Mac OS X; en) Opera 8.01\",\n\t\t405: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.01\",\n\t\t406: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; tr) Opera 8.02\",\n\t\t407: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.02\",\n\t\t408: \"Opera/8.5 (Windows NT 5.0; U; en)\",\n\t\t409: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.5\",\n\t\t410: \"Opera/8.51 (Windows NT 5.1; U; en)\",\n\t\t411: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 8.51\",\n\t\t412: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; de) Opera 8.52\",\n\t\t413: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; pl) Opera 8.52\",\n\t\t414: \"Mozilla/5.0 (Windows NT 5.1; U; en) Opera 8.52\",\n\t\t415: \"Opera/8.53 (Windows NT 5.1; U; en)\",\n\t\t416: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.53\",\n\t\t417: \"Opera/8.54 (Windows NT 5.1; U; en)\",\n\t\t418: \"Opera/8.54 (Windows NT 5.0; U; en)\",\n\t\t419: \"Mozilla/5.0 (X11; Linux i686; U; cs) Opera 8.54\",\n\t\t420: \"Mozilla/4.0 (compatible; MSIE 6.0; KDDI-SA39) Opera 8.60 [en]\",\n\t\t421: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; en) Opera 9.00\",\n\t\t422: \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 9.00\",\n\t\t423: \"Opera/9.00 (Windows NT 5.1; U; en)\",\n\t\t424: \"Opera/9.0 (Windows NT 5.1; U; en)\",\n\t\t425: \"Opera/9.00 (Macintosh; PPC Mac OS X; U; en)\",\n\t\t426: \"Opera/9.02 (Macintosh; PPC Mac OS X; U; en)\",\n\t\t427: \"Opera/9.02 (Windows NT 5.1; U; zh-tw)\",\n\t\t428: \"Opera/9.10 (Windows NT 6.0; U; en)\",\n\t\t429: \"Opera/9.21 (Windows NT 6.0; U; en)\",\n\t\t430: \"Opera/9.22 (Windows NT 5.1; U; en)\",\n\t\t431: \"Opera/9.23 (Windows NT 5.1; U; en)\",\n\t\t432: \"Opera/9.23 (Windows ME; U; en)\",\n\t\t433: \"Opera/9.26 (Windows NT 5.1; U; en)\",\n\t\t434: \"Opera/9.51 (Windows NT 5.1; U; en)\",\n\t\t435: \"Opera/9.52 (Macintosh; Intel Mac OS X; U; en)\",\n\t\t436: \"Opera/9.52 (Windows NT 5.1; U; en)\",\n\t\t437: \"Opera/9.60 (Macintosh; Intel Mac OS X; U; en) Presto/2.1.1\",\n\t\t438: \"Opera/9.60 (Windows NT 5.1; U; en) Presto/2.1.1\",\n\t\t439: \"Opera/9.61 (Windows NT 5.1; U; en) Presto/2.1.1\",\n\t\t440: \"Opera/9.62 (Windows NT 5.1; U; en) Presto/2.1.1\",\n\t\t441: \"Mozilla/4.0 (compatible; MSIE 6.0; X11; Linux i686; en) Opera 10.10\",\n\t\t442: \"Opera/9.80 (Windows NT 6.1; U; en) Presto/2.9.168 Version/11.50\",\n\t\t443: \"Opera/9.80 (Windows NT 6.1; U; en) Presto/2.10.229 Version/11.60\",\n\t\t444: \"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.10.289 Version/12.00\",\n\t}\n\n\t// this *.txt maybe not found if you exec binary, so we just fill several ua\n\ttemp, err := util.ReadFromFile(util.CurDir() + \"/config/ua.txt\")\n\n\tif err == nil {\n\t\tuas := strings.Split(string(temp), \"\\n\")\n\t\tfor i, ua := range uas {\n\t\t\tUa[i] = strings.TrimSpace(strings.Replace(ua, \"\\r\", \"\", -1))\n\t\t}\n\t}\n\n}\n\n// RandomUa back random User-Agent\nfunc RandomUa() string {\n\tlength := len(Ua)\n\tif length == 0 {\n\t\treturn \"\"\n\t}\n\treturn Ua[rand.Intn(length-1)]\n}\n" }, { "path": "miner/util.go", "content": "package miner\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"strings\"\n\n\t\"github.com/hunterhug/marmot/util\"\n)\n\n// Wait some second\nfunc Wait(waitTime int) {\n\tif waitTime <= 0 {\n\t\treturn\n\t} else {\n\t\tLogger.Debugf(\"[GoWorker] Wait %d Second.\", waitTime)\n\t\tutil.Sleep(waitTime)\n\t}\n}\n\n// CopyM Header map[string][]string ,can use to copy a http header, so that they are not effect each other\nfunc CopyM(h http.Header) http.Header {\n\tif h == nil || len(h) == 0 {\n\t\treturn h\n\t}\n\th2 := make(http.Header, len(h))\n\tfor k, vv := range h {\n\t\tvv2 := make([]string, len(vv))\n\t\tcopy(vv2, vv)\n\t\th2[k] = vv2\n\t}\n\treturn h2\n}\n\n// TooSortSizes if a file size smaller than sizes(KB), it will be thrown an error\nfunc TooSortSizes(data []byte, sizes float64) error {\n\tif float64(len(data))/1000 < sizes {\n\t\treturn errors.New(fmt.Sprintf(\"FileSize:%d bytes,%d kb < %f kb dead too sort\", len(data), len(data)/1000, sizes))\n\t}\n\treturn nil\n}\n\n// OutputMaps Just debug a map\nfunc OutputMaps(uuid string, info string, args map[string][]string) {\n\ts := \"\\n\"\n\tfor k, v := range args {\n\t\ts = s + fmt.Sprintf(\"%-25s | %-6s\\n\", k, strings.Join(v, \" |||| \"))\n\t}\n\tLogger.Debugf(\"%s %s-%s\", uuid, info, s)\n}\n" }, { "path": "miner/worker.go", "content": "package miner\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"os\"\n\t\"strings\"\n\n\t\"mime/multipart\"\n\n\t\"github.com/hunterhug/marmot/util\"\n)\n\n// NewWorkerByClient New Worker by Your Client\nfunc NewWorkerByClient(client *http.Client) *Worker {\n\tworker := new(Worker)\n\tworker.Request = newRequest()\n\tworker.Header = http.Header{}\n\tworker.Response = new(Response)\n\n\tworker.Client = client\n\treturn worker\n}\n\n// NewAPI New API Worker, No Cookie Keep, share the same http client\nfunc NewAPI() *Worker {\n\treturn NewWorkerByClient(NoCookieClient)\n}\n\n// NewWorker New a worker, if ipString is a proxy address, New a proxy client. evey time gen a new http client!\n// Proxy address such as:\n//\n//\thttp://[user]:[password@]ip:port, [] stand it can choose or not. case: socks5://127.0.0.1:1080\nfunc NewWorker(proxyIpString interface{}) (*Worker, error) {\n\tif proxyIpString != nil {\n\t\tclient, err := NewProxyClient(strings.ToLower(proxyIpString.(string)))\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tworker := NewWorkerByClient(client)\n\t\tworker.Ip = proxyIpString.(string)\n\t\treturn worker, err\n\t} else {\n\t\tclient := NewClient()\n\t\tworker := NewWorkerByClient(client)\n\t\tworker.Ip = \"localhost\"\n\t\treturn worker, nil\n\t}\n}\n\n// NewWorkerWithProxy Alias func\nfunc NewWorkerWithProxy(proxyIpString interface{}) (*Worker, error) {\n\treturn NewWorker(proxyIpString)\n}\n\n// NewWorkerWithNoProxy Alias func\nfunc NewWorkerWithNoProxy() *Worker {\n\tw, _ := NewWorker(nil)\n\treturn w\n}\n\n// New Alias Name for NewWorker\nfunc New(ipString interface{}) (*Worker, error) {\n\treturn NewWorker(ipString)\n}\n\n// fast new request\nfunc newRequest() *Request {\n\treq := new(Request)\n\treq.Data = url.Values{}\n\treq.BData = []byte{}\n\treturn req\n}\n\n// Clone Should Clone a new worker if you want to use repeat\nfunc (worker *Worker) Clone() *Worker {\n\tcloneWorker := NewWorkerByClient(worker.Client)\n\tcloneWorker.Ip = worker.Ip\n\treturn cloneWorker\n}\n\n// Go Auto decide which method, Default Get.\nfunc (worker *Worker) Go() (body []byte, e error) {\n\tswitch strings.ToUpper(worker.Method) {\n\tcase POST:\n\t\treturn worker.Post()\n\tcase POSTJSON:\n\t\treturn worker.PostJSON()\n\tcase POSTXML:\n\t\treturn worker.PostXML()\n\tcase POSTFILE:\n\t\treturn worker.PostFILE()\n\tcase PUT:\n\t\treturn worker.Put()\n\tcase PUTJSON:\n\t\treturn worker.PutJSON()\n\tcase PUTXML:\n\t\treturn worker.PutXML()\n\tcase PUTFILE:\n\t\treturn worker.PutFILE()\n\tcase DELETE:\n\t\treturn worker.Delete()\n\tcase OTHER:\n\t\treturn []byte(\"\"), errors.New(\"please use method OtherGo(method, contentType string) or OtherGoBinary(method, contentType string)\")\n\tdefault:\n\t\treturn worker.Get()\n\t}\n}\n\nfunc (worker *Worker) GoByMethod(method string) (body []byte, e error) {\n\treturn worker.SetMethod(method).Go()\n}\n\n// ToString This make effect only your worker exec serial! Attention!\n// Change Your Raw data To string\nfunc (worker *Worker) ToString() string {\n\tif worker.Raw == nil {\n\t\treturn \"\"\n\t}\n\treturn string(worker.Raw)\n}\n\n// JsonToString This make effect only your worker exec serial! Attention!\n// Change Your JSON' like Raw data to string\nfunc (worker *Worker) JsonToString() (string, error) {\n\tif worker.Raw == nil {\n\t\treturn \"\", nil\n\t}\n\ttemp, err := util.JsonBack(worker.Raw)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn string(temp), nil\n}\n\n// Main method I make!\nfunc (worker *Worker) sent(method, contentType string, binary bool) (body []byte, e error) {\n\t// Lock it for save\n\tworker.mux.Lock()\n\tdefer worker.mux.Unlock()\n\n\t// Before Action, we can change or add something before Go()\n\tif worker.BeforeAction != nil {\n\t\tworker.BeforeAction(worker.Ctx, worker)\n\t}\n\n\t// Wait if must\n\tif worker.Wait > 0 {\n\t\tWait(worker.Wait)\n\t}\n\n\t// For debug\n\tuuid := \"[GoWorker]\"\n\tLogger.Debugf(\"%s %s %s\", uuid, method, worker.Url)\n\n\t// New a Request\n\tvar request *http.Request\n\tvar err error\n\n\t// If binary value is true and BData is not empty\n\t// suit for POSTJSON(), POSTFILE()\n\tif len(worker.BData) != 0 && binary {\n\t\tpr := bytes.NewReader(worker.BData)\n\t\trequest, err = http.NewRequest(method, worker.Url, pr)\n\t} else if len(worker.Data) != 0 { // such POST() from table form\n\t\tpr := strings.NewReader(worker.Data.Encode())\n\t\trequest, err = http.NewRequest(method, worker.Url, pr)\n\t} else {\n\t\trequest, err = http.NewRequest(method, worker.Url, nil)\n\t}\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Close avoid EOF\n\t// For client requests, setting this field prevents re-use of\n\t// TCP connections between requests to the same hosts, as if\n\t// Transport.DisableKeepAlives were set.\n\t// maybe you want long connection\n\t//request.Close = true\n\n\t// Clone Header, I add some HTTP header!\n\trequest.Header = CloneHeader(worker.Header)\n\n\t// In fact content type must not empty\n\tif contentType != \"\" {\n\t\trequest.Header.Set(\"Content-Type\", contentType)\n\t}\n\tworker.Request.Request = request\n\n\t// Debug for RequestHeader\n\tOutputMaps(uuid, \"Request Header\", request.Header)\n\n\t// Tolerate abnormal way to create a Worker\n\tif worker.Client == nil {\n\t\tworker.Client = Client\n\t}\n\n\t// Do it\n\tresponse, err := worker.Client.Do(request)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// Close it attention response may be nil\n\tif response != nil {\n\t\t//response.Close = true\n\t\tdefer response.Body.Close()\n\t}\n\n\t// Debug\n\tOutputMaps(uuid, \"Response Header\", response.Header)\n\tLogger.Debugf(\"%s %s %s\", uuid, response.Proto, response.Status)\n\n\t// Read output\n\tbody, e = io.ReadAll(response.Body)\n\tworker.Raw = body\n\n\tworker.ResponseStatusCode = response.StatusCode\n\tworker.Response.Response = response\n\n\t// After action\n\tif worker.AfterAction != nil {\n\t\tworker.AfterAction(worker.Ctx, worker)\n\t}\n\n\treturn\n}\n\nfunc (worker *Worker) sentByFile(method, contentType string, binary bool, filePath string) (e error) {\n\t// Lock it for save\n\tworker.mux.Lock()\n\tdefer worker.mux.Unlock()\n\n\t// Before Action, we can change or add something before Go()\n\tif worker.BeforeAction != nil {\n\t\tworker.BeforeAction(worker.Ctx, worker)\n\t}\n\n\t// Wait if must\n\tif worker.Wait > 0 {\n\t\tWait(worker.Wait)\n\t}\n\n\t// For debug\n\tuuid := \"[GoWorker]\"\n\tLogger.Debugf(\"%s %s %s\", uuid, method, worker.Url)\n\n\t// New a Request\n\tvar request *http.Request\n\tvar err error\n\n\t// If binary value is true and BData is not empty\n\t// suit for POSTJSON(), POSTFILE()\n\tif len(worker.BData) != 0 && binary {\n\t\tpr := bytes.NewReader(worker.BData)\n\t\trequest, err = http.NewRequest(method, worker.Url, pr)\n\t} else if len(worker.Data) != 0 { // such POST() from table form\n\t\tpr := strings.NewReader(worker.Data.Encode())\n\t\trequest, err = http.NewRequest(method, worker.Url, pr)\n\t} else {\n\t\trequest, err = http.NewRequest(method, worker.Url, nil)\n\t}\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Close avoid EOF\n\t// For client requests, setting this field prevents re-use of\n\t// TCP connections between requests to the same hosts, as if\n\t// Transport.DisableKeepAlives were set.\n\t// maybe you want long connection\n\t//request.Close = true\n\n\t// Clone Header, I add some HTTP header!\n\trequest.Header = CloneHeader(worker.Header)\n\n\t// In fact content type must not empty\n\tif contentType != \"\" {\n\t\trequest.Header.Set(\"Content-Type\", contentType)\n\t}\n\tworker.Request.Request = request\n\n\t// Debug for RequestHeader\n\tOutputMaps(uuid, \"Request Header\", request.Header)\n\n\t// Tolerate abnormal way to create a Worker\n\tif worker.Client == nil {\n\t\tworker.Client = Client\n\t}\n\n\t// Do it\n\tresponse, err := worker.Client.Do(request)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Close it attention response may be nil\n\tif response != nil {\n\t\t//response.Close = true\n\t\tdefer response.Body.Close()\n\t}\n\n\t// Debug\n\tOutputMaps(uuid, \"Response Header\", response.Header)\n\tLogger.Debugf(\"%s %s %s\", uuid, response.Proto, response.Status)\n\n\ttotalSize := response.ContentLength\n\tLogger.Debugf(\"Download Size: %.2f MB\\n\", float64(totalSize)/(1024*1024))\n\n\tfile, err := os.Create(filePath)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer file.Close()\n\n\tvar downloaded int64\n\n\tsizeMB := worker.DownloadFileReadMB * 1024 * 1024\n\tif sizeMB <= 0 {\n\t\tsizeMB = 5 * 1024 * 1024 // Default 5MB\n\t}\n\tbuf := make([]byte, sizeMB)\n\tfor {\n\t\tn, err := response.Body.Read(buf)\n\t\tif n > 0 {\n\t\t\tif _, writeErr := file.Write(buf[:n]); writeErr != nil {\n\t\t\t\tos.Remove(filePath)\n\t\t\t\treturn writeErr\n\t\t\t}\n\t\t\tdownloaded += int64(n)\n\n\t\t\tif totalSize > 0 {\n\t\t\t\tpercent := float64(downloaded) / float64(totalSize) * 100\n\t\t\t\tLogger.Debugf(\"\\rDownload: %.1f%% (%.2f/%.2f MB)\",\n\t\t\t\t\tpercent,\n\t\t\t\t\tfloat64(downloaded)/(1024*1024),\n\t\t\t\t\tfloat64(totalSize)/(1024*1024))\n\t\t\t}\n\t\t}\n\n\t\tif err == io.EOF {\n\t\t\tLogger.Debugf(\"\\nDownload Done\")\n\t\t\tbreak\n\t\t}\n\t\tif err != nil {\n\t\t\tos.Remove(filePath)\n\t\t\treturn err\n\t\t}\n\t}\n\n\tworker.ResponseStatusCode = response.StatusCode\n\tworker.Response.Response = response\n\n\t// After action\n\tif worker.AfterAction != nil {\n\t\tworker.AfterAction(worker.Ctx, worker)\n\t}\n\n\treturn\n}\n\n// Get method\nfunc (worker *Worker) Get() (body []byte, e error) {\n\tworker.Clear()\n\treturn worker.sent(GET, \"\", false)\n}\n\nfunc (worker *Worker) Delete() (body []byte, e error) {\n\tworker.Clear()\n\treturn worker.sent(DELETE, \"\", false)\n}\n\n// Post Almost include bellow:\n/*\n\t\"application/x-www-form-urlencoded\"\n\t\"application/json\"\n\t\"text/xml\"\n\t\"multipart/form-data\"\n*/\nfunc (worker *Worker) Post() (body []byte, e error) {\n\treturn worker.sent(POST, HTTPFORMContentType, false)\n}\n\nfunc (worker *Worker) PostJSON() (body []byte, e error) {\n\treturn worker.sent(POST, HTTPJSONContentType, true)\n}\n\nfunc (worker *Worker) PostXML() (body []byte, e error) {\n\treturn worker.sent(POST, HTTPXMLContentType, true)\n}\n\nfunc (worker *Worker) PostFILE() (body []byte, e error) {\n\treturn worker.sentFile(POST)\n\n}\n\nvar quoteEscaper = strings.NewReplacer(\"\\\\\", \"\\\\\\\\\", `\"`, \"\\\\\\\"\")\n\nfunc escapeQuotes(s string) string {\n\treturn quoteEscaper.Replace(s)\n}\n\nfunc (worker *Worker) sentFile(method string) ([]byte, error) {\n\tif worker.FileName == \"\" || worker.FileFormName == \"\" {\n\t\treturn nil, errors.New(\"fileName or fileFormName must not empty\")\n\t}\n\n\tif len(worker.BData) == 0 {\n\t\treturn nil, errors.New(\"BData must not empty\")\n\t}\n\n\tbodyBuf := &bytes.Buffer{}\n\tbodyWriter := multipart.NewWriter(bodyBuf)\n\n\tfileWriter, err := bodyWriter.CreateFormFile(worker.FileFormName, worker.FileName)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t_, err = fileWriter.Write(worker.BData)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor k, v := range worker.Data {\n\t\tfor _, vv := range v {\n\t\t\tif vv != \"\" {\n\t\t\t\tbodyWriter.WriteField(k, vv)\n\t\t\t}\n\t\t}\n\t}\n\n\tcontentType := bodyWriter.FormDataContentType()\n\n\terr = bodyWriter.Close()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tworker.SetBData(bodyBuf.Bytes())\n\n\treturn worker.sent(method, contentType, true)\n}\n\nfunc (worker *Worker) Put() (body []byte, e error) {\n\treturn worker.sent(PUT, HTTPFORMContentType, false)\n}\n\nfunc (worker *Worker) PutJSON() (body []byte, e error) {\n\treturn worker.sent(PUT, HTTPJSONContentType, true)\n}\n\nfunc (worker *Worker) PutXML() (body []byte, e error) {\n\treturn worker.sent(PUT, HTTPXMLContentType, true)\n}\n\nfunc (worker *Worker) PutFILE() (body []byte, e error) {\n\treturn worker.sentFile(PUT)\n}\n\n/*\nOtherGo Method\n\n\t Method = \"OPTIONS\" ; Section 9.2\n\t | \"GET\" ; Section 9.3\n\t | \"HEAD\" ; Section 9.4\n\t | \"POST\" ; Section 9.5\n\t | \"PUT\" ; Section 9.6\n\t | \"DELETE\" ; Section 9.7\n\t | \"TRACE\" ; Section 9.8\n\t | \"CONNECT\" ; Section 9.9\n\t | extension-method\n\textension-method = token\n\t token = 1*\n\nContent Type\n\n\t\"application/x-www-form-urlencoded\"\n\t\"application/json\"\n\t\"text/xml\"\n\t\"multipart/form-data\"\n*/\nfunc (worker *Worker) OtherGo(method, contentType string) (body []byte, e error) {\n\treturn worker.sent(method, contentType, false)\n}\n\nfunc (worker *Worker) OtherGoBinary(method, contentType string) (body []byte, e error) {\n\treturn worker.sent(method, contentType, true)\n}\n\nfunc (worker *Worker) OtherDownload(method, contentType string, filePath string) (e error) {\n\treturn worker.sentByFile(method, contentType, true, filePath)\n}\n" }, { "path": "util/crypto.go", "content": "package util\n\nimport (\n\t\"crypto/hmac\"\n\t\"crypto/md5\"\n\t\"crypto/sha256\"\n\t\"encoding/base64\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"net/url\"\n\t\"strings\"\n)\n\n// ComputeHmac256 HMAC with the SHA256\nfunc ComputeHmac256(message string, secret string) string {\n\tkey := []byte(secret)\n\th := hmac.New(sha256.New, key)\n\th.Write([]byte(message))\n\treturn base64.StdEncoding.EncodeToString(h.Sum(nil))\n}\n\n// StrToMd5 create md5 string\nfunc StrToMd5(s string) string {\n\th := md5.New()\n\th.Write([]byte(s))\n\trs := hex.EncodeToString(h.Sum(nil))\n\treturn rs\n}\n\nfunc Md5(str string) string {\n\treturn StrToMd5(str)\n}\n\nfunc Base64E(body string) string {\n\tstr := []byte(body)\n\tdata := base64.StdEncoding.EncodeToString(str)\n\treturn data\n}\n\nfunc Base64D(body string) string {\n\tdata, err := base64.StdEncoding.DecodeString(body)\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\ts := fmt.Sprintf(\"%q\", data)\n\ts = strings.Replace(s, \"\\\"\", \"\", -1)\n\treturn s\n}\n\nfunc UrlE(s string) string {\n\treturn url.QueryEscape(s)\n}\n\nfunc UrlD(s string) string {\n\ts, e := url.QueryUnescape(s)\n\tif e != nil {\n\t\treturn e.Error()\n\t} else {\n\t\treturn s\n\t}\n}\n\nfunc Md5FS(src io.Reader) string {\n\th := md5.New()\n\tif err := CopyFF(src, h); err != nil {\n\t\tfmt.Println(err.Error())\n\t\treturn \"\"\n\t}\n\treturn fmt.Sprintf(\"%x\", h.Sum([]byte(\"hunterhug\")))\n}\n" }, { "path": "util/file.go", "content": "package util\n\nimport (\n\t\"errors\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"path/filepath\"\n\t\"runtime\"\n\t\"strings\"\n)\n\nfunc CurDir() string {\n\t_, filename, _, _ := runtime.Caller(1)\n\treturn filepath.Dir(filename)\n}\n\nfunc GetBinaryCurrentPath() (string, error) {\n\tfile, err := exec.LookPath(os.Args[0])\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tpath, err := filepath.Abs(file)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tif strings.Contains(path, \"command-line-arguments\") {\n\t\treturn GetCurrentPath()\n\t}\n\n\ti := strings.LastIndex(path, \"/\")\n\tif i < 0 {\n\t\ti = strings.LastIndex(path, \"\\\\\")\n\t}\n\n\tif i < 0 {\n\t\treturn \"\", errors.New(`error: Can't find \"/\" or \"\\\"`)\n\t}\n\n\treturn path[0 : i+1], nil\n}\n\nfunc GetCurrentPath() (string, error) {\n\treturn os.Getwd()\n}\n\nfunc SaveToFile(filePath string, content []byte) error {\n\terr := os.WriteFile(filePath, content, 0777)\n\treturn err\n}\n\nfunc ReadFromFile(filepath string) ([]byte, error) {\n\treturn os.ReadFile(filepath)\n}\n\nfunc GetFilenameInfo(filepath string) (os.FileInfo, error) {\n\tinfo, err := os.Stat(filepath)\n\treturn info, err\n}\n\nfunc Rename(oldFilename string, newFilename string) error {\n\treturn os.Rename(oldFilename, newFilename)\n}\n\nfunc MakeDir(fileDir string) error {\n\treturn os.MkdirAll(fileDir, 0777)\n}\n\nfunc MakeDirByFile(filepath string) error {\n\ttemp := strings.Split(filepath, \"/\")\n\tif len(temp) <= 2 {\n\t\treturn errors.New(\"please input complete file name like ./dir/filename or /home/dir/filename\")\n\t}\n\tdirPath := strings.Join(temp[0:len(temp)-1], \"/\")\n\treturn MakeDir(dirPath)\n}\n\nfunc FileExist(filename string) bool {\n\tfi, err := os.Stat(filename)\n\tif err != nil {\n\t\treturn false\n\t}\n\tif fi.IsDir() {\n\t\treturn false\n\t} else {\n\t\treturn true\n\t}\n}\n\nfunc WalkDir(dirPth, suffix string) (files []string, err error) {\n\tfiles = make([]string, 0, 30)\n\tsuffix = strings.ToUpper(suffix)\n\terr = filepath.Walk(dirPth, func(filename string, fi os.FileInfo, err error) error {\n\t\tif fi.IsDir() {\n\t\t\treturn nil\n\t\t}\n\t\tif strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {\n\t\t\tfiles = append(files, filename)\n\t\t}\n\t\treturn nil\n\t})\n\treturn files, err\n}\n\nfunc ListDir(dirPth string, suffix string) (files []string, err error) {\n\tfiles = make([]string, 0, 10)\n\tdir, err := os.ReadDir(dirPth)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tsuffix = strings.ToUpper(suffix)\n\n\tfor _, fi := range dir {\n\t\tif fi.IsDir() {\n\t\t\tcontinue\n\t\t}\n\n\t\tif strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {\n\t\t\tfiles = append(files, dirPth+\"/\"+fi.Name())\n\t\t}\n\t}\n\n\treturn files, nil\n}\n\nfunc ListDirOnlyName(dirPth string, suffix string) (files []string, err error) {\n\tfiles = make([]string, 0, 10)\n\n\tdir, err := os.ReadDir(dirPth)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tsuffix = strings.ToUpper(suffix)\n\n\tfor _, fi := range dir {\n\t\tif fi.IsDir() {\n\t\t\tcontinue\n\t\t}\n\t\tif strings.HasSuffix(strings.ToUpper(fi.Name()), suffix) {\n\t\t\tfiles = append(files, fi.Name())\n\t\t}\n\t}\n\n\treturn files, nil\n}\n\nfunc HasFile(s string) bool {\n\tf, err := os.Open(s)\n\tif err != nil && os.IsNotExist(err) {\n\t\treturn false\n\t}\n\tf.Close()\n\treturn true\n}\n\nfunc CopyFF(src io.Reader, dst io.Writer) error {\n\t_, err := io.Copy(dst, src)\n\treturn err\n}\n\nfunc CopyFS(src io.Reader, dst string) error {\n\tf, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer f.Close()\n\t_, err = io.Copy(f, src)\n\treturn err\n}\n\nfunc IsFile(filepath string) bool {\n\tinfo, err := os.Stat(filepath)\n\tif err != nil {\n\t\treturn false\n\t} else {\n\t\tif info.IsDir() {\n\t\t\treturn false\n\t\t} else {\n\t\t\treturn true\n\t\t}\n\t}\n}\n\nfunc IsDir(filepath string) bool {\n\tinfo, err := os.Stat(filepath)\n\tif err != nil {\n\t\treturn false\n\t} else {\n\t\tif info.IsDir() {\n\t\t\treturn true\n\t\t} else {\n\t\t\treturn false\n\t\t}\n\t}\n}\n\nfunc SizeofDir(dirPth string) int {\n\tif IsDir(dirPth) {\n\t\tfiles, _ := os.ReadDir(dirPth)\n\t\treturn len(files)\n\t}\n\n\treturn 0\n}\n\nfunc GetFileSuffix(f string) string {\n\tfa := strings.Split(f, \".\")\n\tif len(fa) == 0 {\n\t\treturn \"\"\n\t} else {\n\t\treturn fa[len(fa)-1]\n\t}\n}\n\nfunc ValidFileName(filename string) string {\n\tpatterns := []string{\n\t\t\" \", \"#01#\",\n\t\t\"\\\\\", \"#02#\",\n\t\t\"/\", \"#03#\",\n\t\t\":\", \"#04#\",\n\t\t\"\\\"\", \"#05#\",\n\t\t\"?\", \"#06#\",\n\t\t\"<\", \"#07#\",\n\t\t\">\", \"#08#\",\n\t\t\"|\", \"#09#\",\n\t}\n\tr := strings.NewReplacer(patterns...)\n\tdudu := r.Replace(filename)\n\treturn dudu\n}\n\nfunc ValidBackFileName(filename string) string {\n\tpatterns := []string{\n\t\t\"#01#\", \" \",\n\t\t\"#02#\", \"\\\\\",\n\t\t\"#03#\", \"/\",\n\t\t\"#04#\", \":\",\n\t\t\"#05#\", \"\\\"\",\n\t\t\"#06#\", \"?\",\n\t\t\"#07#\", \"<\",\n\t\t\"#08#\", \">\",\n\t\t\"#09#\", \"|\",\n\t}\n\tr := strings.NewReplacer(patterns...)\n\tdudu := r.Replace(filename)\n\treturn dudu\n}\n" }, { "path": "util/io.go", "content": "package util\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\nfunc Input(say, defaults string) string {\n\tfmt.Println(say)\n\n\tvar str string\n\t_, _ = fmt.Scanln(&str)\n\n\tif strings.TrimSpace(str) == \"\" {\n\t\tif strings.TrimSpace(defaults) != \"\" {\n\t\t\treturn defaults\n\t\t} else {\n\t\t\tfmt.Println(\"can not empty\")\n\t\t\treturn Input(say, defaults)\n\t\t}\n\t}\n\n\treturn str\n}\n" }, { "path": "util/json.go", "content": "package util\n\nimport (\n\t\"encoding/json\"\n)\n\nfunc JsonBack(s []byte) ([]byte, error) {\n\ttemp := new(interface{})\n\n\terr := json.Unmarshal(s, temp)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tresult, err := json.Marshal(temp)\n\treturn result, err\n}\n" }, { "path": "util/time.go", "content": "package util\n\nimport (\n\t\"time\"\n)\n\nfunc Sleep(waitTime int) {\n\ttime.Sleep(time.Duration(waitTime) * time.Second)\n}\n\nfunc Second(times int) time.Duration {\n\treturn time.Duration(times) * time.Second\n}\n\nfunc TodayString(level int) string {\n\tformats := \"20060102150405\"\n\tswitch level {\n\tcase 1:\n\t\tformats = \"2006\"\n\tcase 2:\n\t\tformats = \"200601\"\n\tcase 3:\n\t\tformats = \"20060102\"\n\tcase 4:\n\t\tformats = \"2006010215\"\n\tcase 5:\n\t\tformats = \"200601021504\"\n\tdefault:\n\n\t}\n\treturn time.Now().Format(formats)\n}\n" }, { "path": "util/util.go", "content": "package util\n\nimport (\n\t\"errors\"\n\t\"strconv\"\n\t\"strings\"\n)\n\nfunc SI(s string) (i int, e error) {\n\ti, e = strconv.Atoi(s)\n\treturn\n}\n\nfunc IS(i int) string {\n\treturn strconv.Itoa(i)\n}\n\nfunc ToLower(s string) string {\n\treturn strings.ToLower(s)\n}\n\nfunc DivideStringList(files []string, num int) (map[int][]string, error) {\n\tlength := len(files)\n\tsplit := map[int][]string{}\n\tif num <= 0 {\n\t\treturn split, errors.New(\"num must not negative\")\n\t}\n\n\tif num > length {\n\t\tnum = length\n\t}\n\n\tprocess := length / num\n\tfor i := 0; i < num; i++ {\n\t\tsplit[i] = append(split[i], files[i*process:(i+1)*process]...)\n\t}\n\n\tremain := files[num*process:]\n\tfor i := 0; i < len(remain); i++ {\n\t\tsplit[i%num] = append(split[i%num], remain[i])\n\t}\n\n\treturn split, nil\n}\n\nfunc InArray(sa []string, a string) bool {\n\tfor _, v := range sa {\n\t\tif a == v {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc Substr(str string, start, length int) string {\n\trs := []rune(str)\n\trl := len(rs)\n\tend := 0\n\n\tif start < 0 {\n\t\tstart = rl - 1 + start\n\t}\n\n\tend = start + length\n\n\tif start > end {\n\t\tstart, end = end, start\n\t}\n\n\tif start < 0 {\n\t\tstart = 0\n\t}\n\n\tif start > rl {\n\t\tstart = rl\n\t}\n\n\tif end < 0 {\n\t\tend = 0\n\t}\n\n\tif end > rl {\n\t\tend = rl\n\t}\n\n\treturn string(rs[start:end])\n}\n" } ]