[
{
"path": ".gitignore",
"content": "target/\npom.xml.tag\npom.xml.releaseBackup\npom.xml.versionsBackup\npom.xml.next\nrelease.properties\ndependency-reduced-pom.xml\nbuildNumber.properties\n.mvn/timing.properties\n# https://github.com/takari/maven-wrapper#usage-without-binary-jar\n.mvn/wrapper/maven-wrapper.jar\n\n# Eclipse m2e generated files\n# Eclipse Core\n.project\n# JDT-specific (Eclipse Java Development Tools)\n.classpath\n.metadata\nbin/\ntmp/\n*.tmp\n*.bak\n*.swp\n*~.nib\nlocal.properties\n.settings/\n.loadpath\n.recommenders\n\n# External tool builders\n.externalToolBuilders/\n\n# Locally stored \"Eclipse launch configurations\"\n*.launch\n\n# PyDev specific (Python IDE for Eclipse)\n*.pydevproject\n\n# CDT-specific (C/C++ Development Tooling)\n.cproject\n\n# CDT- autotools\n.autotools\n\n# Java annotation processor (APT)\n.factorypath\n\n# PDT-specific (PHP Development Tools)\n.buildpath\n\n# sbteclipse plugin\n.target\n\n# Tern plugin\n.tern-project\n\n# TeXlipse plugin\n.texlipse\n\n# STS (Spring Tool Suite)\n.springBeans\n\n# Code Recommenders\n.recommenders/\n\n# Annotation Processing\n.apt_generated/\n.apt_generated_test/\n\n# Scala IDE specific (Scala & Java development for Eclipse)\n.cache-main\n.scala_dependencies\n.worksheet\n\n# Uncomment this line if you wish to ignore the project description file.\n# Typically, this file would be tracked if it contains build/dependency configurations:\n#.project\n"
},
{
"path": ".travis.yml",
"content": "language: java \njdk:\n - openjdk9\n"
},
{
"path": "LICENSE",
"content": "Apache License\nVersion 2.0, January 2004\nhttp://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n\"License\" shall mean the terms and conditions for use, reproduction, and\ndistribution as defined by Sections 1 through 9 of this document.\n\n\"Licensor\" shall mean the copyright owner or entity authorized by the copyright\nowner that is granting the License.\n\n\"Legal Entity\" shall mean the union of the acting entity and all other entities\nthat control, are controlled by, or are under common control with that entity.\nFor the purposes of this definition, \"control\" means (i) the power, direct or\nindirect, to cause the direction or management of such entity, whether by\ncontract or otherwise, or (ii) ownership of fifty percent (50%) or more of the\noutstanding shares, or (iii) beneficial ownership of such entity.\n\n\"You\" (or \"Your\") shall mean an individual or Legal Entity exercising\npermissions granted by this License.\n\n\"Source\" form shall mean the preferred form for making modifications, including\nbut not limited to software source code, documentation source, and configuration\nfiles.\n\n\"Object\" form shall mean any form resulting from mechanical transformation or\ntranslation of a Source form, including but not limited to compiled object code,\ngenerated documentation, and conversions to other media types.\n\n\"Work\" shall mean the work of authorship, whether in Source or Object form, made\navailable under the License, as indicated by a copyright notice that is included\nin or attached to the work (an example is provided in the Appendix below).\n\n\"Derivative Works\" shall mean any work, whether in Source or Object form, that\nis based on (or derived from) the Work and for which the editorial revisions,\nannotations, elaborations, or other modifications represent, as a whole, an\noriginal work of authorship. For the purposes of this License, Derivative Works\nshall not include works that remain separable from, or merely link (or bind by\nname) to the interfaces of, the Work and Derivative Works thereof.\n\n\"Contribution\" shall mean any work of authorship, including the original version\nof the Work and any modifications or additions to that Work or Derivative Works\nthereof, that is intentionally submitted to Licensor for inclusion in the Work\nby the copyright owner or by an individual or Legal Entity authorized to submit\non behalf of the copyright owner. For the purposes of this definition,\n\"submitted\" means any form of electronic, verbal, or written communication sent\nto the Licensor or its representatives, including but not limited to\ncommunication on electronic mailing lists, source code control systems, and\nissue tracking systems that are managed by, or on behalf of, the Licensor for\nthe purpose of discussing and improving the Work, but excluding communication\nthat is conspicuously marked or otherwise designated in writing by the copyright\nowner as \"Not a Contribution.\"\n\n\"Contributor\" shall mean Licensor and any individual or Legal Entity on behalf\nof whom a Contribution has been received by Licensor and subsequently\nincorporated within the Work.\n\n2. Grant of Copyright License.\n\nSubject to the terms and conditions of this License, each Contributor hereby\ngrants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,\nirrevocable copyright license to reproduce, prepare Derivative Works of,\npublicly display, publicly perform, sublicense, and distribute the Work and such\nDerivative Works in Source or Object form.\n\n3. Grant of Patent License.\n\nSubject to the terms and conditions of this License, each Contributor hereby\ngrants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,\nirrevocable (except as stated in this section) patent license to make, have\nmade, use, offer to sell, sell, import, and otherwise transfer the Work, where\nsuch license applies only to those patent claims licensable by such Contributor\nthat are necessarily infringed by their Contribution(s) alone or by combination\nof their Contribution(s) with the Work to which such Contribution(s) was\nsubmitted. If You institute patent litigation against any entity (including a\ncross-claim or counterclaim in a lawsuit) alleging that the Work or a\nContribution incorporated within the Work constitutes direct or contributory\npatent infringement, then any patent licenses granted to You under this License\nfor that Work shall terminate as of the date such litigation is filed.\n\n4. Redistribution.\n\nYou may reproduce and distribute copies of the Work or Derivative Works thereof\nin any medium, with or without modifications, and in Source or Object form,\nprovided that You meet the following conditions:\n\nYou must give any other recipients of the Work or Derivative Works a copy of\nthis License; and\nYou must cause any modified files to carry prominent notices stating that You\nchanged the files; and\nYou must retain, in the Source form of any Derivative Works that You distribute,\nall copyright, patent, trademark, and attribution notices from the Source form\nof the Work, excluding those notices that do not pertain to any part of the\nDerivative Works; and\nIf the Work includes a \"NOTICE\" text file as part of its distribution, then any\nDerivative Works that You distribute must include a readable copy of the\nattribution notices contained within such NOTICE file, excluding those notices\nthat do not pertain to any part of the Derivative Works, in at least one of the\nfollowing places: within a NOTICE text file distributed as part of the\nDerivative Works; within the Source form or documentation, if provided along\nwith the Derivative Works; or, within a display generated by the Derivative\nWorks, if and wherever such third-party notices normally appear. The contents of\nthe NOTICE file are for informational purposes only and do not modify the\nLicense. You may add Your own attribution notices within Derivative Works that\nYou distribute, alongside or as an addendum to the NOTICE text from the Work,\nprovided that such additional attribution notices cannot be construed as\nmodifying the License.\nYou may add Your own copyright statement to Your modifications and may provide\nadditional or different license terms and conditions for use, reproduction, or\ndistribution of Your modifications, or for any such Derivative Works as a whole,\nprovided Your use, reproduction, and distribution of the Work otherwise complies\nwith the conditions stated in this License.\n\n5. Submission of Contributions.\n\nUnless You explicitly state otherwise, any Contribution intentionally submitted\nfor inclusion in the Work by You to the Licensor shall be under the terms and\nconditions of this License, without any additional terms or conditions.\nNotwithstanding the above, nothing herein shall supersede or modify the terms of\nany separate license agreement you may have executed with Licensor regarding\nsuch Contributions.\n\n6. Trademarks.\n\nThis License does not grant permission to use the trade names, trademarks,\nservice marks, or product names of the Licensor, except as required for\nreasonable and customary use in describing the origin of the Work and\nreproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty.\n\nUnless required by applicable law or agreed to in writing, Licensor provides the\nWork (and each Contributor provides its Contributions) on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,\nincluding, without limitation, any warranties or conditions of TITLE,\nNON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are\nsolely responsible for determining the appropriateness of using or\nredistributing the Work and assume any risks associated with Your exercise of\npermissions under this License.\n\n8. Limitation of Liability.\n\nIn no event and under no legal theory, whether in tort (including negligence),\ncontract, or otherwise, unless required by applicable law (such as deliberate\nand grossly negligent acts) or agreed to in writing, shall any Contributor be\nliable to You for damages, including any direct, indirect, special, incidental,\nor consequential damages of any character arising as a result of this License or\nout of the use or inability to use the Work (including but not limited to\ndamages for loss of goodwill, work stoppage, computer failure or malfunction, or\nany and all other commercial damages or losses), even if such Contributor has\nbeen advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability.\n\nWhile redistributing the Work or Derivative Works thereof, You may choose to\noffer, and charge a fee for, acceptance of support, warranty, indemnity, or\nother liability obligations and/or rights consistent with this License. However,\nin accepting such obligations, You may act only on Your own behalf and on Your\nsole responsibility, not on behalf of any other Contributor, and only if You\nagree to indemnify, defend, and hold each Contributor harmless for any liability\nincurred by, or claims asserted against, such Contributor by reason of your\naccepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work\n\nTo apply the Apache License to your work, attach the following boilerplate\nnotice, with the fields enclosed by brackets \"{}\" replaced with your own\nidentifying information. (Don't include the brackets!) The text should be\nenclosed in the appropriate comment syntax for the file format. We also\nrecommend that a file or class name and description of purpose be included on\nthe same \"printed page\" as the copyright notice for easier identification within\nthird-party archives.\n\n Copyright 2025 code4craft\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License."
},
{
"path": "README-zh.md",
"content": "\n\n\n[](https://maven-badges.herokuapp.com/maven-central/us.codecraft/webmagic-parent/)\n[](https://www.apache.org/licenses/LICENSE-2.0.html)\n[](https://travis-ci.org/code4craft/webmagic)\n\n官方网站[http://webmagic.io/](http://webmagic.io/)\n\n>webmagic是一个开源的Java垂直爬虫框架，目标是简化爬虫的开发流程，让开发者专注于逻辑功能的开发。webmagic的核心非常简单，但是覆盖爬虫的整个流程，也是很好的学习爬虫开发的材料。\n\n\nwebmagic的主要特色：\n\n* 完全模块化的设计，强大的可扩展性。\n* 核心简单但是涵盖爬虫的全部流程，灵活而强大，也是学习爬虫入门的好材料。\n* 提供丰富的抽取页面API。\n* 无配置，但是可通过POJO+注解形式实现一个爬虫。\n* 支持多线程。\n* 支持分布式。\n* 支持爬取js动态渲染的页面。\n* 无框架依赖，可以灵活的嵌入到项目中去。\n\nwebmagic的架构和设计参考了以下两个项目，感谢以下两个项目的作者：\n\npython爬虫 **scrapy** [https://github.com/scrapy/scrapy](https://github.com/scrapy/scrapy)\n\nJava爬虫 **Spiderman** [http://git.oschina.net/l-weiwei/spiderman](http://git.oschina.net/l-weiwei/spiderman)\n\nwebmagic的github地址：[https://github.com/code4craft/webmagic](https://github.com/code4craft/webmagic)。\n\n## 快速开始\n\n### 使用maven\n\nwebmagic使用maven管理依赖，在项目中添加对应的依赖即可使用webmagic：\n\n```xml\n\n us.codecraft\n webmagic-core\n ${webmagic.version}\n\n\n us.codecraft\n webmagic-extension\n ${webmagic.version}\n\n```\n \nWebMagic 使用slf4j-log4j12作为slf4j的实现.如果你自己定制了slf4j的实现，请在项目中去掉此依赖。\n\n```xml\n\n \n org.slf4j\n slf4j-log4j12\n \n\n```\n\n#### 项目结构\n\t\nwebmagic主要包括两个包：\n\n* **webmagic-core**\n\t\n\twebmagic核心部分，只包含爬虫基本模块和基本抽取器。webmagic-core的目标是成为网页爬虫的一个教科书般的实现。\n\t\n* **webmagic-extension**\n\t\n\twebmagic的扩展模块，提供一些更方便的编写爬虫的工具。包括注解格式定义爬虫、JSON、分布式等支持。\n\t\nwebmagic还包含两个可用的扩展包，因为这两个包都依赖了比较重量级的工具，所以从主要包中抽离出来，这些包需要下载源码后自己编译：：\n\n* **webmagic-saxon**\n\n\twebmagic与Saxon结合的模块。Saxon是一个XPath、XSLT的解析工具，webmagic依赖Saxon来进行XPath2.0语法解析支持。\n\n* **webmagic-selenium**\n\n\twebmagic与Selenium结合的模块。Selenium是一个模拟浏览器进行页面渲染的工具，webmagic依赖Selenium进行动态页面的抓取。\n\t\n在项目中，你可以根据需要依赖不同的包。\n\n### 不使用maven\n\n在项目的**lib**目录下，有依赖的所有jar包，直接在IDE里import即可。\n\n### 第一个爬虫\n\n#### 定制PageProcessor\n\nPageProcessor是webmagic-core的一部分，定制一个PageProcessor即可实现自己的爬虫逻辑。以下是抓取osc博客的一段代码：\n\n```java\npublic class OschinaBlogPageProcessor implements PageProcessor {\n\n private Site site = Site.me().setDomain(\"my.oschina.net\");\n\n @Override\n public void process(Page page) {\n List links = page.getHtml().links().regex(\"http://my\\\\.oschina\\\\.net/flashsword/blog/\\\\d+\").all();\n page.addTargetRequests(links);\n page.putField(\"title\", page.getHtml().xpath(\"//div[@class='BlogEntity']/div[@class='BlogTitle']/h1\").toString());\n page.putField(\"content\", page.getHtml().$(\"div.content\").toString());\n page.putField(\"tags\",page.getHtml().xpath(\"//div[@class='BlogTags']/a/text()\").all());\n }\n\n @Override\n public Site getSite() {\n return site;\n\n }\n\n public static void main(String[] args) {\n Spider.create(new OschinaBlogPageProcessor()).addUrl(\"http://my.oschina.net/flashsword/blog\")\n .addPipeline(new ConsolePipeline()).run();\n }\n}\n```\n\n\n这里通过page.addTargetRequests()方法来增加要抓取的URL，并通过page.putField()来保存抽取结果。page.getHtml().xpath()则是按照某个规则对结果进行抽取，这里抽取支持链式调用。调用结束后，toString()表示转化为单个String，all()则转化为一个String列表。\n\nSpider是爬虫的入口类。Pipeline是结果输出和持久化的接口，这里ConsolePipeline表示结果输出到控制台。\n\n执行这个main方法，即可在控制台看到抓取结果。webmagic默认有3秒抓取间隔，请耐心等待。\n\n#### 使用注解\n\nwebmagic-extension包括了注解方式编写爬虫的方法，只需基于一个POJO增加注解即可完成一个爬虫。以下仍然是抓取oschina博客的一段代码，功能与OschinaBlogPageProcesser完全相同：\n\n```java\n@TargetUrl(\"http://my.oschina.net/flashsword/blog/\\\\d+\")\npublic class OschinaBlog {\n\n @ExtractBy(\"//title\")\n private String title;\n\n @ExtractBy(value = \"div.BlogContent\",type = ExtractBy.Type.Css)\n private String content;\n\n @ExtractBy(value = \"//div[@class='BlogTags']/a/text()\", multi = true)\n private List tags;\n\n public static void main(String[] args) {\n OOSpider.create(\n \tSite.me(),\n\t\t\tnew ConsolePageModelPipeline(), OschinaBlog.class).addUrl(\"http://my.oschina.net/flashsword/blog\").run();\n }\n}\n```\n\n这个例子定义了一个Model类，Model类的字段'title'、'content'、'tags'均为要抽取的属性。这个类在Pipeline里是可以复用的。\n\n### 详细文档\n\n见[http://webmagic.io/docs/](http://webmagic.io/docs/)。\n\n### 示例\n\nwebmagic-samples目录里有一些定制PageProcessor以抽取不同站点的例子。\n\nwebmagic的使用可以参考：[oschina openapi 应用：博客搬家](https://git.oschina.net/yashin/MoveBlog)\n\n\n### 协议\n\nwebmagic遵循[Apache 2.0协议](http://opensource.org/licenses/Apache-2.0)\n\n### 邮件组:\n\nGmail：\n[https://groups.google.com/forum/#!forum/webmagic-java](https://groups.google.com/forum/#!forum/webmagic-java)\n\nQQ:\n[http://list.qq.com/cgi-bin/qf_invite?id=023a01f505246785f77c5a5a9aff4e57ab20fcdde871e988](http://list.qq.com/cgi-bin/qf_invite?id=023a01f505246785f77c5a5a9aff4e57ab20fcdde871e988)\n\n### QQ群：\n\n373225642(已满) 542327088\n\n### 相关项目：\n\n[Gather Platform](https://github.com/gsh199449/spider)\n\nGather Platform 数据抓取平台是一套基于Webmagic内核的,具有Web任务配置和任务管理界面的数据采集与搜索平台。\n"
},
{
"path": "README.md",
"content": "\n\n[Readme in Chinese](https://github.com/code4craft/webmagic/tree/master/README-zh.md)\n\n\n[](https://maven-badges.herokuapp.com/maven-central/us.codecraft/webmagic-parent/)\n[](https://www.apache.org/licenses/LICENSE-2.0.html)\n[](https://travis-ci.org/code4craft/webmagic)\n\n>A scalable crawler framework. It covers the whole lifecycle of crawler: downloading, url management, content extraction and persistent. It can simplify the development of a specific crawler.\n\n## Features:\n\n* Simple core with high flexibility.\n* Simple API for html extracting.\n* Annotation with POJO to customize a crawler, no configuration.\n* Multi-thread and Distribution support.\n* Easy to be integrated.\n\n## Install:\n \nAdd dependencies to your pom.xml:\n\n```xml\n\n us.codecraft\n webmagic-core\n ${webmagic.version}\n\n\n us.codecraft\n webmagic-extension\n ${webmagic.version}\n\n```\n \nWebMagic use slf4j with slf4j-log4j12 implementation. If you customized your slf4j implementation, please exclude slf4j-log4j12.\n\n```xml\n\n \n org.slf4j\n slf4j-log4j12\n \n\n```\n\n\n## Get Started:\n\n### First crawler:\n\nWrite a class implements PageProcessor. For example, I wrote a crawler of github repository information.\n\n```java\npublic class GithubRepoPageProcessor implements PageProcessor {\n\n private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);\n\n @Override\n public void process(Page page) {\n page.addTargetRequests(page.getHtml().links().regex(\"(https://github\\\\.com/\\\\w+/\\\\w+)\").all());\n page.putField(\"author\", page.getUrl().regex(\"https://github\\\\.com/(\\\\w+)/.*\").toString());\n page.putField(\"name\", page.getHtml().xpath(\"//h1[@class='public']/strong/a/text()\").toString());\n if (page.getResultItems().get(\"name\")==null){\n //skip this page\n page.setSkip(true);\n }\n page.putField(\"readme\", page.getHtml().xpath(\"//div[@id='readme']/tidyText()\"));\n }\n\n @Override\n public Site getSite() {\n return site;\n }\n\n public static void main(String[] args) {\n Spider.create(new GithubRepoPageProcessor()).addUrl(\"https://github.com/code4craft\").thread(5).run();\n }\n}\n```\n\n* `page.addTargetRequests(links)`\n\t\n\tAdd urls for crawling.\n \nYou can also use annotation way:\n\n```java\n@TargetUrl(\"https://github.com/\\\\w+/\\\\w+\")\n@HelpUrl(\"https://github.com/\\\\w+\")\npublic class GithubRepo {\n\n @ExtractBy(value = \"//h1[@class='public']/strong/a/text()\", notNull = true)\n private String name;\n\n @ExtractByUrl(\"https://github\\\\.com/(\\\\w+)/.*\")\n private String author;\n\n @ExtractBy(\"//div[@id='readme']/tidyText()\")\n private String readme;\n\n public static void main(String[] args) {\n OOSpider.create(Site.me().setSleepTime(1000)\n , new ConsolePageModelPipeline(), GithubRepo.class)\n .addUrl(\"https://github.com/code4craft\").thread(5).run();\n }\n}\n```\n\t\t\n### Docs and samples:\n\nDocuments: [http://webmagic.io/docs/](http://webmagic.io/docs/)\n\nThe architecture of webmagic (referred to [Scrapy](http://scrapy.org/))\n\n\n\nThere are more examples in `webmagic-samples` package.\n\n### License:\n\nLicensed under [Apache 2.0 license](http://opensource.org/licenses/Apache-2.0)\n\n### Thanks:\n\nTo write webmagic, I refered to the projects below :\n\n* **Scrapy**\n\n\tA crawler framework in Python.\n \n\t[http://scrapy.org/](http://scrapy.org/)\n\n* **Spiderman**\n\n\tAnother crawler framework in Java.\n\t\n\t[http://git.oschina.net/l-weiwei/spiderman](http://git.oschina.net/l-weiwei/spiderman)\n\n### Mail-list:\n\n[https://groups.google.com/forum/#!forum/webmagic-java](https://groups.google.com/forum/#!forum/webmagic-java)\n\n[http://list.qq.com/cgi-bin/qf_invite?id=023a01f505246785f77c5a5a9aff4e57ab20fcdde871e988](http://list.qq.com/cgi-bin/qf_invite?id=023a01f505246785f77c5a5a9aff4e57ab20fcdde871e988)\n\nQQ Group: 373225642 542327088\n\n### Related Project\n\n* Gather Platform\n\t\n\tA web console based on WebMagic for Spider configuration and management.\n\n"
},
{
"path": "pom.xml",
"content": "\n\n 4.0.0\n \n org.oxerr\n oxerr-parent\n 2.3.1\n \n \n us.codecraft\n 1.0.4-SNAPSHOT\n pom\n \n UTF-8\n UTF-8\n 11\n 11\n 3.23.1\n 1.5.0\n 4.4\n 2.14.0\n 3.12.0\n 2.0.19.graal\n 3.0.13\n 32.0.0-jre\n 2.29\n 4.5.13\n 4.4.15\n 3.7.1\n 9.4.12.1\n 2.9.0\n 5.10.2\n 1.10.2\n 2.7.3\n 2.23.1\n 2.0.2-beta\n 1.3.0\n 1.2.0\n 12.4\n 4.14.1\n 2.0.4\n 4.0.0.RELEASE\n 0.3.5\n \n webmagic\n webmagic\n \n A crawler framework. It covers the whole lifecycle of crawler: downloading, url management, content\n extraction and persistent. It can simply the development of a specific crawler.\n \n https://github.com/code4craft/webmagic/\n \n \n code4craft\n Yihua huang\n code4crafer@gmail.com\n \n \n yuany\n Ligang Yao\n ligang.yao@answers.com\n \n \n \n scm:git:git@github.com:code4craft/webmagic.git\n scm:git:git@github.com:code4craft/webmagic.git\n git@github.com:code4craft/webmagic.git\n WebMagic-${project.version}\n \n \n \n Apache License, Version 2.0\n http://www.apache.org/licenses/LICENSE-2.0\n \n \n\n \n webmagic-core\n webmagic-extension/\n webmagic-scripts/\n webmagic-selenium\n webmagic-saxon\n webmagic-samples\n webmagic-coverage\n \n\n \n \n org.apache.logging.log4j\n log4j-core\n test\n \n \n org.apache.logging.log4j\n log4j-slf4j2-impl\n test\n \n \n org.junit.jupiter\n junit-jupiter-engine\n test\n \n \n org.junit.vintage\n junit-vintage-engine\n test\n \n \n org.junit.platform\n junit-platform-launcher\n test\n \n \n org.junit.platform\n junit-platform-runner\n test\n \n \n\n \n \n \n org.mockito\n mockito-all\n ${mockito-all.version}\n test\n \n \n org.apache.httpcomponents\n httpclient\n ${httpclient.version}\n \n \n org.apache.httpcomponents\n httpcore\n ${httpcore.version}\n \n \n org.apache.logging.log4j\n log4j-core\n ${log4j2.version}\n \n \n org.apache.logging.log4j\n log4j-slf4j2-impl\n ${log4j2.version}\n \n \n com.google.guava\n guava\n ${guava.version}\n \n \n com.jayway.jsonpath\n json-path\n ${json-path.version}\n \n \n org.junit.jupiter\n junit-jupiter-engine\n ${junit.version}\n \n \n org.junit.vintage\n junit-vintage-engine\n ${junit.version}\n \n \n org.junit.platform\n junit-platform-launcher\n ${junit.platform.version}\n \n \n org.junit.platform\n junit-platform-runner\n ${junit.platform.version}\n \n \n org.slf4j\n slf4j-api\n ${slf4j.version}\n \n \n us.codecraft\n xsoup\n 0.3.7\n \n \n com.alibaba\n fastjson\n ${fastjson.version}\n \n \n com.github.dreamhead\n moco-core\n ${moco.version}\n test\n \n \n org.slf4j\n slf4j-simple\n \n \n \n \n org.assertj\n assertj-core\n ${assertj.version}\n test\n \n \n org.apache.commons\n commons-lang3\n ${commons-lang3.version}\n \n \n org.apache.commons\n commons-collections4\n ${commons-collections4.version}\n \n \n commons-io\n commons-io\n ${commons-io.version}\n \n \n org.codehaus.groovy\n groovy-all\n ${groovy-all.version}\n \n \n org.jruby\n jruby\n ${jruby.version}\n \n \n org.python\n jython\n ${jython.version}\n \n \n org.seleniumhq.selenium\n selenium-java\n ${selenium-java.version}\n \n \n net.sf.saxon\n Saxon-HE\n ${saxon-he.version}\n \n \n net.sourceforge.htmlcleaner\n htmlcleaner\n ${htmlcleaner.version}\n \n \n com.github.detro\n phantomjsdriver\n ${phantomjsdriver.version}\n \n \n commons-cli\n commons-cli\n ${commons-cli.version}\n \n \n redis.clients\n jedis\n ${jedis.version}\n \n \n \n\n \n \n \n org.apache.maven.plugins\n maven-javadoc-plugin\n \n WebMagic ${project.version}\n en_US\n\n \n false\n\n \n \n \n aggregate\n \n aggregate\n \n site\n \n \n attach-javadocs\n \n jar\n \n \n \n \n \n org.jacoco\n jacoco-maven-plugin\n \n \n \n prepare-agent\n \n \n \n report\n verify\n \n report\n \n \n \n \n \n com.amashchenko.maven.plugin\n gitflow-maven-plugin\n \n \n WebMagic-\n \n \n \n \n \n\n\n"
},
{
"path": "src/site/site.xml",
"content": "\n \n org.apache.maven.skins\n maven-fluido-skin\n 1.11.1\n \n \n \n \n \n \n \n \n true\n true\n true\n pull-right\n \n \n\n"
},
{
"path": "webmagic-core/README.md",
"content": "webmagic-core\n-------\nwebmagic核心部分。只包含爬虫基本模块和基本抽取器。webmagic-core的目标是成为网页爬虫的一个教科书般的实现。"
},
{
"path": "webmagic-core/module_webmagic-core.xml",
"content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n"
},
{
"path": "webmagic-core/pom.xml",
"content": "\n\n \n us.codecraft\n webmagic\n 1.0.4-SNAPSHOT\n \n 4.0.0\n\n webmagic-core\n\n \n \n org.apache.httpcomponents\n httpclient\n \n\n \n org.apache.commons\n commons-lang3\n \n\n \n us.codecraft\n xsoup\n \n\n \n com.github.dreamhead\n moco-core\n \n\n \n org.slf4j\n slf4j-api\n \n\n \n org.mockito\n mockito-all\n \n\n \n org.apache.commons\n commons-collections4\n \n\n \n org.assertj\n assertj-core\n \n\n \n commons-io\n commons-io\n \n\n \n com.jayway.jsonpath\n json-path\n \n\n \n com.alibaba\n fastjson\n \n\n \n\n\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/Page.java",
"content": "package us.codecraft.webmagic;\n\nimport org.apache.commons.lang3.StringUtils;\nimport us.codecraft.webmagic.selector.Html;\nimport us.codecraft.webmagic.selector.Json;\nimport us.codecraft.webmagic.selector.Selectable;\nimport us.codecraft.webmagic.utils.UrlUtils;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * Object storing extracted result and urls to fetch. \n * Not thread safe. \n * Main method： \n * {@link #getUrl()} get url of current page \n * {@link #getHtml()} get content of current page \n * {@link #putField(String, Object)} save extracted result \n * {@link #getResultItems()} get extract results to be used in {@link us.codecraft.webmagic.pipeline.Pipeline} \n * {@link #addTargetRequests(Iterable)} {@link #addTargetRequest(String)} add urls to fetch \n *\n * @author code4crafter@gmail.com \n * @see us.codecraft.webmagic.downloader.Downloader\n * @see us.codecraft.webmagic.processor.PageProcessor\n * @since 0.1.0\n */\npublic class Page {\n\n private Request request;\n\n private ResultItems resultItems = new ResultItems();\n\n private Html html;\n\n private Json json;\n\n private String rawText;\n\n private Selectable url;\n\n private Map> headers;\n\n private int statusCode;\n\n private boolean downloadSuccess;\n\n private byte[] bytes;\n\n private List targetRequests = new ArrayList<>();\n\n private String charset;\n\n /**\n * Returns a {@link Page} with {@link #downloadSuccess} is {@code true},\n * and {@link #request} is specified.\n *\n * @param request the request.\n * @since 1.0.2\n */\n public static Page ofSuccess(Request request) {\n return new Page(request, true);\n }\n\n /**\n * Returns a {@link Page} with {@link #downloadSuccess} is {@code true},\n * and {@link #request} is specified.\n *\n * @param request the request.\n * @since 1.0.2\n */\n public static Page ofFailure(Request request) {\n return new Page(request, false);\n }\n\n public Page() {\n }\n\n /**\n * Constructs a {@link Page} with {@link #request}\n * and {@link #downloadSuccess} specified.\n *\n * @param request the request.\n * @param downloadSuccess the download success flag.\n * @since 1.0.2\n */\n private Page(Request request, boolean downloadSuccess) {\n this.request = request;\n this.downloadSuccess = downloadSuccess;\n }\n\n /**\n * Returns a {@link Page} with {@link #downloadSuccess} is {@code false}.\n *\n * @return the page.\n * @deprecated Use {@link #fail(Request)} instead.\n */\n @Deprecated\n public static Page fail() {\n return fail(null);\n }\n\n /**\n * Returns a {@link Page} with {@link #downloadSuccess} is {@code false},\n * and {@link #request} is specified.\n *\n * @param request the {@link Request}.\n * @return the page.\n * @since 0.10.0\n * @deprecated Use {@link #ofFailure(Request)} instead.\n */\n @Deprecated(since = \"1.0.2\", forRemoval = true)\n public static Page fail(Request request){\n Page page = new Page();\n page.setRequest(request);\n page.setDownloadSuccess(false);\n return page;\n }\n\n public Page setSkip(boolean skip) {\n resultItems.setSkip(skip);\n return this;\n\n }\n\n /**\n * store extract results\n *\n * @param key key\n * @param field field\n */\n public void putField(String key, Object field) {\n resultItems.put(key, field);\n }\n\n /**\n * get html content of page\n *\n * @return html\n */\n public Html getHtml() {\n if (html == null) {\n html = new Html(rawText, request.getUrl());\n }\n return html;\n }\n\n /**\n * get json content of page\n *\n * @return json\n * @since 0.5.0\n */\n public Json getJson() {\n if (json == null) {\n json = new Json(rawText);\n }\n return json;\n }\n\n /**\n * @param html html\n * @deprecated since 0.4.0\n * The html is parse just when first time of calling {@link #getHtml()}, so use {@link #setRawText(String)} instead.\n */\n @Deprecated\n\tpublic void setHtml(Html html) {\n this.html = html;\n }\n\n public List getTargetRequests() {\n return targetRequests;\n }\n\n /**\n * add urls to fetch\n *\n * @param requests requests\n */\n public void addTargetRequests(Iterable requests) {\n \taddTargetRequests(requests, 0); // Default priority is 0\n }\n\n /**\n * add urls to fetch\n *\n * @param requests requests\n * @param priority priority\n */\n public void addTargetRequests(Iterable requests, long priority) {\n \tif(requests == null) {\n \t\treturn;\n \t}\n \t\n for (String req : requests) {\n \taddRequestIfValid(req, priority);\n }\n }\n \n /**\n * Helper method to add a request if it's valid.\n *\n * @param url URL to add\n * @param priority Priority for the URL\n */\n private void addRequestIfValid(String url, long priority) {\n if (StringUtils.isBlank(url) || url.equals(\"#\") || url.startsWith(\"javascript:\")) {\n return;\n }\n\n String canonicalizedUrl = UrlUtils.canonicalizeUrl(url, this.url.toString());\n Request req = new Request(canonicalizedUrl);\n if(priority > 0) {\n req.setPriority(priority);\n }\n targetRequests.add(req);\n }\n\n /**\n * add url to fetch\n *\n * @param requestString requestString\n */\n public void addTargetRequest(String requestString) {\n if (StringUtils.isBlank(requestString) || requestString.equals(\"#\")) {\n return;\n }\n requestString = UrlUtils.canonicalizeUrl(requestString, url.toString());\n targetRequests.add(new Request(requestString));\n }\n\n /**\n * add requests to fetch\n *\n * @param request request\n */\n public void addTargetRequest(Request request) {\n targetRequests.add(request);\n }\n\n /**\n * get url of current page\n *\n * @return url of current page\n */\n public Selectable getUrl() {\n return url;\n }\n\n public void setUrl(Selectable url) {\n this.url = url;\n }\n\n /**\n * get request of current page\n *\n * @return request\n */\n public Request getRequest() {\n return request;\n }\n\n public void setRequest(Request request) {\n this.request = request;\n this.resultItems.setRequest(request);\n }\n\n public ResultItems getResultItems() {\n return resultItems;\n }\n\n public int getStatusCode() {\n return statusCode;\n }\n\n public void setStatusCode(int statusCode) {\n this.statusCode = statusCode;\n }\n\n public String getRawText() {\n return rawText;\n }\n\n public Page setRawText(String rawText) {\n this.rawText = rawText;\n return this;\n }\n\n public Map> getHeaders() {\n return headers;\n }\n\n public void setHeaders(Map> headers) {\n this.headers = headers;\n }\n\n public boolean isDownloadSuccess() {\n return downloadSuccess;\n }\n\n public void setDownloadSuccess(boolean downloadSuccess) {\n this.downloadSuccess = downloadSuccess;\n }\n\n public byte[] getBytes() {\n return bytes;\n }\n\n public void setBytes(byte[] bytes) {\n this.bytes = bytes;\n }\n\n public String getCharset() {\n return charset;\n }\n\n public void setCharset(String charset) {\n this.charset = charset;\n }\n\n @Override\n public String toString() {\n return \"Page{\" +\n \"request=\" + request +\n \", resultItems=\" + resultItems +\n \", html=\" + html +\n \", json=\" + json +\n \", rawText='\" + rawText + '\\'' +\n \", url=\" + url +\n \", headers=\" + headers +\n \", statusCode=\" + statusCode +\n \", downloadSuccess=\" + downloadSuccess +\n \", targetRequests=\" + targetRequests +\n \", charset='\" + charset + '\\'' +\n \", bytes=\" + Arrays.toString(bytes) +\n '}';\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/Request.java",
"content": "package us.codecraft.webmagic;\n\nimport java.io.Serializable;\nimport java.util.Collections;\nimport java.util.HashMap;\nimport java.util.Map;\n\nimport us.codecraft.webmagic.downloader.Downloader;\nimport us.codecraft.webmagic.model.HttpRequestBody;\nimport us.codecraft.webmagic.utils.Experimental;\n\n/**\n * Object contains url to crawl. \n * It contains some additional information. \n *\n * @author code4crafter@gmail.com \n * @since 0.1.0\n */\npublic class Request implements Serializable {\n\n private static final long serialVersionUID = 2062192774891352043L;\n\n public static final String CYCLE_TRIED_TIMES = \"_cycle_tried_times\";\n\n private String url;\n\n private String method;\n\n private HttpRequestBody requestBody;\n\n /**\n * this req use this downloader\n */\n private Downloader downloader;\n\n /**\n * Store additional information in extras.\n */\n private Map extras = new HashMap<>();\n\n /**\n * cookies for current url, if not set use Site's cookies\n */\n private Map cookies = new HashMap();\n\n private Map headers = new HashMap();\n\n /**\n * Priority of the request. \n * The bigger will be processed earlier. \n * @see us.codecraft.webmagic.scheduler.PriorityScheduler\n */\n private long priority;\n\n /**\n * When it is set to TRUE, the downloader will not try to parse response body to text.\n *\n */\n private boolean binaryContent = false;\n\n private String charset;\n\n public Request() {\n }\n\n public Request(String url) {\n this.url = url;\n }\n\n public long getPriority() {\n return priority;\n }\n\n /**\n * Set the priority of request for sorting. \n * Need a scheduler supporting priority. \n * @see us.codecraft.webmagic.scheduler.PriorityScheduler\n *\n * @param priority priority\n * @return this\n */\n @Experimental\n public Request setPriority(long priority) {\n this.priority = priority;\n return this;\n }\n\n @SuppressWarnings(\"unchecked\")\n public T getExtra(String key) {\n if (extras == null) {\n return null;\n }\n return (T) extras.get(key);\n }\n\n public Request putExtra(String key, T value) {\n extras.put(key, value);\n return this;\n }\n\n public String getUrl() {\n return url;\n }\n\n public Map getExtras() {\n return Collections.unmodifiableMap(extras);\n }\n\n public Request setExtras(Map extras) {\n this.extras.putAll(extras);\n return this;\n }\n\n public Request setUrl(String url) {\n this.url = url;\n return this;\n }\n\n /**\n * The http method of the request. Get for default.\n * @return httpMethod\n * @see us.codecraft.webmagic.utils.HttpConstant.Method\n * @since 0.5.0\n */\n public String getMethod() {\n return method;\n }\n\n public Request setMethod(String method) {\n this.method = method;\n return this;\n }\n\n @Override\n public int hashCode() {\n int result = url != null ? url.hashCode() : 0;\n result = 31 * result + (method != null ? method.hashCode() : 0);\n return result;\n }\n\n @Override\n public boolean equals(Object o) {\n if (this == o) return true;\n if (o == null || getClass() != o.getClass()) return false;\n\n Request request = (Request) o;\n\n if (url != null ? !url.equals(request.url) : request.url != null) return false;\n return method != null ? method.equals(request.method) : request.method == null;\n }\n\n public Request addCookie(String name, String value) {\n cookies.put(name, value);\n return this;\n }\n\n public Request addHeader(String name, String value) {\n headers.put(name, value);\n return this;\n }\n\n public Map getCookies() {\n return cookies;\n }\n\n public Map getHeaders() {\n return headers;\n }\n\n public HttpRequestBody getRequestBody() {\n return requestBody;\n }\n\n public void setRequestBody(HttpRequestBody requestBody) {\n this.requestBody = requestBody;\n }\n\n public boolean isBinaryContent() {\n return binaryContent;\n }\n\n public Downloader getDownloader() {\n return downloader;\n }\n\n public void setDownloader(Downloader downloader) {\n this.downloader = downloader;\n }\n\n public Request setBinaryContent(boolean binaryContent) {\n this.binaryContent = binaryContent;\n return this;\n }\n\n public String getCharset() {\n return charset;\n }\n\n public Request setCharset(String charset) {\n this.charset = charset;\n return this;\n }\n\n @Override\n public String toString() {\n return \"Request{\" +\n \"url='\" + url + '\\'' +\n \", method='\" + method + '\\'' +\n \", extras=\" + extras +\n \", priority=\" + priority +\n \", headers=\" + headers +\n \", cookies=\"+ cookies+\n '}';\n }\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/ResultItems.java",
"content": "package us.codecraft.webmagic;\n\nimport java.util.LinkedHashMap;\nimport java.util.Map;\n\n/**\n * Object contains extract results. \n * It is contained in Page and will be processed in pipeline.\n *\n * @author code4crafter@gmail.com \n * @since 0.1.0\n * @see Page\n * @see us.codecraft.webmagic.pipeline.Pipeline\n */\npublic class ResultItems {\n\n private Map fields = new LinkedHashMap();\n\n private Request request;\n\n private boolean skip;\n\n @SuppressWarnings(\"unchecked\")\n public T get(String key) {\n Object o = fields.get(key);\n if (o == null) {\n return null;\n }\n return (T) fields.get(key);\n }\n\n public Map getAll() {\n return fields;\n }\n\n public ResultItems put(String key, T value) {\n fields.put(key, value);\n return this;\n }\n\n public Request getRequest() {\n return request;\n }\n\n public ResultItems setRequest(Request request) {\n this.request = request;\n return this;\n }\n\n /**\n * Whether to skip the result. \n * Result which is skipped will not be processed by Pipeline.\n *\n * @return whether to skip the result\n */\n public boolean isSkip() {\n return skip;\n }\n\n\n /**\n * Set whether to skip the result. \n * Result which is skipped will not be processed by Pipeline.\n *\n * @param skip whether to skip the result\n * @return this\n */\n public ResultItems setSkip(boolean skip) {\n this.skip = skip;\n return this;\n }\n\n @Override\n public String toString() {\n return \"ResultItems{\" +\n \"fields=\" + fields +\n \", request=\" + request +\n \", skip=\" + skip +\n '}';\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/Site.java",
"content": "package us.codecraft.webmagic;\n\nimport java.util.HashMap;\nimport java.util.HashSet;\nimport java.util.LinkedHashMap;\nimport java.util.Map;\nimport java.util.Set;\nimport java.util.UUID;\n\nimport us.codecraft.webmagic.utils.HttpConstant;\n\n/**\n * Object contains setting for crawler. \n *\n * @author code4crafter@gmail.com \n * @see us.codecraft.webmagic.processor.PageProcessor\n * @since 0.1.0\n */\npublic class Site {\n\n private String domain;\n\n private String userAgent;\n\n private Map defaultCookies = new LinkedHashMap();\n\n private Map> cookies = new HashMap>();\n\n private String charset;\n\n private String defaultCharset;\n\n private int sleepTime = 5000;\n\n private int retryTimes = 0;\n\n private int cycleRetryTimes = 0;\n\n private int retrySleepTime = 1000;\n\n private int timeOut = 5000;\n\n private static final Set DEFAULT_STATUS_CODE_SET = new HashSet();\n\n private Set acceptStatCode = DEFAULT_STATUS_CODE_SET;\n\n private Map headers = new HashMap();\n\n private boolean useGzip = true;\n\n private boolean disableCookieManagement = false;\n\n static {\n DEFAULT_STATUS_CODE_SET.add(HttpConstant.StatusCode.CODE_200);\n }\n\n /**\n * new a Site\n *\n * @return new site\n */\n public static Site me() {\n return new Site();\n }\n\n /**\n * Add a cookie with domain {@link #getDomain()}\n *\n * @param name name\n * @param value value\n * @return this\n */\n public Site addCookie(String name, String value) {\n defaultCookies.put(name, value);\n return this;\n }\n\n /**\n * Add a cookie with specific domain.\n *\n * @param domain domain\n * @param name name\n * @param value value\n * @return this\n */\n public Site addCookie(String domain, String name, String value) {\n if (!cookies.containsKey(domain)){\n cookies.put(domain,new HashMap());\n }\n cookies.get(domain).put(name, value);\n return this;\n }\n\n /**\n * set user agent\n *\n * @param userAgent userAgent\n * @return this\n */\n public Site setUserAgent(String userAgent) {\n this.userAgent = userAgent;\n return this;\n }\n\n /**\n * get cookies\n *\n * @return get cookies\n */\n public Map getCookies() {\n return defaultCookies;\n }\n\n /**\n * get cookies of all domains\n *\n * @return get cookies\n */\n public Map> getAllCookies() {\n return cookies;\n }\n\n /**\n * get user agent\n *\n * @return user agent\n */\n public String getUserAgent() {\n return userAgent;\n }\n\n /**\n * get domain\n *\n * @return get domain\n */\n public String getDomain() {\n return domain;\n }\n\n /**\n * set the domain of site.\n *\n * @param domain domain\n * @return this\n */\n public Site setDomain(String domain) {\n this.domain = domain;\n return this;\n }\n\n /**\n * Set charset of page manually. \n * When charset is not set or set to null, it can be auto detected by Http header.\n *\n * @param charset charset\n * @return this\n */\n public Site setCharset(String charset) {\n this.charset = charset;\n return this;\n }\n\n /**\n * get charset set manually\n *\n * @return charset\n */\n public String getCharset() {\n return charset;\n }\n\n /**\n * Set default charset of page.\n *\n * When charset detect failed, use this default charset.\n *\n * @param defaultCharset the default charset\n * @return this\n * @since 0.9.0\n */\n public Site setDefaultCharset(String defaultCharset) {\n this.defaultCharset = defaultCharset;\n return this;\n }\n\n /**\n * The default charset if charset detected failed.\n *\n * @return the defulat charset\n * @since 0.9.0\n */\n public String getDefaultCharset() {\n return defaultCharset;\n }\n\n public int getTimeOut() {\n return timeOut;\n }\n\n /**\n * set timeout for downloader in ms\n *\n * @param timeOut timeOut\n * @return this\n */\n public Site setTimeOut(int timeOut) {\n this.timeOut = timeOut;\n return this;\n }\n\n /**\n * Set acceptStatCode. \n * When status code of http response is in acceptStatCodes, it will be processed. \n * {200} by default. \n * It is not necessarily to be set. \n *\n * @param acceptStatCode acceptStatCode\n * @return this\n */\n public Site setAcceptStatCode(Set acceptStatCode) {\n this.acceptStatCode = acceptStatCode;\n return this;\n }\n\n /**\n * get acceptStatCode\n *\n * @return acceptStatCode\n */\n public Set getAcceptStatCode() {\n return acceptStatCode;\n }\n\n /**\n * Set the interval between the processing of two pages. \n * Time unit is milliseconds. \n *\n * @param sleepTime sleepTime\n * @return this\n */\n public Site setSleepTime(int sleepTime) {\n this.sleepTime = sleepTime;\n return this;\n }\n\n /**\n * Get the interval between the processing of two pages. \n * Time unit is milliseconds. \n *\n * @return the interval between the processing of two pages,\n */\n public int getSleepTime() {\n return sleepTime;\n }\n\n /**\n * Get retry times immediately when download fail, 0 by default. \n *\n * @return retry times when download fail\n */\n public int getRetryTimes() {\n return retryTimes;\n }\n\n public Map getHeaders() {\n return headers;\n }\n\n /**\n * Put an Http header for downloader. \n * Use {@link #addCookie(String, String)} for cookie and {@link #setUserAgent(String)} for user-agent. \n *\n * @param key key of http header, there are some keys constant in {@link HttpConstant.Header}\n * @param value value of header\n * @return this\n */\n public Site addHeader(String key, String value) {\n headers.put(key, value);\n return this;\n }\n\n /**\n * Set retry times when download fail, 0 by default. \n *\n * @param retryTimes retryTimes\n * @return this\n */\n public Site setRetryTimes(int retryTimes) {\n this.retryTimes = retryTimes;\n return this;\n }\n\n /**\n * When cycleRetryTimes is more than 0, it will add back to scheduler and try download again. \n *\n * @return retry times when download fail\n */\n public int getCycleRetryTimes() {\n return cycleRetryTimes;\n }\n\n /**\n * Set cycleRetryTimes times when download fail, 0 by default. \n *\n * @param cycleRetryTimes cycleRetryTimes\n * @return this\n */\n public Site setCycleRetryTimes(int cycleRetryTimes) {\n this.cycleRetryTimes = cycleRetryTimes;\n return this;\n }\n\n public boolean isUseGzip() {\n return useGzip;\n }\n\n public int getRetrySleepTime() {\n return retrySleepTime;\n }\n\n /**\n * Set retry sleep times when download fail, 1000 by default. \n *\n * @param retrySleepTime retrySleepTime\n * @return this\n */\n public Site setRetrySleepTime(int retrySleepTime) {\n this.retrySleepTime = retrySleepTime;\n return this;\n }\n\n /**\n * Whether use gzip. \n * Default is true, you can set it to false to disable gzip.\n *\n * @param useGzip useGzip\n * @return this\n */\n public Site setUseGzip(boolean useGzip) {\n this.useGzip = useGzip;\n return this;\n }\n\n public boolean isDisableCookieManagement() {\n return disableCookieManagement;\n }\n\n /**\n * Downloader is supposed to store response cookie.\n * Disable it to ignore all cookie fields and stay clean.\n * Warning: Set cookie will still NOT work if disableCookieManagement is true.\n * @param disableCookieManagement disableCookieManagement\n * @return this\n */\n public Site setDisableCookieManagement(boolean disableCookieManagement) {\n this.disableCookieManagement = disableCookieManagement;\n return this;\n }\n\n public Task toTask() {\n return new Task() {\n @Override\n public String getUUID() {\n String uuid = Site.this.getDomain();\n if (uuid == null) {\n uuid = UUID.randomUUID().toString();\n }\n return uuid;\n }\n\n @Override\n public Site getSite() {\n return Site.this;\n }\n };\n }\n\n @Override\n public boolean equals(Object o) {\n if (this == o) return true;\n if (o == null || getClass() != o.getClass()) return false;\n\n Site site = (Site) o;\n\n if (cycleRetryTimes != site.cycleRetryTimes) return false;\n if (retryTimes != site.retryTimes) return false;\n if (sleepTime != site.sleepTime) return false;\n if (timeOut != site.timeOut) return false;\n if (acceptStatCode != null ? !acceptStatCode.equals(site.acceptStatCode) : site.acceptStatCode != null)\n return false;\n if (charset != null ? !charset.equals(site.charset) : site.charset != null) return false;\n if (defaultCookies != null ? !defaultCookies.equals(site.defaultCookies) : site.defaultCookies != null)\n return false;\n if (domain != null ? !domain.equals(site.domain) : site.domain != null) return false;\n if (headers != null ? !headers.equals(site.headers) : site.headers != null) return false;\n if (userAgent != null ? !userAgent.equals(site.userAgent) : site.userAgent != null) return false;\n\n return true;\n }\n\n @Override\n public int hashCode() {\n int result = domain != null ? domain.hashCode() : 0;\n result = 31 * result + (userAgent != null ? userAgent.hashCode() : 0);\n result = 31 * result + (defaultCookies != null ? defaultCookies.hashCode() : 0);\n result = 31 * result + (charset != null ? charset.hashCode() : 0);\n result = 31 * result + sleepTime;\n result = 31 * result + retryTimes;\n result = 31 * result + cycleRetryTimes;\n result = 31 * result + timeOut;\n result = 31 * result + (acceptStatCode != null ? acceptStatCode.hashCode() : 0);\n result = 31 * result + (headers != null ? headers.hashCode() : 0);\n return result;\n }\n\n @Override\n public String toString() {\n return \"Site{\" +\n \"domain='\" + domain + '\\'' +\n \", userAgent='\" + userAgent + '\\'' +\n \", cookies=\" + defaultCookies +\n \", charset='\" + charset + '\\'' +\n \", sleepTime=\" + sleepTime +\n \", retryTimes=\" + retryTimes +\n \", cycleRetryTimes=\" + cycleRetryTimes +\n \", timeOut=\" + timeOut +\n \", acceptStatCode=\" + acceptStatCode +\n \", headers=\" + headers +\n '}';\n }\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java",
"content": "package us.codecraft.webmagic;\n\n\nimport java.io.Closeable;\nimport java.io.IOException;\nimport java.util.ArrayList;\nimport java.util.Collection;\nimport java.util.Date;\nimport java.util.List;\nimport java.util.UUID;\nimport java.util.concurrent.ExecutorService;\nimport java.util.concurrent.atomic.AtomicInteger;\nimport java.util.concurrent.atomic.AtomicLong;\nimport org.apache.commons.collections4.CollectionUtils;\nimport org.apache.commons.lang3.SerializationUtils;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport us.codecraft.webmagic.downloader.Downloader;\nimport us.codecraft.webmagic.downloader.HttpClientDownloader;\nimport us.codecraft.webmagic.pipeline.CollectorPipeline;\nimport us.codecraft.webmagic.pipeline.ConsolePipeline;\nimport us.codecraft.webmagic.pipeline.Pipeline;\nimport us.codecraft.webmagic.pipeline.ResultItemsCollectorPipeline;\nimport us.codecraft.webmagic.processor.PageProcessor;\nimport us.codecraft.webmagic.scheduler.QueueScheduler;\nimport us.codecraft.webmagic.scheduler.Scheduler;\nimport us.codecraft.webmagic.thread.CountableThreadPool;\nimport us.codecraft.webmagic.utils.UrlUtils;\nimport us.codecraft.webmagic.utils.WMCollections;\n\n/**\n * Entrance of a crawler. \n * A spider contains four modules: Downloader, Scheduler, PageProcessor and\n * Pipeline. \n * Every module is a field of Spider. \n * The modules are defined in interface. \n * You can customize a spider with various implementations of them. \n * Examples: \n * \n * A simple crawler: \n * Spider.create(new SimplePageProcessor(\"http://my.oschina.net/\",\n * \"http://my.oschina.net/*blog/*\")).run(); \n * \n * Store results to files by FilePipeline: \n * Spider.create(new SimplePageProcessor(\"http://my.oschina.net/\",\n * \"http://my.oschina.net/*blog/*\")) \n * .pipeline(new FilePipeline(\"/data/temp/webmagic/\")).run(); \n * \n * Use FileCacheQueueScheduler to store urls and cursor in files, so that a\n * Spider can resume the status when shutdown. \n * Spider.create(new SimplePageProcessor(\"http://my.oschina.net/\",\n * \"http://my.oschina.net/*blog/*\")) \n * .scheduler(new FileCacheQueueScheduler(\"/data/temp/webmagic/cache/\")).run(); \n *\n * @author code4crafter@gmail.com \n * @see Downloader\n * @see Scheduler\n * @see PageProcessor\n * @see Pipeline\n * @since 0.1.0\n */\npublic class Spider implements Runnable, Task {\n\n protected Downloader downloader;\n\n protected List pipelines = new ArrayList();\n\n protected PageProcessor pageProcessor;\n\n protected List startRequests;\n\n protected Site site;\n\n protected String uuid;\n \n protected SpiderScheduler scheduler;\n \n protected Logger logger = LoggerFactory.getLogger(getClass());\n\n protected CountableThreadPool threadPool;\n\n protected ExecutorService executorService;\n\n protected int threadNum = 1;\n\n protected AtomicInteger stat = new AtomicInteger(STAT_INIT);\n\n protected volatile boolean exitWhenComplete = true;\n\n protected final static int STAT_INIT = 0;\n\n protected final static int STAT_RUNNING = 1;\n\n protected final static int STAT_STOPPED = 2;\n\n protected boolean spawnUrl = true;\n\n protected boolean destroyWhenExit = true;\n\n private List spiderListeners;\n\n private final AtomicLong pageCount = new AtomicLong(0);\n\n private Date startTime;\n\n private long emptySleepTime = 30000;\n\n /**\n * create a spider with pageProcessor.\n *\n * @param pageProcessor pageProcessor\n * @return new spider\n * @see PageProcessor\n */\n public static Spider create(PageProcessor pageProcessor) {\n return new Spider(pageProcessor);\n }\n\n /**\n * create a spider with pageProcessor.\n *\n * @param pageProcessor pageProcessor\n */\n public Spider(PageProcessor pageProcessor) {\n this.pageProcessor = pageProcessor;\n this.site = pageProcessor.getSite();\n this.scheduler = new SpiderScheduler(new QueueScheduler());\n }\n\n /**\n * Set startUrls of Spider. \n * Prior to startUrls of Site.\n *\n * @param startUrls startUrls\n * @return this\n */\n public Spider startUrls(List startUrls) {\n checkIfRunning();\n this.startRequests = UrlUtils.convertToRequests(startUrls);\n return this;\n }\n\n /**\n * Set startUrls of Spider. \n * Prior to startUrls of Site.\n *\n * @param startRequests startRequests\n * @return this\n */\n public Spider startRequest(List startRequests) {\n checkIfRunning();\n this.startRequests = startRequests;\n return this;\n }\n\n /**\n * Set an uuid for spider. \n * Default uuid is domain of site. \n *\n * @param uuid uuid\n * @return this\n */\n public Spider setUUID(String uuid) {\n this.uuid = uuid;\n return this;\n }\n\n /**\n * set scheduler for Spider\n *\n * @param scheduler scheduler\n * @return this\n * @see #setScheduler(us.codecraft.webmagic.scheduler.Scheduler)\n */\n @Deprecated\n public Spider scheduler(Scheduler scheduler) {\n return setScheduler(scheduler);\n }\n\n /**\n * set scheduler for Spider\n *\n * @param updateScheduler scheduler\n * @return this\n * @see Scheduler\n * @since 0.2.1\n */\n public Spider setScheduler(Scheduler updateScheduler) {\n checkIfRunning();\n Scheduler oldScheduler = scheduler.getScheduler();\n scheduler.setScheduler(updateScheduler);\n if (oldScheduler != null) {\n Request request;\n while ((request = oldScheduler.poll(this)) != null) {\n this.scheduler.push(request, this);\n }\n }\n return this;\n }\n\n /**\n * add a pipeline for Spider\n *\n * @param pipeline pipeline\n * @return this\n * @see #addPipeline(us.codecraft.webmagic.pipeline.Pipeline)\n * @deprecated\n */\n @Deprecated\n public Spider pipeline(Pipeline pipeline) {\n return addPipeline(pipeline);\n }\n\n /**\n * add a pipeline for Spider\n *\n * @param pipeline pipeline\n * @return this\n * @see Pipeline\n * @since 0.2.1\n */\n public Spider addPipeline(Pipeline pipeline) {\n checkIfRunning();\n this.pipelines.add(pipeline);\n return this;\n }\n\n /**\n * set pipelines for Spider\n *\n * @param pipelines pipelines\n * @return this\n * @see Pipeline\n * @since 0.4.1\n */\n public Spider setPipelines(List pipelines) {\n checkIfRunning();\n this.pipelines = pipelines;\n return this;\n }\n\n /**\n * clear the pipelines set\n *\n * @return this\n */\n public Spider clearPipeline() {\n pipelines = new ArrayList();\n return this;\n }\n\n /**\n * set the downloader of spider\n *\n * @param downloader downloader\n * @return this\n * @see #setDownloader(us.codecraft.webmagic.downloader.Downloader)\n * @deprecated\n */\n @Deprecated\n public Spider downloader(Downloader downloader) {\n return setDownloader(downloader);\n }\n\n /**\n * set the downloader of spider\n *\n * @param downloader downloader\n * @return this\n * @see Downloader\n */\n public Spider setDownloader(Downloader downloader) {\n checkIfRunning();\n this.downloader = downloader;\n return this;\n }\n\n protected void initComponent() {\n if (downloader == null) {\n this.downloader = new HttpClientDownloader();\n }\n if (pipelines.isEmpty()) {\n pipelines.add(new ConsolePipeline());\n }\n downloader.setThread(threadNum);\n if (threadPool == null || threadPool.isShutdown()) {\n if (executorService != null && !executorService.isShutdown()) {\n threadPool = new CountableThreadPool(threadNum, executorService);\n } else {\n threadPool = new CountableThreadPool(threadNum);\n }\n }\n if (startRequests != null) {\n for (Request request : startRequests) {\n addRequest(request);\n }\n startRequests.clear();\n }\n startTime = new Date();\n }\n\n @Override\n public void run() {\n checkRunningStat();\n initComponent();\n logger.info(\"Spider {} started!\", getUUID());\n // interrupt won't be necessarily detected\n while (!Thread.currentThread().isInterrupted() && stat.get() == STAT_RUNNING) {\n Request poll = scheduler.poll(this);\n if (poll == null) {\n if (threadPool.getThreadAlive() == 0) {\n //no alive thread anymore , try again\n poll = scheduler.poll(this);\n if (poll == null) {\n if (exitWhenComplete) {\n break;\n } else {\n // wait\n try {\n Thread.sleep(emptySleepTime);\n continue;\n } catch (InterruptedException e) {\n Thread.currentThread().interrupt();\n break;\n }\n }\n }\n } else {\n // wait until new url added，\n if (scheduler.waitNewUrl(threadPool, emptySleepTime)) {\n // if interrupted\n break;\n }\n continue;\n }\n }\n final Request request = poll;\n //this may swallow the interruption\n threadPool.execute(new Runnable() {\n @Override\n public void run() {\n try {\n processRequest(request);\n onSuccess(request);\n } catch (Exception e) {\n onError(request, e);\n logger.error(\"process request \" + request + \" error\", e);\n } finally {\n pageCount.incrementAndGet();\n scheduler.signalNewUrl();\n }\n }\n });\n }\n stat.set(STAT_STOPPED);\n // release some resources\n if (destroyWhenExit) {\n close();\n }\n logger.info(\"Spider {} closed! {} pages downloaded.\", getUUID(), pageCount.get());\n }\n\n /**\n * @deprecated Use {@link #onError(Request, Exception)} instead.\n */\n @Deprecated\n protected void onError(Request request) {\n }\n\n protected void onError(Request request, Exception e) {\n this.onError(request);\n\n if (CollectionUtils.isNotEmpty(spiderListeners)) {\n for (SpiderListener spiderListener : spiderListeners) {\n spiderListener.onError(request, e);\n }\n }\n }\n\n protected void onSuccess(Request request) {\n if (CollectionUtils.isNotEmpty(spiderListeners)) {\n for (SpiderListener spiderListener : spiderListeners) {\n spiderListener.onSuccess(request);\n }\n }\n }\n\n private void checkRunningStat() {\n while (true) {\n int statNow = stat.get();\n if (statNow == STAT_RUNNING) {\n throw new IllegalStateException(\"Spider is already running!\");\n }\n if (stat.compareAndSet(statNow, STAT_RUNNING)) {\n break;\n }\n }\n }\n\n public void close() {\n destroyEach(downloader);\n destroyEach(pageProcessor);\n destroyEach(scheduler);\n for (Pipeline pipeline : pipelines) {\n destroyEach(pipeline);\n }\n threadPool.shutdown();\n }\n\n private void destroyEach(Object object) {\n if (object instanceof Closeable) {\n try {\n ((Closeable) object).close();\n } catch (IOException e) {\n e.printStackTrace();\n }\n }\n }\n\n /**\n * Process specific urls without url discovering.\n *\n * @param urls urls to process\n */\n public void test(String... urls) {\n initComponent();\n if (urls.length > 0) {\n for (String url : urls) {\n processRequest(new Request(url));\n }\n }\n }\n\n private void processRequest(Request request) {\n Page page;\n if (null != request.getDownloader()){\n page = request.getDownloader().download(request,this);\n }else {\n page = downloader.download(request, this);\n }\n if (page.isDownloadSuccess()){\n onDownloadSuccess(request, page);\n } else {\n onDownloaderFail(request);\n }\n }\n\n private void onDownloadSuccess(Request request, Page page) {\n if (site.getAcceptStatCode().contains(page.getStatusCode())){\n pageProcessor.process(page);\n extractAndAddRequests(page, spawnUrl);\n if (!page.getResultItems().isSkip()) {\n for (Pipeline pipeline : pipelines) {\n pipeline.process(page.getResultItems(), this);\n }\n }\n } else {\n logger.info(\"page status code error, page {} , code: {}\", request.getUrl(), page.getStatusCode());\n }\n sleep(site.getSleepTime());\n }\n\n private void onDownloaderFail(Request request) {\n if (site.getCycleRetryTimes() == 0) {\n sleep(site.getSleepTime());\n } else {\n // for cycle retry\n doCycleRetry(request);\n }\n }\n\n private void doCycleRetry(Request request) {\n Object cycleTriedTimesObject = request.getExtra(Request.CYCLE_TRIED_TIMES);\n if (cycleTriedTimesObject == null) {\n addRequest(SerializationUtils.clone(request).setPriority(0).putExtra(Request.CYCLE_TRIED_TIMES, 1));\n } else {\n int cycleTriedTimes = (Integer) cycleTriedTimesObject;\n cycleTriedTimes++;\n if (cycleTriedTimes < site.getCycleRetryTimes()) {\n addRequest(SerializationUtils.clone(request).setPriority(0).putExtra(Request.CYCLE_TRIED_TIMES, cycleTriedTimes));\n }\n }\n sleep(site.getRetrySleepTime());\n }\n\n protected void sleep(int time) {\n try {\n Thread.sleep(time);\n } catch (InterruptedException e) {\n logger.error(\"Thread interrupted when sleep\",e);\n Thread.currentThread().interrupt();\n }\n }\n\n protected void extractAndAddRequests(Page page, boolean spawnUrl) {\n if (spawnUrl && CollectionUtils.isNotEmpty(page.getTargetRequests())) {\n for (Request request : page.getTargetRequests()) {\n addRequest(request);\n }\n }\n }\n\n private void addRequest(Request request) {\n if (site.getDomain() == null && request != null && request.getUrl() != null) {\n site.setDomain(UrlUtils.getDomain(request.getUrl()));\n }\n scheduler.push(request, this);\n }\n\n protected void checkIfRunning() {\n if (stat.get() == STAT_RUNNING) {\n throw new IllegalStateException(\"Spider is already running!\");\n }\n }\n\n public void runAsync() {\n Thread thread = new Thread(this);\n thread.setDaemon(false);\n thread.start();\n }\n\n /**\n * Add urls to crawl. \n *\n * @param urls urls\n * @return this\n */\n public Spider addUrl(String... urls) {\n for (String url : urls) {\n addRequest(new Request(url));\n }\n scheduler.signalNewUrl();\n return this;\n }\n\n /**\n * Download urls synchronizing.\n *\n * @param urls urls\n * @param type of process result\n * @return list downloaded\n */\n public List getAll(Collection urls) {\n destroyWhenExit = false;\n spawnUrl = false;\n if (startRequests!=null){\n startRequests.clear();\n }\n for (Request request : UrlUtils.convertToRequests(urls)) {\n addRequest(request);\n }\n CollectorPipeline collectorPipeline = getCollectorPipeline();\n pipelines.add(collectorPipeline);\n run();\n spawnUrl = true;\n destroyWhenExit = true;\n return collectorPipeline.getCollected();\n }\n\n protected CollectorPipeline getCollectorPipeline() {\n return new ResultItemsCollectorPipeline();\n }\n\n public T get(String url) {\n List urls = WMCollections.newArrayList(url);\n List resultItemses = getAll(urls);\n if (resultItemses != null && resultItemses.size() > 0) {\n return resultItemses.get(0);\n } else {\n return null;\n }\n }\n\n /**\n * Add urls with information to crawl. \n *\n * @param requests requests\n * @return this\n */\n public Spider addRequest(Request... requests) {\n for (Request request : requests) {\n addRequest(request);\n }\n scheduler.signalNewUrl();\n return this;\n }\n\n public void start() {\n runAsync();\n }\n\n public void stop() {\n if (stat.compareAndSet(STAT_RUNNING, STAT_STOPPED)) {\n logger.info(\"Spider \" + getUUID() + \" stop success!\");\n } else {\n logger.info(\"Spider \" + getUUID() + \" stop fail!\");\n }\n }\n\n /**\n * Stop when all tasks in the queue are completed and all worker threads are also completed\n */\n public void stopWhenComplete(){\n this.exitWhenComplete = true;\n }\n\n /**\n * start with more than one threads\n *\n * @param threadNum threadNum\n * @return this\n */\n public Spider thread(int threadNum) {\n checkIfRunning();\n this.threadNum = threadNum;\n if (threadNum <= 0) {\n throw new IllegalArgumentException(\"threadNum should be more than one!\");\n }\n return this;\n }\n\n /**\n * start with more than one threads\n *\n * @param executorService executorService to run the spider\n * @param threadNum threadNum\n * @return this\n */\n public Spider thread(ExecutorService executorService, int threadNum) {\n checkIfRunning();\n this.threadNum = threadNum;\n if (threadNum <= 0) {\n throw new IllegalArgumentException(\"threadNum should be more than one!\");\n }\n this.executorService = executorService;\n return this;\n }\n\n public boolean isExitWhenComplete() {\n return exitWhenComplete;\n }\n\n /**\n * Exit when complete. \n * True: exit when all url of the site is downloaded. \n * False: not exit until call stop() manually. \n *\n * @param exitWhenComplete exitWhenComplete\n * @return this\n */\n public Spider setExitWhenComplete(boolean exitWhenComplete) {\n this.exitWhenComplete = exitWhenComplete;\n return this;\n }\n\n public boolean isSpawnUrl() {\n return spawnUrl;\n }\n\n /**\n * Get page count downloaded by spider.\n *\n * @return total downloaded page count\n * @since 0.4.1\n */\n public long getPageCount() {\n return pageCount.get();\n }\n\n /**\n * Get running status by spider.\n *\n * @return running status\n * @see Status\n * @since 0.4.1\n */\n public Status getStatus() {\n return Status.fromValue(stat.get());\n }\n\n\n public enum Status {\n Init(0), Running(1), Stopped(2);\n\n private Status(int value) {\n this.value = value;\n }\n\n private int value;\n\n int getValue() {\n return value;\n }\n\n public static Status fromValue(int value) {\n for (Status status : Status.values()) {\n if (status.getValue() == value) {\n return status;\n }\n }\n //default value\n return Init;\n }\n }\n\n /**\n * Get thread count which is running\n *\n * @return thread count which is running\n * @since 0.4.1\n */\n public int getThreadAlive() {\n if (threadPool == null) {\n return 0;\n }\n return threadPool.getThreadAlive();\n }\n\n /**\n * Whether add urls extracted to download. \n * Add urls to download when it is true, and just download seed urls when it is false. \n * DO NOT set it unless you know what it means!\n *\n * @param spawnUrl spawnUrl\n * @return this\n * @since 0.4.0\n */\n public Spider setSpawnUrl(boolean spawnUrl) {\n this.spawnUrl = spawnUrl;\n return this;\n }\n\n @Override\n public String getUUID() {\n if (uuid != null) {\n return uuid;\n }\n if (site != null) {\n return site.getDomain();\n }\n uuid = UUID.randomUUID().toString();\n return uuid;\n }\n\n public Spider setExecutorService(ExecutorService executorService) {\n checkIfRunning();\n this.executorService = executorService;\n return this;\n }\n\n @Override\n public Site getSite() {\n return site;\n }\n\n public List getSpiderListeners() {\n return spiderListeners;\n }\n\n public Spider setSpiderListeners(List spiderListeners) {\n this.spiderListeners = spiderListeners;\n return this;\n }\n\n public Date getStartTime() {\n return startTime;\n }\n\n public Scheduler getScheduler() {\n return scheduler.getScheduler();\n }\n\n /**\n * Set wait time when no url is polled.
\n *\n * @param emptySleepTime In MILLISECONDS.\n * @return this\n */\n public Spider setEmptySleepTime(long emptySleepTime) {\n if(emptySleepTime<=0){\n throw new IllegalArgumentException(\"emptySleepTime should be more than zero!\");\n }\n this.emptySleepTime = emptySleepTime;\n return this;\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/SpiderListener.java",
"content": "package us.codecraft.webmagic;\n\n/**\n * Listener of Spider on page processing. Used for monitor and such on.\n *\n * @author code4crafer@gmail.com\n * @since 0.5.0\n */\npublic interface SpiderListener {\n\n void onSuccess(Request request);\n\n /**\n * @deprecated Use {@link #onError(Request, Exception)} instead.\n */\n @Deprecated\n default void onError(Request request) {\n }\n\n default void onError(Request request, Exception e) {\n this.onError(request);\n }\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/SpiderScheduler.java",
"content": "package us.codecraft.webmagic;\n\nimport java.util.concurrent.TimeUnit;\nimport java.util.concurrent.locks.Condition;\nimport java.util.concurrent.locks.ReentrantLock;\n\nimport us.codecraft.webmagic.scheduler.Scheduler;\nimport us.codecraft.webmagic.thread.CountableThreadPool;\n\npublic class SpiderScheduler {\n private Scheduler scheduler;\n private final ReentrantLock newUrlLock = new ReentrantLock();\n private final Condition newUrlCondition = newUrlLock.newCondition();\n\n public SpiderScheduler(Scheduler scheduler) {\n this.scheduler = scheduler;\n }\n\n public Scheduler getScheduler() {\n return scheduler;\n }\n\n public void setScheduler(Scheduler scheduler) {\n this.scheduler = scheduler;\n }\n\n public Request poll(Spider spider) {\n return scheduler.poll(spider);\n }\n\n public void push(Request request, Spider spider) {\n scheduler.push(request, spider);\n }\n\n public boolean waitNewUrl(CountableThreadPool threadPool, long emptySleepTime) {\n newUrlLock.lock();\n try {\n if (threadPool.getThreadAlive() == 0) {\n return false;\n }\n newUrlCondition.await(emptySleepTime, TimeUnit.MILLISECONDS);\n return false;\n } catch (InterruptedException e) {\n return true;\n } finally {\n newUrlLock.unlock();\n }\n }\n\n public void signalNewUrl() {\n try {\n newUrlLock.lock();\n newUrlCondition.signalAll();\n } finally {\n newUrlLock.unlock();\n }\n }\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/Task.java",
"content": "package us.codecraft.webmagic;\n\n/**\n * Interface for identifying different tasks. \n *\n * @author code4crafter@gmail.com \n * @since 0.1.0\n * @see us.codecraft.webmagic.scheduler.Scheduler\n * @see us.codecraft.webmagic.pipeline.Pipeline\n */\npublic interface Task {\n\n /**\n * unique id for a task.\n *\n * @return uuid\n */\n public String getUUID();\n\n /**\n * site of a task\n *\n * @return site\n */\n public Site getSite();\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/downloader/AbstractDownloader.java",
"content": "package us.codecraft.webmagic.downloader;\n\nimport us.codecraft.webmagic.Page;\nimport us.codecraft.webmagic.Request;\nimport us.codecraft.webmagic.Site;\nimport us.codecraft.webmagic.Task;\nimport us.codecraft.webmagic.selector.Html;\n\n/**\n * Base class of downloader with some common methods.\n *\n * @author code4crafter@gmail.com\n * @since 0.5.0\n */\npublic abstract class AbstractDownloader implements Downloader {\n\n /**\n * A simple method to download a url.\n *\n * @param url url\n * @return html\n */\n public Html download(String url) {\n return download(url, null);\n }\n\n /**\n * A simple method to download a url.\n *\n * @param url url\n * @param charset charset\n * @return html\n */\n public Html download(String url, String charset) {\n Page page = download(new Request(url), Site.me().setCharset(charset).toTask());\n return (Html) page.getHtml();\n }\n\n /**\n * @param request the {@link Request}.\n * @deprecated Use {@link #onSuccess(Page, Task)} instead.\n */\n @Deprecated\n protected void onSuccess(Request request) {\n }\n\n /**\n * @param request the {@link Request}.\n * @param task the {@link Task}.\n * @since 0.7.6\n * @deprecated Use {@link #onSuccess(Page, Task)} instead.\n */\n @Deprecated\n protected void onSuccess(Request request, Task task) {\n this.onSuccess(request);\n }\n\n /**\n * @param page the {@link Page}.\n * @param task the {@link Task}.\n * @since 0.10.0\n */\n protected void onSuccess(Page page, Task task) {\n this.onSuccess(page.getRequest(), task);\n }\n\n /**\n * @param request the {@link Request}.\n * @deprecated Use {@link #onError(Page, Task, Throwable)} instead.\n */\n @Deprecated\n protected void onError(Request request) {\n }\n\n /**\n * @param request the {@link Request}.\n * @param task the {@link Task}.\n * @param e the exception.\n * @since 0.7.6\n * @deprecated Use {@link #onError(Page, Task, Throwable)} instead.\n */\n @Deprecated\n protected void onError(Request request, Task task, Throwable e) {\n this.onError(request);\n }\n\n /**\n * @param page the {@link Page}.\n * @param task the {@link Task}.\n * @param e the exception.\n * @since 0.10.0\n */\n protected void onError(Page page, Task task, Throwable e) {\n this.onError(page.getRequest(), task, e);\n }\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/downloader/CustomRedirectStrategy.java",
"content": "package us.codecraft.webmagic.downloader;\n\nimport java.net.URI;\n\nimport org.apache.http.HttpRequest;\nimport org.apache.http.HttpResponse;\nimport org.apache.http.ProtocolException;\nimport org.apache.http.client.methods.HttpGet;\nimport org.apache.http.client.methods.HttpPost;\nimport org.apache.http.client.methods.HttpRequestWrapper;\nimport org.apache.http.client.methods.HttpUriRequest;\nimport org.apache.http.impl.client.LaxRedirectStrategy;\nimport org.apache.http.protocol.HttpContext;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n/**\n *支持post 302跳转策略实现类\n *HttpClient默认跳转：httpClientBuilder.setRedirectStrategy(new LaxRedirectStrategy());\n *上述代码在post/redirect/post这种情况下不会传递原有请求的数据信息。所以参考了下SeimiCrawler这个项目的重定向策略。\n *原代码地址：https://github.com/zhegexiaohuozi/SeimiCrawler/blob/master/project/src/main/java/cn/wanghaomiao/seimi/http/hc/SeimiRedirectStrategy.java\n */\npublic class CustomRedirectStrategy extends LaxRedirectStrategy {\n private Logger logger = LoggerFactory.getLogger(getClass());\n\n @Override\n public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response, HttpContext context) throws ProtocolException {\n URI uri = getLocationURI(request, response, context);\n String method = request.getRequestLine().getMethod();\n if (\"post\".equalsIgnoreCase(method)) {\n try {\n HttpRequestWrapper httpRequestWrapper = (HttpRequestWrapper) request;\n httpRequestWrapper.setURI(uri);\n httpRequestWrapper.removeHeaders(\"Content-Length\");\n return httpRequestWrapper;\n } catch (Exception e) {\n logger.error(\"强转为HttpRequestWrapper出错\");\n }\n return new HttpPost(uri);\n } else {\n return new HttpGet(uri);\n }\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/downloader/Downloader.java",
"content": "package us.codecraft.webmagic.downloader;\n\nimport us.codecraft.webmagic.Page;\nimport us.codecraft.webmagic.Request;\nimport us.codecraft.webmagic.Task;\n\n/**\n * Downloader is the part that downloads web pages and store in Page object. \n * Downloader has {@link #setThread(int)} method because downloader is always the bottleneck of a crawler,\n * there are always some mechanisms such as pooling in downloader, and pool size is related to thread numbers.\n *\n * @author code4crafter@gmail.com \n * @since 0.1.0\n */\npublic interface Downloader {\n\n /**\n * Downloads web pages and store in Page object.\n *\n * @param request request\n * @param task task\n * @return page\n */\n public Page download(Request request, Task task);\n\n /**\n * Tell the downloader how many threads the spider used.\n * @param threadNum number of threads\n */\n public void setThread(int threadNum);\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java",
"content": "package us.codecraft.webmagic.downloader;\n\nimport java.io.IOException;\nimport java.nio.charset.Charset;\nimport java.util.HashMap;\nimport java.util.Map;\nimport java.util.Optional;\n\nimport org.apache.commons.io.IOUtils;\nimport org.apache.http.HttpEntity;\nimport org.apache.http.HttpResponse;\nimport org.apache.http.client.methods.CloseableHttpResponse;\nimport org.apache.http.impl.client.CloseableHttpClient;\nimport org.apache.http.util.EntityUtils;\n\nimport us.codecraft.webmagic.Page;\nimport us.codecraft.webmagic.Request;\nimport us.codecraft.webmagic.Site;\nimport us.codecraft.webmagic.Task;\nimport us.codecraft.webmagic.proxy.Proxy;\nimport us.codecraft.webmagic.proxy.ProxyProvider;\nimport us.codecraft.webmagic.selector.PlainText;\nimport us.codecraft.webmagic.utils.CharsetUtils;\nimport us.codecraft.webmagic.utils.HttpClientUtils;\n\n/**\n * The http downloader based on HttpClient.\n *\n * @author code4crafter@gmail.com \n * @since 0.1.0\n */\npublic class HttpClientDownloader extends AbstractDownloader {\n\n private final Map httpClients = new HashMap();\n\n private HttpClientGenerator httpClientGenerator = new HttpClientGenerator();\n\n private HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter();\n\n private ProxyProvider proxyProvider;\n\n private boolean responseHeader = true;\n\n public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) {\n this.httpUriRequestConverter = httpUriRequestConverter;\n }\n\n public void setProxyProvider(ProxyProvider proxyProvider) {\n this.proxyProvider = proxyProvider;\n }\n\n private CloseableHttpClient getHttpClient(Site site) {\n if (site == null) {\n return httpClientGenerator.getClient(null);\n }\n String domain = site.getDomain();\n CloseableHttpClient httpClient = httpClients.get(domain);\n if (httpClient == null) {\n synchronized (this) {\n httpClient = httpClients.get(domain);\n if (httpClient == null) {\n httpClient = httpClientGenerator.getClient(site);\n httpClients.put(domain, httpClient);\n }\n }\n }\n return httpClient;\n }\n\n @Override\n public Page download(Request request, Task task) {\n if (task == null || task.getSite() == null) {\n throw new NullPointerException(\"task or site can not be null\");\n }\n CloseableHttpResponse httpResponse = null;\n CloseableHttpClient httpClient = getHttpClient(task.getSite());\n Proxy proxy = proxyProvider != null ? proxyProvider.getProxy(request, task) : null;\n HttpClientRequestContext requestContext = httpUriRequestConverter.convert(request, task.getSite(), proxy);\n Page page = null;\n try {\n httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());\n page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);\n onSuccess(page, task);\n return page;\n } catch (IOException e) {\n page = Page.ofFailure(request);\n onError(page, task, e);\n return page;\n } finally {\n if (httpResponse != null) {\n //ensure the connection is released back to pool\n EntityUtils.consumeQuietly(httpResponse.getEntity());\n }\n if (proxyProvider != null && proxy != null) {\n proxyProvider.returnProxy(proxy, page, task);\n }\n }\n }\n\n @Override\n public void setThread(int thread) {\n httpClientGenerator.setPoolSize(thread);\n }\n\n protected Page handleResponse(Request request, String charset, HttpResponse httpResponse, Task task) throws IOException {\n HttpEntity entity = httpResponse.getEntity();\n byte[] bytes = entity != null ? IOUtils.toByteArray(entity.getContent()) : new byte[0];\n String contentType = entity != null && entity.getContentType() != null ? entity.getContentType().getValue() : null;\n Page page = Page.ofSuccess(request);\n page.setBytes(bytes);\n if (!request.isBinaryContent()) {\n if (charset == null) {\n charset = getHtmlCharset(contentType, bytes, task);\n }\n page.setCharset(charset);\n page.setRawText(new String(bytes, charset));\n }\n page.setUrl(new PlainText(request.getUrl()));\n page.setRequest(request);\n page.setStatusCode(httpResponse.getStatusLine().getStatusCode());\n if (responseHeader) {\n page.setHeaders(HttpClientUtils.convertHeaders(httpResponse.getAllHeaders()));\n }\n return page;\n }\n\n private String getHtmlCharset(String contentType, byte[] contentBytes, Task task) throws IOException {\n String charset = CharsetUtils.detectCharset(contentType, contentBytes);\n if (charset == null) {\n charset = Optional.ofNullable(task.getSite().getDefaultCharset()).orElseGet(Charset.defaultCharset()::name);\n }\n return charset;\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java",
"content": "package us.codecraft.webmagic.downloader;\n\nimport org.apache.commons.lang3.JavaVersion;\nimport org.apache.commons.lang3.SystemUtils;\nimport org.apache.http.HttpException;\nimport org.apache.http.HttpRequest;\nimport org.apache.http.HttpRequestInterceptor;\nimport org.apache.http.client.CookieStore;\nimport org.apache.http.config.Registry;\nimport org.apache.http.config.RegistryBuilder;\nimport org.apache.http.config.SocketConfig;\nimport org.apache.http.conn.socket.ConnectionSocketFactory;\nimport org.apache.http.conn.socket.PlainConnectionSocketFactory;\nimport org.apache.http.conn.ssl.SSLConnectionSocketFactory;\nimport org.apache.http.impl.client.*;\nimport org.apache.http.impl.conn.PoolingHttpClientConnectionManager;\nimport org.apache.http.impl.cookie.BasicClientCookie;\nimport org.apache.http.protocol.HttpContext;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport us.codecraft.webmagic.Site;\n\nimport javax.net.ssl.SSLContext;\nimport javax.net.ssl.TrustManager;\nimport javax.net.ssl.X509TrustManager;\nimport java.io.IOException;\nimport java.security.KeyManagementException;\nimport java.security.NoSuchAlgorithmException;\nimport java.security.cert.CertificateException;\nimport java.security.cert.X509Certificate;\nimport java.util.Map;\n\n/**\n * @author code4crafter@gmail.com \n * @since 0.4.0\n */\npublic class HttpClientGenerator {\n\n private Logger logger = LoggerFactory.getLogger(getClass());\n\n private PoolingHttpClientConnectionManager connectionManager;\n\n public HttpClientGenerator() {\n Registry reg = RegistryBuilder.create()\n .register(\"http\", PlainConnectionSocketFactory.INSTANCE)\n .register(\"https\", buildSSLConnectionSocketFactory())\n .build();\n connectionManager = new PoolingHttpClientConnectionManager(reg);\n connectionManager.setDefaultMaxPerRoute(100);\n }\n\n private SSLConnectionSocketFactory buildSSLConnectionSocketFactory() {\n try {\n SSLContext sslContext = createIgnoreVerifySSL();\n String[] supportedProtocols;\n if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_11)) {\n supportedProtocols = new String[]{\"SSLv3\", \"TLSv1\", \"TLSv1.1\", \"TLSv1.2\", \"TLSv1.3\"};\n } else {\n supportedProtocols = new String[]{\"SSLv3\", \"TLSv1\", \"TLSv1.1\", \"TLSv1.2\"};\n }\n logger.debug(\"supportedProtocols: {}\", String.join(\", \", supportedProtocols));\n return new SSLConnectionSocketFactory(sslContext, supportedProtocols,\n null,\n //不进行主机校验\n (host, sslSession) -> true); // 优先绕过安全证书\n } catch (KeyManagementException | NoSuchAlgorithmException e) {\n logger.error(\"ssl connection fail\", e);\n }\n return SSLConnectionSocketFactory.getSocketFactory();\n }\n\n private SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {\n // 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法\n X509TrustManager trustManager = new X509TrustManager() {\n\n @Override\n public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {\n }\n\n @Override\n public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {\n }\n\n @Override\n public X509Certificate[] getAcceptedIssuers() {\n return null;\n }\n\n };\n\n SSLContext sc = SSLContext.getInstance(\"TLS\");\n sc.init(null, new TrustManager[]{trustManager}, null);\n return sc;\n }\n\n public HttpClientGenerator setPoolSize(int poolSize) {\n connectionManager.setMaxTotal(poolSize);\n return this;\n }\n\n public CloseableHttpClient getClient(Site site) {\n return generateClient(site);\n }\n\n private CloseableHttpClient generateClient(Site site) {\n HttpClientBuilder httpClientBuilder = HttpClients.custom();\n\n httpClientBuilder.setConnectionManager(connectionManager);\n if (site.getUserAgent() != null) {\n httpClientBuilder.setUserAgent(site.getUserAgent());\n } else {\n httpClientBuilder.setUserAgent(\"\");\n }\n if (site.isUseGzip()) {\n httpClientBuilder.addInterceptorFirst(new HttpRequestInterceptor() {\n\n public void process(\n final HttpRequest request,\n final HttpContext context) throws HttpException, IOException {\n if (!request.containsHeader(\"Accept-Encoding\")) {\n request.addHeader(\"Accept-Encoding\", \"gzip\");\n }\n }\n });\n }\n //解决post/redirect/post 302跳转问题\n httpClientBuilder.setRedirectStrategy(new CustomRedirectStrategy());\n\n SocketConfig.Builder socketConfigBuilder = SocketConfig.custom();\n socketConfigBuilder.setSoKeepAlive(true).setTcpNoDelay(true);\n socketConfigBuilder.setSoTimeout(site.getTimeOut());\n SocketConfig socketConfig = socketConfigBuilder.build();\n httpClientBuilder.setDefaultSocketConfig(socketConfig);\n connectionManager.setDefaultSocketConfig(socketConfig);\n httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));\n generateCookie(httpClientBuilder, site);\n return httpClientBuilder.build();\n }\n\n private void generateCookie(HttpClientBuilder httpClientBuilder, Site site) {\n if (site.isDisableCookieManagement()) {\n httpClientBuilder.disableCookieManagement();\n return;\n }\n CookieStore cookieStore = new BasicCookieStore();\n for (Map.Entry cookieEntry : site.getCookies().entrySet()) {\n BasicClientCookie cookie = new BasicClientCookie(cookieEntry.getKey(), cookieEntry.getValue());\n cookie.setDomain(site.getDomain());\n cookieStore.addCookie(cookie);\n }\n for (Map.Entry> domainEntry : site.getAllCookies().entrySet()) {\n for (Map.Entry cookieEntry : domainEntry.getValue().entrySet()) {\n BasicClientCookie cookie = new BasicClientCookie(cookieEntry.getKey(), cookieEntry.getValue());\n cookie.setDomain(domainEntry.getKey());\n cookieStore.addCookie(cookie);\n }\n }\n httpClientBuilder.setDefaultCookieStore(cookieStore);\n }\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientRequestContext.java",
"content": "package us.codecraft.webmagic.downloader;\n\nimport org.apache.http.client.methods.HttpUriRequest;\nimport org.apache.http.client.protocol.HttpClientContext;\n\n/**\n * @author code4crafter@gmail.com\n * Date: 17/4/8\n * Time: 19:43\n * @since 0.7.0\n */\npublic class HttpClientRequestContext {\n\n private HttpUriRequest httpUriRequest;\n\n private HttpClientContext httpClientContext;\n\n public HttpUriRequest getHttpUriRequest() {\n return httpUriRequest;\n }\n\n public void setHttpUriRequest(HttpUriRequest httpUriRequest) {\n this.httpUriRequest = httpUriRequest;\n }\n\n public HttpClientContext getHttpClientContext() {\n return httpClientContext;\n }\n\n public void setHttpClientContext(HttpClientContext httpClientContext) {\n this.httpClientContext = httpClientContext;\n }\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpUriRequestConverter.java",
"content": "package us.codecraft.webmagic.downloader;\n\nimport org.apache.http.HttpHost;\nimport org.apache.http.auth.AuthState;\nimport org.apache.http.auth.ChallengeState;\nimport org.apache.http.auth.UsernamePasswordCredentials;\nimport org.apache.http.client.CookieStore;\nimport org.apache.http.client.config.CookieSpecs;\nimport org.apache.http.client.config.RequestConfig;\nimport org.apache.http.client.methods.HttpUriRequest;\nimport org.apache.http.client.methods.RequestBuilder;\nimport org.apache.http.client.protocol.HttpClientContext;\nimport org.apache.http.entity.ByteArrayEntity;\nimport org.apache.http.impl.auth.BasicScheme;\nimport org.apache.http.impl.client.BasicCookieStore;\nimport org.apache.http.impl.cookie.BasicClientCookie;\nimport us.codecraft.webmagic.Request;\nimport us.codecraft.webmagic.Site;\nimport us.codecraft.webmagic.proxy.Proxy;\nimport us.codecraft.webmagic.utils.HttpConstant;\nimport us.codecraft.webmagic.utils.UrlUtils;\n\nimport java.util.Map;\n\n/**\n * @author code4crafter@gmail.com\n * Date: 17/3/18\n * Time: 11:28\n *\n * @since 0.7.0\n */\npublic class HttpUriRequestConverter {\n\n public HttpClientRequestContext convert(Request request, Site site, Proxy proxy) {\n HttpClientRequestContext httpClientRequestContext = new HttpClientRequestContext();\n httpClientRequestContext.setHttpUriRequest(convertHttpUriRequest(request, site, proxy));\n httpClientRequestContext.setHttpClientContext(convertHttpClientContext(request, site, proxy));\n return httpClientRequestContext;\n }\n\n private HttpClientContext convertHttpClientContext(Request request, Site site, Proxy proxy) {\n HttpClientContext httpContext = new HttpClientContext();\n if (proxy != null && proxy.getUsername() != null) {\n AuthState authState = new AuthState();\n BasicScheme proxyAuthScheme = new BasicScheme(ChallengeState.PROXY);\n UsernamePasswordCredentials proxyCredentials = new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword());\n authState.update(proxyAuthScheme, proxyCredentials);\n httpContext.setAttribute(HttpClientContext.PROXY_AUTH_STATE, authState);\n }\n if (request.getCookies() != null && !request.getCookies().isEmpty()) {\n CookieStore cookieStore = new BasicCookieStore();\n for (Map.Entry cookieEntry : request.getCookies().entrySet()) {\n BasicClientCookie cookie1 = new BasicClientCookie(cookieEntry.getKey(), cookieEntry.getValue());\n cookie1.setDomain(UrlUtils.removePort(UrlUtils.getDomain(request.getUrl())));\n cookieStore.addCookie(cookie1);\n }\n httpContext.setCookieStore(cookieStore);\n }\n return httpContext;\n }\n\n private HttpUriRequest convertHttpUriRequest(Request request, Site site, Proxy proxy) {\n RequestBuilder requestBuilder = selectRequestMethod(request).setUri(UrlUtils.fixIllegalCharacterInUrl(request.getUrl()));\n if (site.getHeaders() != null) {\n for (Map.Entry headerEntry : site.getHeaders().entrySet()) {\n requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());\n }\n }\n\n RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();\n if (site != null) {\n requestConfigBuilder.setConnectionRequestTimeout(site.getTimeOut())\n .setSocketTimeout(site.getTimeOut())\n .setConnectTimeout(site.getTimeOut())\n .setCookieSpec(CookieSpecs.STANDARD);\n }\n\n if (proxy != null) {\n requestConfigBuilder.setProxy(new HttpHost(proxy.getHost(), proxy.getPort(), proxy.getScheme()));\n }\n requestBuilder.setConfig(requestConfigBuilder.build());\n HttpUriRequest httpUriRequest = requestBuilder.build();\n if (request.getHeaders() != null && !request.getHeaders().isEmpty()) {\n for (Map.Entry header : request.getHeaders().entrySet()) {\n httpUriRequest.addHeader(header.getKey(), header.getValue());\n }\n }\n return httpUriRequest;\n }\n\n private RequestBuilder selectRequestMethod(Request request) {\n String method = request.getMethod();\n if (method == null || method.equalsIgnoreCase(HttpConstant.Method.GET)) {\n //default get\n return RequestBuilder.get();\n } else if (method.equalsIgnoreCase(HttpConstant.Method.POST)) {\n return addFormParams(RequestBuilder.post(),request);\n } else if (method.equalsIgnoreCase(HttpConstant.Method.HEAD)) {\n return RequestBuilder.head();\n } else if (method.equalsIgnoreCase(HttpConstant.Method.PUT)) {\n return addFormParams(RequestBuilder.put(), request);\n } else if (method.equalsIgnoreCase(HttpConstant.Method.DELETE)) {\n return RequestBuilder.delete();\n } else if (method.equalsIgnoreCase(HttpConstant.Method.TRACE)) {\n return RequestBuilder.trace();\n }\n throw new IllegalArgumentException(\"Illegal HTTP Method \" + method);\n }\n\n private RequestBuilder addFormParams(RequestBuilder requestBuilder, Request request) {\n if (request.getRequestBody() != null) {\n ByteArrayEntity entity = new ByteArrayEntity(request.getRequestBody().getBody());\n entity.setContentType(request.getRequestBody().getContentType());\n requestBuilder.setEntity(entity);\n }\n return requestBuilder;\n }\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/downloader/package.html",
"content": "\n\t\nDownloader is the part that downloads web pages and store in Page object.\n\t\n\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/model/HttpRequestBody.java",
"content": "package us.codecraft.webmagic.model;\n\nimport org.apache.http.NameValuePair;\nimport org.apache.http.client.utils.URLEncodedUtils;\nimport org.apache.http.message.BasicNameValuePair;\n\nimport java.io.Serializable;\nimport java.io.UnsupportedEncodingException;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.Map;\n\n/**\n * @author code4crafter@gmail.com\n * Date: 17/4/8\n */\npublic class HttpRequestBody implements Serializable {\n\n private static final long serialVersionUID = 5659170945717023595L;\n\n public static abstract class ContentType {\n\n public static final String JSON = \"application/json\";\n\n public static final String XML = \"text/xml\";\n\n public static final String FORM = \"application/x-www-form-urlencoded\";\n\n public static final String MULTIPART = \"multipart/form-data\";\n }\n\n private byte[] body;\n\n private String contentType;\n\n private String encoding;\n\n public HttpRequestBody() {\n }\n\n public HttpRequestBody(byte[] body, String contentType, String encoding) {\n this.body = body;\n this.contentType = contentType;\n this.encoding = encoding;\n }\n\n public String getContentType() {\n return contentType;\n }\n\n public String getEncoding() {\n return encoding;\n }\n\n public void setBody(byte[] body) {\n this.body = body;\n }\n\n public void setContentType(String contentType) {\n this.contentType = contentType;\n }\n\n public void setEncoding(String encoding) {\n this.encoding = encoding;\n }\n\n public static HttpRequestBody json(String json, String encoding) {\n try {\n return new HttpRequestBody(json.getBytes(encoding), ContentType.JSON, encoding);\n } catch (UnsupportedEncodingException e) {\n throw new IllegalArgumentException(\"illegal encoding \" + encoding, e);\n }\n }\n\n public static HttpRequestBody xml(String xml, String encoding) {\n try {\n return new HttpRequestBody(xml.getBytes(encoding), ContentType.XML, encoding);\n } catch (UnsupportedEncodingException e) {\n throw new IllegalArgumentException(\"illegal encoding \" + encoding, e);\n }\n }\n\n public static HttpRequestBody custom(byte[] body, String contentType, String encoding) {\n return new HttpRequestBody(body, contentType, encoding);\n }\n\n public static HttpRequestBody form(Map params, String encoding){\n List nameValuePairs = new ArrayList(params.size());\n for (Map.Entry entry : params.entrySet()) {\n nameValuePairs.add(new BasicNameValuePair(entry.getKey(), String.valueOf(entry.getValue())));\n }\n try {\n return new HttpRequestBody(URLEncodedUtils.format(nameValuePairs, encoding).getBytes(encoding), ContentType.FORM, encoding);\n } catch (UnsupportedEncodingException e) {\n throw new IllegalArgumentException(\"illegal encoding \" + encoding, e);\n }\n }\n\n public byte[] getBody() {\n return body;\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/package.html",
"content": "\n\t\n
\n Main class \"Spider\" and models.\n
\n\t\n\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/CollectorPipeline.java",
"content": "package us.codecraft.webmagic.pipeline;\n\nimport java.util.List;\n\n/**\n * Pipeline that can collect and store results. \n * Used for {@link us.codecraft.webmagic.Spider#getAll(java.util.Collection)}\n *\n * @author code4crafter@gmail.com\n * @since 0.4.0\n */\npublic interface CollectorPipeline extends Pipeline {\n\n /**\n * Get all results collected.\n *\n * @return collected results\n */\n public List getCollected();\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ConsolePipeline.java",
"content": "package us.codecraft.webmagic.pipeline;\n\nimport us.codecraft.webmagic.ResultItems;\nimport us.codecraft.webmagic.Task;\n\nimport java.util.Map;\n\n/**\n * Write results in console. \n * Usually used in test.\n *\n * @author code4crafter@gmail.com \n * @since 0.1.0\n */\npublic class ConsolePipeline implements Pipeline {\n\n @Override\n public void process(ResultItems resultItems, Task task) {\n System.out.println(\"get page: \" + resultItems.getRequest().getUrl());\n for (Map.Entry entry : resultItems.getAll().entrySet()) {\n System.out.println(entry.getKey() + \":\\t\" + entry.getValue());\n }\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java",
"content": "package us.codecraft.webmagic.pipeline;\n\nimport org.apache.commons.codec.digest.DigestUtils;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport us.codecraft.webmagic.ResultItems;\nimport us.codecraft.webmagic.Task;\nimport us.codecraft.webmagic.utils.FilePersistentBase;\n\nimport java.io.FileOutputStream;\nimport java.io.IOException;\nimport java.io.OutputStreamWriter;\nimport java.io.PrintWriter;\nimport java.util.Map;\n\n/**\n * Store results in files. \n *\n * @author code4crafter@gmail.com \n * @since 0.1.0\n */\npublic class FilePipeline extends FilePersistentBase implements Pipeline {\n\n private Logger logger = LoggerFactory.getLogger(getClass());\n\n /**\n * create a FilePipeline with default path\"/data/webmagic/\"\n */\n public FilePipeline() {\n setPath(\"/data/webmagic/\");\n }\n\n public FilePipeline(String path) {\n setPath(path);\n }\n\n @Override\n public void process(ResultItems resultItems, Task task) {\n String path = this.path + PATH_SEPERATOR + task.getUUID() + PATH_SEPERATOR;\n try {\n PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(getFile(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl()) + \".html\")),\"UTF-8\"));\n printWriter.println(\"url:\\t\" + resultItems.getRequest().getUrl());\n for (Map.Entry entry : resultItems.getAll().entrySet()) {\n if (entry.getValue() instanceof Iterable) {\n Iterable value = (Iterable) entry.getValue();\n printWriter.println(entry.getKey() + \":\");\n for (Object o : value) {\n printWriter.println(o);\n }\n } else {\n printWriter.println(entry.getKey() + \":\\t\" + entry.getValue());\n }\n }\n printWriter.close();\n } catch (IOException e) {\n logger.warn(\"write file error\", e);\n }\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/Pipeline.java",
"content": "package us.codecraft.webmagic.pipeline;\n\nimport us.codecraft.webmagic.ResultItems;\nimport us.codecraft.webmagic.Task;\n\n/**\n * Pipeline is the persistent and offline process part of crawler. \n * The interface Pipeline can be implemented to customize ways of persistent.\n *\n * @author code4crafter@gmail.com \n * @since 0.1.0\n * @see ConsolePipeline\n * @see FilePipeline\n */\npublic interface Pipeline {\n\n /**\n * Process extracted results.\n *\n * @param resultItems resultItems\n * @param task task\n */\n public void process(ResultItems resultItems, Task task);\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ResultItemsCollectorPipeline.java",
"content": "package us.codecraft.webmagic.pipeline;\n\nimport us.codecraft.webmagic.ResultItems;\nimport us.codecraft.webmagic.Task;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n * @author code4crafter@gmail.com\n * @since 0.4.0\n */\npublic class ResultItemsCollectorPipeline implements CollectorPipeline {\n\n private List collector = new ArrayList();\n\n @Override\n public synchronized void process(ResultItems resultItems, Task task) {\n collector.add(resultItems);\n }\n\n @Override\n public List getCollected() {\n return collector;\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/package.html",
"content": "\n\t\nPipeline is the persistent and offline process part of crawler.\n\t\n\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/processor/PageProcessor.java",
"content": "package us.codecraft.webmagic.processor;\n\nimport us.codecraft.webmagic.Page;\nimport us.codecraft.webmagic.Site;\n\n/**\n * Interface to be implemented to customize a crawler.\n *\n *
\n * In PageProcessor, you can customize:\n *
\n *
\n *
start URLs and other settings in {@link Site}
\n *
how the URLs to fetch are detected
\n *
how the data are extracted and stored
\n *
\n *\n * @author code4crafter@gmail.com \n * @see Site\n * @see Page\n * @since 0.1.0\n */\npublic interface PageProcessor {\n\n /**\n * Processes the page, extract URLs to fetch, extract the data and store.\n *\n * @param page page\n */\n void process(Page page);\n\n /**\n * Returns the site settings.\n *\n * @return site\n * @see Site\n */\n default Site getSite() {\n return Site.me();\n }\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/processor/SimplePageProcessor.java",
"content": "package us.codecraft.webmagic.processor;\n\nimport us.codecraft.webmagic.Page;\nimport us.codecraft.webmagic.Site;\n\nimport java.util.List;\n\n/**\n * A simple PageProcessor.\n *\n * @author code4crafter@gmail.com \n * @since 0.1.0\n */\npublic class SimplePageProcessor implements PageProcessor {\n\n private String urlPattern;\n\n private Site site;\n\n public SimplePageProcessor(String urlPattern) {\n this.site = Site.me();\n //compile \"*\" expression to regex\n this.urlPattern = \"(\" + urlPattern.replace(\".\", \"\\\\.\").replace(\"*\", \"[^\\\"'#]*\") + \")\";\n\n }\n\n @Override\n public void process(Page page) {\n List requests = page.getHtml().links().regex(urlPattern).all();\n //add urls to fetch\n page.addTargetRequests(requests);\n //extract by XPath\n page.putField(\"title\", page.getHtml().xpath(\"//title\"));\n page.putField(\"html\", page.getHtml().toString());\n //extract by Readability\n page.putField(\"content\", page.getHtml().smartContent());\n }\n\n @Override\n public Site getSite() {\n //settings\n return site;\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/BaiduBaikePageProcessor.java",
"content": "package us.codecraft.webmagic.processor.example;\n\nimport us.codecraft.webmagic.Page;\nimport us.codecraft.webmagic.ResultItems;\nimport us.codecraft.webmagic.Site;\nimport us.codecraft.webmagic.Spider;\nimport us.codecraft.webmagic.processor.PageProcessor;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n * @author code4crafter@gmail.com \n * @since 0.4.0\n */\npublic class BaiduBaikePageProcessor implements PageProcessor {\n\n private Site site = Site.me()//.setHttpProxy(new HttpHost(\"127.0.0.1\",8888))\n .setRetryTimes(3).setSleepTime(1000).setUseGzip(true);\n\n @Override\n public void process(Page page) {\n page.putField(\"name\", page.getHtml().css(\"dl.lemmaWgt-lemmaTitle h1\",\"text\").toString());\n page.putField(\"description\", page.getHtml().xpath(\"//div[@class='lemma-summary']/allText()\"));\n }\n\n @Override\n public Site getSite() {\n return site;\n }\n\n public static void main(String[] args) {\n //single download\n Spider spider = Spider.create(new BaiduBaikePageProcessor()).thread(2);\n String urlTemplate = \"http://baike.baidu.com/search/word?word=%s&pic=1&sug=1&enc=utf8\";\n ResultItems resultItems = spider.get(String.format(urlTemplate, \"水力发电\"));\n System.out.println(resultItems);\n\n //multidownload\n List list = new ArrayList();\n list.add(String.format(urlTemplate,\"风力发电\"));\n list.add(String.format(urlTemplate,\"太阳能\"));\n list.add(String.format(urlTemplate,\"地热发电\"));\n list.add(String.format(urlTemplate,\"地热发电\"));\n List resultItemses = spider.getAll(list);\n for (ResultItems resultItemse : resultItemses) {\n System.out.println(resultItemse.getAll());\n }\n spider.close();\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/GithubRepoPageProcessor.java",
"content": "package us.codecraft.webmagic.processor.example;\n\nimport us.codecraft.webmagic.Page;\nimport us.codecraft.webmagic.Site;\nimport us.codecraft.webmagic.Spider;\nimport us.codecraft.webmagic.processor.PageProcessor;\n\n/**\n * @author code4crafter@gmail.com \n * @since 0.3.2\n */\npublic class GithubRepoPageProcessor implements PageProcessor {\n\n private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);\n\n @Override\n public void process(Page page) {\n page.addTargetRequests(page.getHtml().links().regex(\"(https://github\\\\.com/[\\\\w\\\\-]+/[\\\\w\\\\-]+)\").all());\n page.addTargetRequests(page.getHtml().links().regex(\"(https://github\\\\.com/[\\\\w\\\\-])\").all());\n page.putField(\"author\", page.getUrl().regex(\"https://github\\\\.com/(\\\\w+)/.*\").toString());\n page.putField(\"name\", page.getHtml().xpath(\"//h1[@class='public']/strong/a/text()\").toString());\n if (page.getResultItems().get(\"name\")==null){\n //skip this page\n page.setSkip(true);\n }\n page.putField(\"readme\", page.getHtml().xpath(\"//div[@id='readme']/tidyText()\"));\n }\n\n @Override\n public Site getSite() {\n return site;\n }\n\n public static void main(String[] args) {\n Spider.create(new GithubRepoPageProcessor()).addUrl(\"https://github.com/code4craft\").thread(5).run();\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/ZhihuPageProcessor.java",
"content": "package us.codecraft.webmagic.processor.example;\n\nimport us.codecraft.webmagic.Page;\nimport us.codecraft.webmagic.Site;\nimport us.codecraft.webmagic.Spider;\nimport us.codecraft.webmagic.processor.PageProcessor;\n\n/**\n * @author code4crafter@gmail.com \n * @since 0.6.0\n */\npublic class ZhihuPageProcessor implements PageProcessor {\n\n private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);\n\n @Override\n public void process(Page page) {\n page.addTargetRequests(page.getHtml().links().regex(\"https://www\\\\.zhihu\\\\.com/question/\\\\d+/answer/\\\\d+.*\").all());\n page.putField(\"title\", page.getHtml().xpath(\"//h1[@class='QuestionHeader-title']/text()\").toString());\n page.putField(\"question\", page.getHtml().xpath(\"//div[@class='QuestionRichText']//tidyText()\").toString());\n page.putField(\"answer\", page.getHtml().xpath(\"//div[@class='QuestionAnswer-content']/tidyText()\").toString());\n if (page.getResultItems().get(\"title\")==null){\n //skip this page\n page.setSkip(true);\n }\n }\n\n @Override\n public Site getSite() {\n return site;\n }\n\n public static void main(String[] args) {\n Spider.create(new ZhihuPageProcessor()).addUrl(\"https://www.zhihu.com/explore\").run();\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/processor/package.html",
"content": "\n\t\nPageProcessor custom part of a crawler for specific site.\n\t\n\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/proxy/Proxy.java",
"content": "package us.codecraft.webmagic.proxy;\n\nimport java.io.UnsupportedEncodingException;\nimport java.net.URI;\nimport java.net.URISyntaxException;\nimport java.net.URLEncoder;\nimport java.nio.charset.StandardCharsets;\n\nimport org.apache.commons.lang3.StringUtils;\n\npublic class Proxy {\n\n private String scheme;\n\n private String host;\n\n private int port;\n\n private String username;\n\n private String password;\n\n public static Proxy create(final URI uri) {\n Proxy proxy = new Proxy(uri.getHost(), uri.getPort(), uri.getScheme());\n String userInfo = uri.getUserInfo();\n if (userInfo != null) {\n String[] up = userInfo.split(\":\");\n if (up.length == 1) {\n proxy.username = up[0].isEmpty() ? null : up[0];\n } else {\n proxy.username = up[0].isEmpty() ? null : up[0];\n proxy.password = up[1].isEmpty() ? null : up[1];\n }\n }\n return proxy;\n }\n\n public Proxy(String host, int port) {\n this(host, port, null);\n }\n\n public Proxy(String host, int port, String scheme) {\n this.host = host;\n this.port = port;\n this.scheme = scheme;\n }\n\n public Proxy(String host, int port, String username, String password) {\n this.host = host;\n this.port = port;\n this.username = username;\n this.password = password;\n }\n\n public String getScheme() {\n return scheme;\n }\n\n public void setScheme(String scheme) {\n this.scheme = scheme;\n }\n\n\tpublic String getHost() {\n return host;\n }\n\n public int getPort() {\n return port;\n }\n\n public String getUsername() {\n return username;\n }\n\n public String getPassword() {\n return password;\n }\n\n public URI toURI() {\n final StringBuilder userInfoBuffer = new StringBuilder();\n if (username != null) {\n userInfoBuffer.append(urlencode(username));\n }\n if (password != null) {\n userInfoBuffer.append(\":\").append(urlencode(password));\n }\n final String userInfo = StringUtils.defaultIfEmpty(userInfoBuffer.toString(), null);\n URI uri;\n try {\n uri = new URI(scheme, userInfo, host, port, null, null, null);\n } catch (URISyntaxException e) {\n throw new IllegalArgumentException(e.getMessage(), e);\n }\n return uri;\n }\n\n private String urlencode(String s) {\n String enc = StandardCharsets.UTF_8.name();\n try {\n return URLEncoder.encode(s, enc);\n } catch (UnsupportedEncodingException e) {\n throw new IllegalArgumentException(e);\n }\n }\n\n @Override\n public boolean equals(Object o) {\n if (this == o) return true;\n if (o == null || getClass() != o.getClass()) return false;\n\n Proxy proxy = (Proxy) o;\n\n if (port != proxy.port) return false;\n if (host != null ? !host.equals(proxy.host) : proxy.host != null) return false;\n if (scheme != null ? !scheme.equals(proxy.scheme) : proxy.scheme != null) return false;\n if (username != null ? !username.equals(proxy.username) : proxy.username != null) return false;\n return password != null ? password.equals(proxy.password) : proxy.password == null;\n }\n\n @Override\n public int hashCode() {\n int result = host != null ? host.hashCode() : 0;\n result = 31 * result + port;\n result = 31 * result + (scheme != null ? scheme.hashCode() : 0);\n result = 31 * result + (username != null ? username.hashCode() : 0);\n result = 31 * result + (password != null ? password.hashCode() : 0);\n return result;\n }\n\n @Override\n public String toString() {\n return this.toURI().toString();\n }\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/proxy/ProxyProvider.java",
"content": "package us.codecraft.webmagic.proxy;\n\nimport us.codecraft.webmagic.Page;\nimport us.codecraft.webmagic.Request;\nimport us.codecraft.webmagic.Task;\n\n/**\n * Proxy provider. \n * \n * @since 0.7.0\n */\npublic interface ProxyProvider {\n\n /**\n *\n * Return proxy to Provider when complete a download.\n * @param proxy the proxy config contains host,port and identify info\n * @param page the download result\n * @param task the download task\n */\n void returnProxy(Proxy proxy, Page page, Task task);\n\n /**\n * Get a proxy for task by some strategy.\n * @param task the download task\n * @return proxy \n * @deprecated Use {@link #getProxy(Request, Task)} instead.\n */\n @Deprecated\n default Proxy getProxy(Task task) {\n throw new UnsupportedOperationException();\n }\n\n /**\n * Returns a proxy for the request.\n *\n * @param request the request\n * @param task the download task\n * @return proxy\n * @since 0.9.0\n */\n default Proxy getProxy(Request request, Task task) {\n return this.getProxy(task);\n }\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/proxy/SimpleProxyProvider.java",
"content": "package us.codecraft.webmagic.proxy;\n\nimport us.codecraft.webmagic.Page;\nimport us.codecraft.webmagic.Request;\nimport us.codecraft.webmagic.Task;\n\nimport java.util.ArrayList;\nimport java.util.Collections;\nimport java.util.List;\nimport java.util.concurrent.atomic.AtomicInteger;\n\n/**\n * A simple ProxyProvider. Provide proxy as round-robin without heartbeat and error check. It can be used when all proxies are stable.\n * @author code4crafter@gmail.com\n * Date: 17/4/16\n * Time: 10:18\n * @since 0.7.0\n */\npublic class SimpleProxyProvider implements ProxyProvider {\n\n private final List proxies;\n\n private final AtomicInteger pointer;\n\n public SimpleProxyProvider(List proxies) {\n this(proxies, new AtomicInteger(-1));\n }\n\n private SimpleProxyProvider(List proxies, AtomicInteger pointer) {\n this.proxies = proxies;\n this.pointer = pointer;\n }\n\n public static SimpleProxyProvider from(Proxy... proxies) {\n List proxiesTemp = new ArrayList(proxies.length);\n for (Proxy proxy : proxies) {\n proxiesTemp.add(proxy);\n }\n return new SimpleProxyProvider(Collections.unmodifiableList(proxiesTemp));\n }\n\n @Override\n public void returnProxy(Proxy proxy, Page page, Task task) {\n //Donothing\n }\n\n @Override\n public Proxy getProxy(Request request, Task task) {\n return proxies.get(incrForLoop());\n }\n\n private int incrForLoop() {\n int p = pointer.incrementAndGet();\n int size = proxies.size();\n if (p < size) {\n return p;\n }\n while (!pointer.compareAndSet(p, p % size)) {\n p = pointer.get();\n }\n return p % size;\n }\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/DuplicateRemovedScheduler.java",
"content": "package us.codecraft.webmagic.scheduler;\n\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\nimport us.codecraft.webmagic.Request;\nimport us.codecraft.webmagic.Task;\nimport us.codecraft.webmagic.scheduler.component.DuplicateRemover;\nimport us.codecraft.webmagic.scheduler.component.HashSetDuplicateRemover;\nimport us.codecraft.webmagic.utils.HttpConstant;\n\n/**\n * Remove duplicate urls and only push urls which are not duplicate.
\n *\n * @author code4crafer@gmail.com\n * @since 0.5.0\n */\npublic abstract class DuplicateRemovedScheduler implements Scheduler {\n\n protected Logger logger = LoggerFactory.getLogger(getClass());\n\n private DuplicateRemover duplicatedRemover = new HashSetDuplicateRemover();\n\n public DuplicateRemover getDuplicateRemover() {\n return duplicatedRemover;\n }\n\n public DuplicateRemovedScheduler setDuplicateRemover(DuplicateRemover duplicatedRemover) {\n this.duplicatedRemover = duplicatedRemover;\n return this;\n }\n\n @Override\n public void push(Request request, Task task) {\n logger.trace(\"get a candidate url {}\", request.getUrl());\n if (shouldReserved(request) || noNeedToRemoveDuplicate(request) || !duplicatedRemover.isDuplicate(request, task)) {\n logger.debug(\"push to queue {}\", request.getUrl());\n pushWhenNoDuplicate(request, task);\n }\n }\n\n protected boolean shouldReserved(Request request) {\n return request.getExtra(Request.CYCLE_TRIED_TIMES) != null;\n }\n\n protected boolean noNeedToRemoveDuplicate(Request request) {\n return HttpConstant.Method.POST.equalsIgnoreCase(request.getMethod());\n }\n\n protected void pushWhenNoDuplicate(Request request, Task task) {\n\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/MonitorableScheduler.java",
"content": "package us.codecraft.webmagic.scheduler;\n\nimport us.codecraft.webmagic.Task;\n\n/**\n * The scheduler whose requests can be counted for monitor.\n *\n * @author code4crafter@gmail.com\n * @since 0.5.0\n */\npublic interface MonitorableScheduler extends Scheduler {\n\n public int getLeftRequestsCount(Task task);\n\n public int getTotalRequestsCount(Task task);\n\n}"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/PriorityScheduler.java",
"content": "package us.codecraft.webmagic.scheduler;\n\nimport us.codecraft.webmagic.Request;\nimport us.codecraft.webmagic.Task;\nimport us.codecraft.webmagic.utils.NumberUtils;\n\nimport java.util.Comparator;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.LinkedBlockingQueue;\nimport java.util.concurrent.PriorityBlockingQueue;\n\n/**\n * Priority scheduler. Request with higher priority will poll earlier. \n *\n * @author code4crafter@gmail.com \n * @since 0.2.1\n */\npublic class PriorityScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler {\n\n public static final int INITIAL_CAPACITY = 5;\n\n private BlockingQueue noPriorityQueue = new LinkedBlockingQueue();\n\n private PriorityBlockingQueue priorityQueuePlus = new PriorityBlockingQueue(INITIAL_CAPACITY, new Comparator() {\n @Override\n public int compare(Request o1, Request o2) {\n return -NumberUtils.compareLong(o1.getPriority(), o2.getPriority());\n }\n });\n\n private PriorityBlockingQueue priorityQueueMinus = new PriorityBlockingQueue(INITIAL_CAPACITY, new Comparator() {\n @Override\n public int compare(Request o1, Request o2) {\n return -NumberUtils.compareLong(o1.getPriority(), o2.getPriority());\n }\n });\n\n @Override\n public void pushWhenNoDuplicate(Request request, Task task) {\n if (request.getPriority() == 0) {\n noPriorityQueue.add(request);\n } else if (request.getPriority() > 0) {\n priorityQueuePlus.put(request);\n } else {\n priorityQueueMinus.put(request);\n }\n }\n\n @Override\n public synchronized Request poll(Task task) {\n Request poll = priorityQueuePlus.poll();\n if (poll != null) {\n return poll;\n }\n poll = noPriorityQueue.poll();\n if (poll != null) {\n return poll;\n }\n return priorityQueueMinus.poll();\n }\n\n @Override\n public int getLeftRequestsCount(Task task) {\n return noPriorityQueue.size();\n }\n\n @Override\n public int getTotalRequestsCount(Task task) {\n return getDuplicateRemover().getTotalRequestsCount(task);\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/QueueScheduler.java",
"content": "package us.codecraft.webmagic.scheduler;\n\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.LinkedBlockingQueue;\n\nimport us.codecraft.webmagic.Request;\nimport us.codecraft.webmagic.Site;\nimport us.codecraft.webmagic.Task;\n\n/**\n * Basic Scheduler implementation. \n * Store urls to fetch in LinkedBlockingQueue and remove duplicate urls by HashMap.\n *\n * Note: if you use this {@link QueueScheduler}\n * with {@link Site#getCycleRetryTimes()} enabled, you may encountered dead-lock\n * when the queue is full.\n *\n * @author code4crafter@gmail.com \n * @since 0.1.0\n */\npublic class QueueScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler {\n\n private final BlockingQueue queue;\n\n public QueueScheduler() {\n this.queue = new LinkedBlockingQueue<>();\n }\n\n /**\n * Creates a {@code QueueScheduler} with the given (fixed) capacity.\n *\n * @param capacity the capacity of this queue,\n * see {@link LinkedBlockingQueue#LinkedBlockingQueue(int)}\n * @since 0.8.0\n */\n public QueueScheduler(int capacity) {\n this.queue = new LinkedBlockingQueue<>(capacity);\n }\n\n @Override\n public void pushWhenNoDuplicate(Request request, Task task) {\n logger.trace(\"Remaining capacity: {}\", this.queue.remainingCapacity());\n\n try {\n queue.put(request);\n } catch (InterruptedException e) {\n Thread.currentThread().interrupt();\n }\n }\n\n @Override\n public Request poll(Task task) {\n return queue.poll();\n }\n\n @Override\n public int getLeftRequestsCount(Task task) {\n return queue.size();\n }\n\n @Override\n public int getTotalRequestsCount(Task task) {\n return getDuplicateRemover().getTotalRequestsCount(task);\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/Scheduler.java",
"content": "package us.codecraft.webmagic.scheduler;\n\nimport us.codecraft.webmagic.Request;\nimport us.codecraft.webmagic.Task;\n\n/**\n * Scheduler is the part of url management. \n * You can implement interface Scheduler to do:\n * manage urls to fetch\n * remove duplicate urls\n *\n * @author code4crafter@gmail.com \n * @since 0.1.0\n */\npublic interface Scheduler {\n\n /**\n * add a url to fetch\n *\n * @param request request\n * @param task task\n */\n public void push(Request request, Task task);\n\n /**\n * get an url to crawl\n *\n * @param task the task of spider\n * @return the url to crawl\n */\n public Request poll(Task task);\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/component/DuplicateRemover.java",
"content": "package us.codecraft.webmagic.scheduler.component;\n\nimport us.codecraft.webmagic.Request;\nimport us.codecraft.webmagic.Task;\n\n/**\n * Remove duplicate requests.\n * @author code4crafer@gmail.com\n * @since 0.5.1\n */\npublic interface DuplicateRemover {\n /**\n *\n * Check whether the request is duplicate.\n *\n * @param request request\n * @param task task\n * @return true if is duplicate\n */\n public boolean isDuplicate(Request request, Task task);\n\n /**\n * Reset duplicate check.\n * @param task task\n */\n public void resetDuplicateCheck(Task task);\n\n /**\n * Get TotalRequestsCount for monitor.\n * @param task task\n * @return number of total request\n */\n public int getTotalRequestsCount(Task task);\n\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/component/HashSetDuplicateRemover.java",
"content": "package us.codecraft.webmagic.scheduler.component;\n\nimport us.codecraft.webmagic.Request;\nimport us.codecraft.webmagic.Task;\n\nimport java.util.Collections;\nimport java.util.Set;\nimport java.util.concurrent.ConcurrentHashMap;\n\n/**\n * @author code4crafer@gmail.com\n */\npublic class HashSetDuplicateRemover implements DuplicateRemover {\n\n private Set urls = Collections.newSetFromMap(new ConcurrentHashMap());\n\n @Override\n public boolean isDuplicate(Request request, Task task) {\n return !urls.add(getUrl(request));\n }\n\n protected String getUrl(Request request) {\n return request.getUrl();\n }\n\n @Override\n public void resetDuplicateCheck(Task task) {\n urls.clear();\n }\n\n @Override\n public int getTotalRequestsCount(Task task) {\n return urls.size();\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/component/package.html",
"content": "\n\t\nComponent of scheduler.\n\t\n\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/package.html",
"content": "\n\t\nScheduler is the part of url management.\n\t\n\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/selector/AbstractSelectable.java",
"content": "package us.codecraft.webmagic.selector;\n\n\nimport java.util.ArrayList;\nimport java.util.List;\n\nimport org.apache.commons.collections4.CollectionUtils;\n\n/**\n * @author code4crafer@gmail.com\n * @since 0.5.2\n */\npublic abstract class AbstractSelectable implements Selectable {\n\n protected abstract List getSourceTexts();\n\n @Override\n public Selectable css(String selector) {\n return $(selector);\n }\n\n @Override\n public Selectable css(String selector, String attrName) {\n return $(selector, attrName);\n }\n\n protected Selectable select(Selector selector, List strings) {\n List results = new ArrayList();\n for (String string : strings) {\n String result = selector.select(string);\n if (result != null) {\n results.add(result);\n }\n }\n return new PlainText(results);\n }\n\n protected Selectable selectList(Selector selector, List strings) {\n List results = new ArrayList();\n for (String string : strings) {\n List result = selector.selectList(string);\n results.addAll(result);\n }\n return new PlainText(results);\n }\n\n @Override\n public List all() {\n return getSourceTexts();\n }\n\n @Override\n public Selectable jsonPath(String jsonPath) {\n throw new UnsupportedOperationException();\n }\n\n @Override\n public String get() {\n \tList sourceTexts = all();\n if (CollectionUtils.isNotEmpty(sourceTexts)) {\n return sourceTexts.get(0);\n } \n return null;\n \n }\n\n @Override\n public Selectable select(Selector selector) {\n return select(selector, getSourceTexts());\n }\n\n @Override\n public Selectable selectList(Selector selector) {\n return selectList(selector, getSourceTexts());\n }\n\n @Override\n public Selectable regex(String regex) {\n RegexSelector regexSelector = Selectors.regex(regex);\n return selectList(regexSelector, getSourceTexts());\n }\n\n @Override\n public Selectable regex(String regex, int group) {\n RegexSelector regexSelector = Selectors.regex(regex, group);\n return selectList(regexSelector, getSourceTexts());\n }\n\n @Override\n public Selectable replace(String regex, String replacement) {\n ReplaceSelector replaceSelector = new ReplaceSelector(regex,replacement);\n return select(replaceSelector, getSourceTexts());\n }\n\n public String getFirstSourceText() {\n \tList sourceTexts = getSourceTexts();\n if (CollectionUtils.isNotEmpty(sourceTexts)) {\n return sourceTexts.get(0);\n }\n return null;\n }\n\n @Override\n public String toString() {\n return get();\n }\n\n @Override\n public boolean match() {\n return CollectionUtils.isNotEmpty(getSourceTexts());\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/selector/AndSelector.java",
"content": "package us.codecraft.webmagic.selector;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n * All selectors will be arranged as a pipeline. \n * The next selector uses the result of the previous as source.\n * @author code4crafter@gmail.com \n * @since 0.2.0\n */\npublic class AndSelector implements Selector {\n\n private List selectors = new ArrayList();\n\n public AndSelector(Selector... selectors) {\n for (Selector selector : selectors) {\n this.selectors.add(selector);\n }\n }\n\n public AndSelector(List selectors) {\n this.selectors = selectors;\n }\n\n @Override\n public String select(String text) {\n for (Selector selector : selectors) {\n if (text == null) {\n return null;\n }\n text = selector.select(text);\n }\n return text;\n }\n\n @Override\n public List selectList(String text) {\n List results = new ArrayList();\n boolean first = true;\n for (Selector selector : selectors) {\n if (first) {\n results = selector.selectList(text);\n first = false;\n } else {\n List resultsTemp = new ArrayList();\n for (String result : results) {\n resultsTemp.addAll(selector.selectList(result));\n }\n results = resultsTemp;\n if (results == null || results.size() == 0) {\n return results;\n }\n }\n }\n return results;\n }\n}\n"
},
{
"path": "webmagic-core/src/main/java/us/codecraft/webmagic/selector/BaseElementSelector.java",
"content": "package us.codecraft.webmagic.selector;\n\nimport org.jsoup.Jsoup;\nimport org.jsoup.nodes.Document;\nimport org.jsoup.nodes.Element;\nimport us.codecraft.webmagic.utils.BaseSelectorUtils;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n * @author code4crafter@gmail.com\n * @since 0.3.0\n */\npublic abstract class BaseElementSelector implements Selector, ElementSelector {\n private Document parse(String text) {\n // Jsoup could not parse