parser) {\n this.parser = parser;\n }\n\n public String contentType() {\n return contentType;\n }\n\n public Request contentType(String contentType) {\n this.contentType = contentType;\n return this;\n }\n\n public String charset() {\n return charset;\n }\n\n public Request charset(String charset) {\n this.charset = charset;\n return this;\n }\n\n public Request method(String method) {\n this.method = method;\n return this;\n }\n\n public String method() {\n return this.method;\n }\n}\n"
},
{
"path": "src/main/java/io/github/biezhi/elves/response/Body.java",
"content": "package io.github.biezhi.elves.response;\n\nimport org.jsoup.Jsoup;\nimport org.jsoup.select.Elements;\nimport us.codecraft.xsoup.XElements;\nimport us.codecraft.xsoup.Xsoup;\n\nimport java.io.BufferedReader;\nimport java.io.InputStream;\nimport java.io.InputStreamReader;\n\n/**\n * 响应Body\n *\n * @author biezhi\n * @date 2018/1/12\n */\npublic class Body {\n\n private final InputStream inputStream;\n private final String charset;\n private String bodyString;\n\n public Body(InputStream inputStream, String charset) {\n this.inputStream = inputStream;\n this.charset = charset;\n }\n\n @Override\n public String toString() {\n if (null == this.bodyString) {\n StringBuilder html = new StringBuilder(100);\n try {\n BufferedReader br = new BufferedReader(new InputStreamReader(inputStream, charset));\n String temp;\n while ((temp = br.readLine()) != null) {\n html.append(temp).append(\"\\n\");\n }\n } catch (Exception e) {\n e.printStackTrace();\n }\n this.bodyString = html.toString();\n }\n return this.bodyString;\n }\n\n public InputStream getInputStream() {\n return inputStream;\n }\n\n public Elements css(String css) {\n return Jsoup.parse(this.toString()).select(css);\n }\n\n public XElements xpath(String xpath) {\n return Xsoup.compile(xpath).evaluate(Jsoup.parse(this.toString()));\n }\n\n}\n"
},
{
"path": "src/main/java/io/github/biezhi/elves/response/Response.java",
"content": "package io.github.biezhi.elves.response;\n\nimport io.github.biezhi.elves.request.Request;\nimport lombok.Getter;\n\nimport java.io.InputStream;\n\n/**\n * 响应对象\n *\n * @author biezhi\n * @date 2018/1/11\n */\npublic class Response {\n\n @Getter\n private Request request;\n private Body body;\n\n public Response(Request request, InputStream inputStream) {\n this.request = request;\n this.body = new Body(inputStream, request.charset());\n }\n\n public Body body() {\n return body;\n }\n\n}\n"
},
{
"path": "src/main/java/io/github/biezhi/elves/response/Result.java",
"content": "package io.github.biezhi.elves.response;\n\nimport io.github.biezhi.elves.request.Request;\nimport io.github.biezhi.elves.utils.ElvesUtils;\nimport lombok.Data;\nimport lombok.NoArgsConstructor;\n\nimport java.util.ArrayList;\nimport java.util.List;\n\n/**\n * 响应结果封装\n * \n * 存储 Item 数据和新添加的 Request 列表\n *\n * @author biezhi\n * @date 2018/1/12\n */\n@Data\n@NoArgsConstructor\npublic class Result {\n\n private List requests = new ArrayList<>();\n private T item;\n\n public Result(T item) {\n this.item = item;\n }\n\n public Result addRequest(Request request) {\n this.requests.add(request);\n return this;\n }\n\n public Result addRequests(List requests) {\n if (!ElvesUtils.isEmpty(requests)) {\n this.requests.addAll(requests);\n }\n return this;\n }\n\n}\n"
},
{
"path": "src/main/java/io/github/biezhi/elves/scheduler/Scheduler.java",
"content": "package io.github.biezhi.elves.scheduler;\n\nimport io.github.biezhi.elves.request.Request;\nimport io.github.biezhi.elves.response.Response;\nimport lombok.extern.slf4j.Slf4j;\n\nimport java.util.List;\nimport java.util.concurrent.BlockingQueue;\nimport java.util.concurrent.LinkedBlockingQueue;\n\n/**\n * 爬虫调度器\n *\n * @author biezhi\n * @date 2018/1/12\n */\n@Slf4j\npublic class Scheduler {\n\n private BlockingQueue pending = new LinkedBlockingQueue<>();\n private BlockingQueue result = new LinkedBlockingQueue<>();\n\n public void addRequest(Request request) {\n try {\n this.pending.put(request);\n } catch (InterruptedException e) {\n log.error(\"向调度器添加 Request 出错\", e);\n }\n }\n\n public void addResponse(Response response) {\n try {\n this.result.put(response);\n } catch (InterruptedException e) {\n log.error(\"向调度器添加 Response 出错\", e);\n }\n }\n\n public boolean hasRequest() {\n return pending.size() > 0;\n }\n\n public Request nextRequest() {\n try {\n return pending.take();\n } catch (InterruptedException e) {\n log.error(\"从调度器获取 Request 出错\", e);\n return null;\n }\n }\n\n public boolean hasResponse() {\n return result.size() > 0;\n }\n\n public Response nextResponse() {\n try {\n return result.take();\n } catch (InterruptedException e) {\n log.error(\"从调度器获取 Response 出错\", e);\n return null;\n }\n }\n\n public void addRequests(List requests) {\n requests.forEach(this::addRequest);\n }\n\n public void clear() {\n pending.clear();\n }\n\n}\n"
},
{
"path": "src/main/java/io/github/biezhi/elves/spider/Spider.java",
"content": "package io.github.biezhi.elves.spider;\n\nimport io.github.biezhi.elves.config.Config;\nimport io.github.biezhi.elves.event.ElvesEvent;\nimport io.github.biezhi.elves.event.EventManager;\nimport io.github.biezhi.elves.pipeline.Pipeline;\nimport io.github.biezhi.elves.request.Parser;\nimport io.github.biezhi.elves.request.Request;\nimport io.github.biezhi.elves.response.Response;\nimport io.github.biezhi.elves.response.Result;\nimport lombok.Data;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\nimport java.util.List;\nimport java.util.function.Consumer;\n\n/**\n * 爬虫基类\n *\n * @author biezhi\n * @date 2018/1/11\n */\n@Data\npublic abstract class Spider {\n\n protected String name;\n protected Config config;\n protected List startUrls = new ArrayList<>();\n protected List pipelines = new ArrayList<>();\n protected List requests = new ArrayList<>();\n\n public Spider(String name) {\n this.name = name;\n EventManager.registerEvent(ElvesEvent.SPIDER_STARTED, this::onStart);\n }\n\n public Spider startUrls(String... urls) {\n this.startUrls.addAll(Arrays.asList(urls));\n return this;\n }\n\n /**\n * 爬虫启动前执行\n */\n public void onStart(Config config) {\n }\n\n /**\n * 添加 Pipeline 处理\n */\n protected Spider addPipeline(Pipeline pipeline) {\n this.pipelines.add(pipeline);\n return this;\n }\n\n /**\n * 构建一个Request\n */\n public Request makeRequest(String url) {\n return makeRequest(url, this::parse);\n }\n\n public Request makeRequest(String url, Parser parser) {\n return new Request(this, url, parser);\n }\n\n /**\n * 解析 DOM\n */\n protected abstract Result parse(Response response);\n\n protected void resetRequest(Consumer requestConsumer) {\n this.resetRequest(this.requests, requestConsumer);\n }\n\n protected void resetRequest(List requests, Consumer requestConsumer) {\n requests.forEach(requestConsumer::accept);\n }\n\n}\n"
},
{
"path": "src/main/java/io/github/biezhi/elves/utils/ElvesUtils.java",
"content": "package io.github.biezhi.elves.utils;\n\nimport java.util.Collection;\nimport java.util.concurrent.TimeUnit;\n\n/**\n * Elves Utils\n *\n * @author biezhi\n * @date 2018/1/12\n */\npublic class ElvesUtils {\n\n public static void sleep(long time){\n try {\n TimeUnit.MILLISECONDS.sleep(time);\n } catch (InterruptedException e) {\n e.printStackTrace();\n }\n }\n\n public static boolean isEmpty(Collection collection){\n return null == collection || collection.size() == 0;\n }\n\n}\n"
},
{
"path": "src/main/java/io/github/biezhi/elves/utils/NamedThreadFactory.java",
"content": "package io.github.biezhi.elves.utils;\n\nimport java.util.concurrent.ThreadFactory;\nimport java.util.concurrent.atomic.LongAdder;\n\npublic class NamedThreadFactory implements ThreadFactory {\n\n private final String prefix;\n private final LongAdder threadNumber = new LongAdder();\n\n public NamedThreadFactory(String prefix) {\n this.prefix = prefix;\n }\n\n @Override\n public Thread newThread(Runnable runnable) {\n threadNumber.add(1);\n return new Thread(runnable, prefix + \"@thread-\" + threadNumber.intValue());\n }\n}"
},
{
"path": "src/test/java/io/github/biezhi/elves/event/ElvesEventTest.java",
"content": "package io.github.biezhi.elves.event;\n\nimport io.github.biezhi.elves.Elves;\nimport io.github.biezhi.elves.config.Config;\nimport io.github.biezhi.elves.response.Response;\nimport io.github.biezhi.elves.response.Result;\nimport io.github.biezhi.elves.spider.Spider;\n\n/**\n * @author biezhi\n * @date 2018/1/12\n */\npublic class ElvesEventTest {\n\n public static void main(String[] args) {\n Elves.me(new Spider(\"测试爬虫\") {\n @Override\n public Result parse(Response response) {\n return new Result<>(response.body().toString());\n }\n }, Config.me()).onStart(config -> System.out.println(\"asasas\")).start();\n }\n\n}\n"
},
{
"path": "src/test/java/io/github/biezhi/elves/examples/DoubanExample.java",
"content": "package io.github.biezhi.elves.examples;\n\nimport io.github.biezhi.elves.Elves;\nimport io.github.biezhi.elves.config.Config;\nimport io.github.biezhi.elves.pipeline.Pipeline;\nimport io.github.biezhi.elves.request.Request;\nimport io.github.biezhi.elves.response.Response;\nimport io.github.biezhi.elves.response.Result;\nimport io.github.biezhi.elves.spider.Spider;\nimport lombok.extern.slf4j.Slf4j;\nimport org.jsoup.nodes.Element;\nimport org.jsoup.select.Elements;\n\nimport java.util.List;\nimport java.util.stream.Collectors;\n\n/**\n * 豆瓣电影示例\n *\n * @author biezhi\n * @date 2018/1/11\n */\npublic class DoubanExample {\n\n @Slf4j\n static class DoubanSpider extends Spider {\n\n public DoubanSpider(String name) {\n super(name);\n this.startUrls(\n \"https://movie.douban.com/tag/爱情\",\n \"https://movie.douban.com/tag/喜剧\",\n \"https://movie.douban.com/tag/动画\",\n \"https://movie.douban.com/tag/动作\",\n \"https://movie.douban.com/tag/史诗\",\n \"https://movie.douban.com/tag/犯罪\");\n }\n\n @Override\n public void onStart(Config config) {\n this.addPipeline((Pipeline>) (item, request) -> log.info(\"保存到文件: {}\", item));\n }\n\n public Result parse(Response response) {\n Result> result = new Result<>();\n Elements elements = response.body().css(\"#content table .pl2 a\");\n\n List titles = elements.stream().map(Element::text).collect(Collectors.toList());\n result.setItem(titles);\n\n // 获取下一页 URL\n Elements nextEl = response.body().css(\"#content > div > div.article > div.paginator > span.next > a\");\n if (null != nextEl && nextEl.size() > 0) {\n String nextPageUrl = nextEl.get(0).attr(\"href\");\n Request nextReq = this.makeRequest(nextPageUrl, this::parse);\n result.addRequest(nextReq);\n }\n return result;\n }\n\n }\n\n public static void main(String[] args) {\n DoubanSpider doubanSpider = new DoubanSpider(\"豆瓣电影\");\n Elves.me(doubanSpider, Config.me()).start();\n }\n\n}\n"
},
{
"path": "src/test/java/io/github/biezhi/elves/examples/MeiziExample.java",
"content": "package io.github.biezhi.elves.examples;\n\nimport io.github.biezhi.elves.Elves;\nimport io.github.biezhi.elves.config.Config;\nimport io.github.biezhi.elves.config.UserAgent;\nimport io.github.biezhi.elves.pipeline.Pipeline;\nimport io.github.biezhi.elves.request.Parser;\nimport io.github.biezhi.elves.request.Request;\nimport io.github.biezhi.elves.response.Response;\nimport io.github.biezhi.elves.response.Result;\nimport io.github.biezhi.elves.spider.Spider;\nimport lombok.extern.slf4j.Slf4j;\nimport org.jsoup.nodes.Element;\nimport org.jsoup.select.Elements;\n\nimport java.io.File;\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.stream.Collectors;\n\n/**\n * 妹子图示例\n *\n * @author biezhi\n * @date 2018/1/12\n */\npublic class MeiziExample {\n\n @Slf4j\n static class MeiziSpider extends Spider {\n\n private String storageDir = \"/Users/biezhi/Desktop/meizi\";\n\n public MeiziSpider(String name) {\n super(name);\n this.startUrls(\n \"http://www.meizitu.com/a/pure.html\",\n \"http://www.meizitu.com/a/cute.html\",\n \"http://www.meizitu.com/a/sexy.html\",\n \"http://www.meizitu.com/a/fuli.html\",\n \"http://www.meizitu.com/a/legs.html\");\n }\n\n @Override\n public void onStart(Config config) {\n this.addPipeline((Pipeline>) (item, request) -> {\n item.forEach(imgUrl -> {\n log.info(\"开始下载: {}\", imgUrl);\n io.github.biezhi.request.Request.get(imgUrl)\n .header(\"Referer\", request.getUrl())\n .header(\"User-Agent\", UserAgent.CHROME_FOR_MAC)\n .connectTimeout(20_000)\n .readTimeout(20_000)\n .receive(new File(storageDir, System.currentTimeMillis() + \".jpg\"));\n });\n\n log.info(\"[{}] 图片下载 OJ8K.\", request.getUrl());\n });\n\n this.requests.forEach(this::resetRequest);\n }\n\n private Request resetRequest(Request request) {\n request.contentType(\"text/html; charset=gb2312\");\n request.charset(\"gb2312\");\n return request;\n }\n\n @Override\n protected Result parse(Response response) {\n Result result = new Result<>();\n Elements elements = response.body().css(\"#maincontent > div.inWrap > ul > li:nth-child(1) > div > div > a\");\n log.info(\"elements size: {}\", elements.size());\n\n List requests = elements.stream()\n .map(element -> element.attr(\"href\"))\n .map(href -> MeiziSpider.this.makeRequest(href, new MeiziSpider.PictureParser()))\n .map(this::resetRequest)\n .collect(Collectors.toList());\n result.addRequests(requests);\n\n // 获取下一页 URL\n Optional nextEl = response.body().css(\"#wp_page_numbers > ul > li > a\").stream().filter(element -> \"下一页\".equals(element.text())).findFirst();\n if (nextEl.isPresent()) {\n String nextPageUrl = \"http://www.meizitu.com/a/\" + nextEl.get().attr(\"href\");\n Request nextReq = MeiziSpider.this.makeRequest(nextPageUrl, this::parse);\n result.addRequest(this.resetRequest(nextReq));\n }\n return result;\n }\n\n static class PictureParser implements Parser> {\n @Override\n public Result> parse(Response response) {\n Elements elements = response.body().css(\"#picture > p > img\");\n List src = elements.stream().map(element -> element.attr(\"src\")).collect(Collectors.toList());\n return new Result<>(src);\n }\n }\n\n }\n\n\n public static void main(String[] args) {\n MeiziSpider meiziSpider = new MeiziSpider(\"妹子图\");\n Elves.me(meiziSpider, Config.me().delay(3000)).start();\n }\n\n}\n"
},
{
"path": "src/test/java/io/github/biezhi/elves/examples/News163Example.java",
"content": "package io.github.biezhi.elves.examples;\n\nimport io.github.biezhi.elves.Elves;\nimport io.github.biezhi.elves.config.Config;\nimport io.github.biezhi.elves.pipeline.Pipeline;\nimport io.github.biezhi.elves.response.Response;\nimport io.github.biezhi.elves.response.Result;\nimport io.github.biezhi.elves.spider.Spider;\nimport lombok.extern.slf4j.Slf4j;\nimport org.jsoup.nodes.Element;\n\nimport java.util.List;\nimport java.util.stream.Collectors;\n\n/**\n * 网易新闻示例\n *\n * @author biezhi\n * @date 2018/1/15\n */\npublic class News163Example {\n\n @Slf4j\n static class News163Spider extends Spider {\n public News163Spider(String name) {\n super(name);\n this.startUrls(\n \"http://news.163.com/special/0001386F/rank_news.html\",\n \"http://news.163.com/special/0001386F/rank_ent.html\", // 娱乐\n \"http://news.163.com/special/0001386F/rank_sports.html\", // 体育\n \"http://news.163.com/special/0001386F/rank_tech.html\", // 科技\n \"http://news.163.com/special/0001386F/game_rank.html\", //游戏\n \"http://news.163.com/special/0001386F/rank_book.html\"); // 读书\n }\n\n @Override\n public void onStart(Config config) {\n this.addPipeline((Pipeline>) (item, request) -> item.forEach(System.out::println));\n this.requests.forEach(request -> {\n request.contentType(\"text/html; charset=gb2312\");\n request.charset(\"gb2312\");\n });\n }\n\n @Override\n protected Result parse(Response response) {\n List titles = response.body().css(\"div.areabg1 .area-half.left div.tabContents td a\").stream()\n .map(Element::text)\n .collect(Collectors.toList());\n\n return new Result(titles);\n }\n }\n\n public static void main(String[] args) {\n Elves.me(new News163Spider(\"网易新闻\")).start();\n }\n\n}\n"
},
{
"path": "src/test/java/io/github/biezhi/elves/examples/QiubaiExample.java",
"content": "package io.github.biezhi.elves.examples;\n\nimport io.github.biezhi.elves.Elves;\nimport io.github.biezhi.elves.config.Config;\nimport io.github.biezhi.elves.pipeline.Pipeline;\nimport io.github.biezhi.elves.request.Request;\nimport io.github.biezhi.elves.response.Response;\nimport io.github.biezhi.elves.response.Result;\nimport io.github.biezhi.elves.spider.Spider;\nimport lombok.extern.slf4j.Slf4j;\nimport org.jsoup.nodes.Element;\n\nimport java.util.List;\nimport java.util.Optional;\nimport java.util.stream.Collectors;\n\n/**\n * 糗事百科示例\n *\n * @author biezhi\n * @date 2018/1/15\n */\npublic class QiubaiExample {\n\n private static final String BASE_URL = \"https://www.qiushibaike.com\";\n\n @Slf4j\n static class QiubaiSpider extends Spider {\n public QiubaiSpider(String name) {\n super(name);\n this.startUrls(BASE_URL);\n }\n\n @Override\n public void onStart(Config config) {\n this.addPipeline((Pipeline>) (items, request) -> {\n log.info(\"=== 段子来了 ===\");\n items.forEach(item -> System.out.println(\"\\r\\n\" + item + \"\\r\\n============END==========\\r\\n\"));\n });\n }\n\n @Override\n protected Result parse(Response response) {\n Result result = new Result();\n\n List items = response.body().css(\"#content-left div.article div.content span\").stream()\n .map(element -> element.text().replace(\"
\", \"\\r\\n\"))\n .collect(Collectors.toList());\n\n result.setItem(items);\n\n // 下一页\n Optional nextEl = response.body().css(\"ul.pagination a span\").stream()\n .filter(element -> \"下一页\".equals(element.text()))\n .map(Element::parent)\n .findFirst();\n if (nextEl.isPresent()) {\n String nextPageUrl = BASE_URL + nextEl.get().attr(\"href\");\n Request nextReq = QiubaiSpider.this.makeRequest(nextPageUrl, this::parse);\n result.addRequest(nextReq);\n }\n return result;\n }\n }\n\n public static void main(String[] args) {\n QiubaiSpider qiubaiSpider = new QiubaiSpider(\"糗事百科\");\n Elves.me(qiubaiSpider, Config.me().delay(2000)).start();\n }\n\n}\n"
}
]
![](\"docs/static/elves.png\")
\n\n## 调用流程图\n\n![](\"docs/static/dispatch.png\")
\n\n## 快速上手\n\n搭建一个爬虫程序需要进行这么几步操作\n\n1. 编写一个爬虫类继承自 `Spider`\n2. 设置要抓取的 URL 列表\n3. 实现 `Spider` 的 `parse` 方法\n4. 添加 `Pipeline` 处理 `parse` 过滤后的数据\n\n举个栗子:\n\n```java\npublic class DoubanSpider extends Spider {\n\n public DoubanSpider(String name) {\n super(name);\n this.startUrls(\n \"https://movie.douban.com/tag/爱情\",\n \"https://movie.douban.com/tag/喜剧\",\n \"https://movie.douban.com/tag/动画\",\n \"https://movie.douban.com/tag/动作\",\n \"https://movie.douban.com/tag/史诗\",\n \"https://movie.douban.com/tag/犯罪\");\n }\n\n @Override\n public void onStart(Config config) {\n this.addPipeline((Pipeline