[
{
"path": ".idea/compiler.xml",
"content": "\n\n \n \n \n \n \n \n \n \n \n \n"
},
{
"path": ".idea/misc.xml",
"content": "\n\n \n \n \n \n \n \n \n"
},
{
"path": ".idea/vcs.xml",
"content": "\n\n \n \n \n"
},
{
"path": ".idea/workspace.xml",
"content": "\n\n \n \n \n \n \n
\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n main\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n \n
\n \n \n \n \n \n \n \n \n 1564232410913\n \n \n 1564232410913\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n 1576766569859\n \n \n \n 1576766569859\n \n \n 1576766587111\n \n \n \n 1576766587111\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n No facets are configured\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n Web|nyaCrawler\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n"
},
{
"path": "README.md",
"content": "# nyaHentaiCrawler\n喵绅士本子爬取（自用）\n\n用于抓取喵绅士的本子，启用main输入本子对应网址即可使用，个人使用版\n\n//注意！！！\n//因源站更新过几版，且添加了反扒延迟等措施，因此已不在适用，仅做参考\n\n2021/1/7更新\n\n现在添加代理，可以本地配置代理爬取，源站依然可用\n\n2021/3/15\n原网址git挂了，现有其他地址访问\n\n2021/12/3\n原网址git又可以使用了，地址直接谷歌搜索喵绅士就行\n原本准备用线程爬取，无需等待，想了下没必要能用就行，现在能看本子的地方太多了\n\n2022/1/25\n喵绅士阵亡了，可惜\n"
},
{
"path": "nyaCrawler.iml",
"content": "\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n"
},
{
"path": "pom.xml",
"content": "\n\n 4.0.0\n\n com.cn\n nyaCrawler\n 1.0-SNAPSHOT\n\n \n \n org.apache.httpcomponents\n httpclient\n 4.5.2\n \n\n\n \n commons-io\n commons-io\n 2.5\n \n\n\n \n \n org.jsoup\n jsoup\n 1.11.3\n \n\n \n\n"
},
{
"path": "src/main/java/com/cn/main/nyaPictureMain.java",
"content": "package com.cn.main;\n\nimport com.cn.util.HttpClientUtil;\nimport org.apache.commons.io.FileUtils;\nimport org.apache.http.HttpEntity;\nimport org.apache.http.HttpHost;\nimport org.apache.http.client.config.RequestConfig;\nimport org.apache.http.client.methods.CloseableHttpResponse;\nimport org.apache.http.client.methods.HttpGet;\nimport org.apache.http.impl.client.CloseableHttpClient;\nimport org.apache.http.impl.client.HttpClients;\nimport org.jsoup.Jsoup;\nimport org.jsoup.nodes.Document;\nimport org.jsoup.nodes.Element;\nimport org.jsoup.select.Elements;\n\nimport java.io.File;\nimport java.io.IOException;\nimport java.io.InputStream;\nimport java.util.ArrayList;\nimport java.util.List;\n\npublic class nyaPictureMain {\n\n //存放目录\n private static String fileSource = \"E://nyaManhua//new//\";\n\n public static void main(String[] args) throws Exception {\n\n\n List urlList = new ArrayList();\n\n //地址\n urlList.add(\"https://zha.doghentai.com/g/338012/\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n urlList.add(\"\");\n\n\n\n\n\n nyaPictureMain.crawlerNyaUrl(urlList);\n String exSite = \"cmd /c start \" + fileSource ;\n Runtime.getRuntime().exec(exSite);\n\n }\n\n\n public static void crawlerNyaPic(int picSum,String fileUrl,String intputFile,String suffix){\n\n try {\n for (int i = 1; i <= picSum; i++) {\n// suffix = \".jpg\"; //随时替换文件格式\n CloseableHttpClient httpClient = HttpClients.createDefault(); // 创建HttpClient实例\n HttpGet httpGet = new HttpGet(fileUrl+i+suffix); // 创建Httpget实例\n //设置Http报文头信息\n httpGet.setHeader(\"User-Agent\", \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36\");\n httpGet.setHeader(\"accept\", \"image/avif,image/webp,image/apng,image/*,*/*;q=0.8\");\n httpGet.setHeader(\"accept-encoding\", \"gzip, deflate, br\");\n httpGet.setHeader(\"referer\", \"https://zha.doghentai.com/\");\n httpGet.setHeader(\"sec-fetch-dest\", \"image\");\n httpGet.setHeader(\"accept-language\", \"zh-CN,zh;q=0.9,en;q=0.8\");\n\n HttpHost proxy = new HttpHost(\"127.0.0.1\", 7890);\n //超时时间单位为毫秒\n RequestConfig defaultRequestConfig = RequestConfig.custom()\n .setConnectTimeout(1000).setSocketTimeout(30000)\n .setProxy(proxy).build();\n httpClient = HttpClients.custom().setDefaultRequestConfig(defaultRequestConfig).build();\n\n\n CloseableHttpResponse response = null;\n response = httpClient.execute(httpGet); // 执行http get请求\n HttpEntity entity = response.getEntity(); // 获取返回实体\n if(null != entity){\n InputStream inputStream = entity.getContent();//返回一个输入流\n //输出图片\n FileUtils.copyInputStreamToFile(inputStream, new File(intputFile+i+suffix));//引用org.apache.commons.io.FileUtils\n System.out.println(i+suffix);\n }\n response.close(); // 关闭response\n httpClient.close(); // 关闭HttpClient实体\n\n }\n\n }catch (Exception e){\n System.out.println(e);\n }\n }\n\n\n public static void crawlerNyaUrl(List urlList) throws Exception {\n\n Integer rateDow = 1;\n for(String url:urlList){\n String html = \"\";\n if(url.length() != 0){\n html = HttpClientUtil.getSource(url);\n\n Document document = Jsoup.parse(html);\n Element element = document.selectFirst(\"div.container\").selectFirst(\"a\");\n String coverImgUrl = element.select(\"img\").attr(\"data-src\");\n\n //获取图片载点\n String[] ourStr = coverImgUrl.split(\"/\");\n //获取后缀\n String[] oursuffix = coverImgUrl.split(\"\\\\.\");\n //获取数量\n Elements picSum = document.select(\"div.thumb-container\");\n //获取本子名字\n String benziName = element.select(\"img\").attr(\"alt\");\n benziName = benziName.replaceAll(\"\\\\?\",\"\").replaceAll(\":\",\"\").replaceAll(\" \",\"\").replaceAll(\"\\\\*\",\"\");\n\n int count = picSum.size();\n int benziN = Integer.parseInt(ourStr[ourStr.length-2]);\n String suffix = \".\"+oursuffix[oursuffix.length-1];\n String fileUrl = \"https://i0.nyacdn.com/galleries/\"+benziN+\"/\";\n String intputFile = fileSource +benziName +\"//\";\n nyaPictureMain.crawlerNyaPic(count,fileUrl,intputFile,suffix);\n\n //缓存完后暂停几秒\n Thread.sleep(3000);\n }\n }\n\n System.out.println(\"喵变态图片缓存成功！！！！\");\n\n\n\n }\n\n\n}\n"
},
{
"path": "src/main/java/com/cn/util/HttpClientUtil.java",
"content": "package com.cn.util;\n\nimport org.apache.http.HttpStatus;\nimport org.apache.http.client.methods.CloseableHttpResponse;\nimport org.apache.http.client.methods.HttpGet;\nimport org.apache.http.impl.client.CloseableHttpClient;\nimport org.apache.http.impl.client.HttpClients;\nimport org.apache.http.util.EntityUtils;\n\nimport java.io.IOException;\n\npublic class HttpClientUtil {\n\n\n public static String getSource(String url) {\n String html = new String();\n HttpGet httpget = new HttpGet(url); //创建Http请求实例，URL 如：https://cd.lianjia.com/\n // 模拟浏览器，避免被服务器拒绝，返回返回403 forbidden的错误信息\n httpget.setHeader(\"User-Agent\",\n \"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36\");\n\n CloseableHttpResponse response = null;\n CloseableHttpClient httpclient = HttpClients.createDefault(); // 使用默认的HttpClient\n try {\n response = httpclient.execute(httpget);\n if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { // 返回 200 表示成功\n html = EntityUtils.toString(response.getEntity(), \"utf-8\"); // 获取服务器响应实体的内容\n }\n } catch (IOException e) {\n e.printStackTrace();\n } finally {\n if (response != null) {\n try {\n response.close();\n } catch (IOException e) {\n e.printStackTrace();\n }\n }\n }\n return html;\n }\n\n\n\n\n}\n"
},
{
"path": "web/WEB-INF/web.xml",
"content": "\n\n"
}
]