Showing preview only (6,533K chars total). Download the full file or copy to clipboard to get everything.
Repository: andeya/pholcus
Branch: master
Commit: 91a56081f6e8
Files: 391
Total size: 6.2 MB
Directory structure:
gitextract_kq_lvwwa/
├── .gitattributes
├── .gitignore
├── LICENSE
├── README.md
├── app/
│ ├── aid/
│ │ ├── history/
│ │ │ ├── failure.go
│ │ │ ├── failure_test.go
│ │ │ ├── history.go
│ │ │ ├── history_test.go
│ │ │ ├── success.go
│ │ │ └── success_test.go
│ │ └── proxy/
│ │ ├── host.go
│ │ ├── host_test.go
│ │ ├── proxy.go
│ │ └── proxy_test.go
│ ├── app.go
│ ├── app_test.go
│ ├── crawler/
│ │ ├── crawler.go
│ │ ├── crawler_test.go
│ │ ├── crawlerpool.go
│ │ ├── crawlerpool_test.go
│ │ ├── spiderqueue.go
│ │ └── spiderqueue_test.go
│ ├── distribute/
│ │ ├── integration_test.go
│ │ ├── interface.go
│ │ ├── master_api.go
│ │ ├── master_api_test.go
│ │ ├── slave_api.go
│ │ ├── slave_api_test.go
│ │ ├── task.go
│ │ ├── task_test.go
│ │ ├── taskjar.go
│ │ ├── taskjar_test.go
│ │ └── teleport/
│ │ ├── client.go
│ │ ├── conn.go
│ │ ├── conn_test.go
│ │ ├── debug.go
│ │ ├── netdata.go
│ │ ├── netdata_test.go
│ │ ├── protocol.go
│ │ ├── protocol_test.go
│ │ ├── return_func.go
│ │ ├── return_func_test.go
│ │ ├── server.go
│ │ ├── teleport.go
│ │ ├── teleport_test.go
│ │ ├── util.go
│ │ └── util_test.go
│ ├── downloader/
│ │ ├── downloader.go
│ │ ├── downloader_surfer.go
│ │ ├── downloader_test.go
│ │ ├── request/
│ │ │ ├── request.go
│ │ │ ├── request_test.go
│ │ │ └── temp.go
│ │ └── surfer/
│ │ ├── agent/
│ │ │ ├── agent.go
│ │ │ ├── agent_bsd.go
│ │ │ ├── agent_linux.go
│ │ │ ├── agent_linux_arm.go
│ │ │ ├── agent_test.go
│ │ │ └── agent_windows.go
│ │ ├── chrome.go
│ │ ├── chrome_stub.go
│ │ ├── chrome_test.go
│ │ ├── example/
│ │ │ └── example.go
│ │ ├── param.go
│ │ ├── param_test.go
│ │ ├── phantom.go
│ │ ├── phantom_stub.go
│ │ ├── request.go
│ │ ├── request_test.go
│ │ ├── surf.go
│ │ ├── surf_stub_test.go
│ │ ├── surf_test.go
│ │ ├── surfer.go
│ │ ├── util.go
│ │ └── util_test.go
│ ├── pipeline/
│ │ ├── collector/
│ │ │ ├── collector.go
│ │ │ ├── collector_test.go
│ │ │ ├── data/
│ │ │ │ ├── data.go
│ │ │ │ └── data_test.go
│ │ │ ├── output_beanstalkd.go
│ │ │ ├── output_beanstalkd_stub.go
│ │ │ ├── output_csv.go
│ │ │ ├── output_data.go
│ │ │ ├── output_data_test.go
│ │ │ ├── output_excel.go
│ │ │ ├── output_file.go
│ │ │ ├── output_kafka.go
│ │ │ ├── output_kafka_stub.go
│ │ │ ├── output_mgo.go
│ │ │ ├── output_mgo_stub.go
│ │ │ ├── output_mysql.go
│ │ │ ├── output_mysql_stub.go
│ │ │ ├── output_util.go
│ │ │ └── output_util_test.go
│ │ ├── output.go
│ │ ├── pipeline.go
│ │ └── pipeline_test.go
│ ├── scheduler/
│ │ ├── matrix.go
│ │ ├── scheduler.go
│ │ └── scheduler_test.go
│ └── spider/
│ ├── common/
│ │ ├── common.go
│ │ ├── common_test.go
│ │ ├── form.go
│ │ └── form_test.go
│ ├── context.go
│ ├── parsejs.go
│ ├── species.go
│ ├── species_test.go
│ ├── spider.go
│ ├── timer.go
│ └── timer_test.go
├── cmd/
│ ├── cmd_test.go
│ └── pholcus-cmd.go
├── common/
│ ├── beanstalkd/
│ │ ├── beanstalkd.go
│ │ └── beanstalkd_test.go
│ ├── bytes/
│ │ ├── bytes.go
│ │ └── bytes_test.go
│ ├── closer/
│ │ ├── closer.go
│ │ └── closer_test.go
│ ├── gc/
│ │ ├── gc.go
│ │ └── gc_test.go
│ ├── goquery/
│ │ ├── .gitattributes
│ │ ├── .gitignore
│ │ ├── .travis.yml
│ │ ├── LICENSE
│ │ ├── README.md
│ │ ├── array.go
│ │ ├── array_test.go
│ │ ├── bench/
│ │ │ ├── v0.1.0
│ │ │ ├── v0.1.1
│ │ │ ├── v0.2.0
│ │ │ ├── v0.2.1-go1.1rc1
│ │ │ ├── v0.3.0
│ │ │ ├── v0.3.2-go1.2
│ │ │ ├── v0.3.2-go1.2-take2
│ │ │ ├── v0.3.2-go1.2rc1
│ │ │ ├── v1.0.0-go1.7
│ │ │ ├── v1.0.1a-go1.7
│ │ │ ├── v1.0.1b-go1.7
│ │ │ └── v1.0.1c-go1.7
│ │ ├── bench_array_test.go
│ │ ├── bench_example_test.go
│ │ ├── bench_expand_test.go
│ │ ├── bench_filter_test.go
│ │ ├── bench_iteration_test.go
│ │ ├── bench_property_test.go
│ │ ├── bench_query_test.go
│ │ ├── bench_traversal_test.go
│ │ ├── doc/
│ │ │ └── tips.md
│ │ ├── doc.go
│ │ ├── example_test.go
│ │ ├── expand.go
│ │ ├── expand_test.go
│ │ ├── filter.go
│ │ ├── filter_test.go
│ │ ├── iteration.go
│ │ ├── iteration_test.go
│ │ ├── manipulation.go
│ │ ├── manipulation_test.go
│ │ ├── misc/
│ │ │ └── git/
│ │ │ └── pre-commit
│ │ ├── property.go
│ │ ├── property_test.go
│ │ ├── query.go
│ │ ├── query_test.go
│ │ ├── testdata/
│ │ │ ├── gotesting.html
│ │ │ ├── gowiki.html
│ │ │ ├── metalreview.html
│ │ │ ├── page.html
│ │ │ ├── page2.html
│ │ │ └── page3.html
│ │ ├── traversal.go
│ │ ├── traversal_test.go
│ │ ├── type.go
│ │ ├── type_test.go
│ │ ├── utilities.go
│ │ └── utilities_test.go
│ ├── kafka/
│ │ ├── kafka.go
│ │ └── kafka_test.go
│ ├── mahonia/
│ │ ├── 8bit.go
│ │ ├── ASCII.go
│ │ ├── README.md
│ │ ├── big5-data.go
│ │ ├── big5.go
│ │ ├── charset.go
│ │ ├── convert_string.go
│ │ ├── cp51932.go
│ │ ├── entity.go
│ │ ├── entity_data.go
│ │ ├── euc-jp.go
│ │ ├── euc-kr-data.go
│ │ ├── euc-kr.go
│ │ ├── fallback.go
│ │ ├── gb18030-data.go
│ │ ├── gb18030.go
│ │ ├── gbk-data.go
│ │ ├── gbk.go
│ │ ├── iso2022jp.go
│ │ ├── jis0201-data.go
│ │ ├── jis0208-data.go
│ │ ├── jis0212-data.go
│ │ ├── kuten.go
│ │ ├── mahonia_test.go
│ │ ├── mahoniconv/
│ │ │ └── mahoniconv.go
│ │ ├── mbcs.go
│ │ ├── ms-jis-data.go
│ │ ├── reader.go
│ │ ├── shiftjis-data.go
│ │ ├── shiftjis.go
│ │ ├── tcvn3.go
│ │ ├── translate.go
│ │ ├── utf16.go
│ │ ├── utf8.go
│ │ └── writer.go
│ ├── mgo/
│ │ ├── count.go
│ │ ├── find.go
│ │ ├── insert.go
│ │ ├── list.go
│ │ ├── mgo.go
│ │ ├── mgo_test.go
│ │ ├── operator.go
│ │ ├── remove.go
│ │ ├── update.go
│ │ ├── update_all.go
│ │ └── upsert.go
│ ├── mysql/
│ │ ├── mysql.go
│ │ └── mysql_test.go
│ ├── ping/
│ │ ├── ping.go
│ │ └── ping_test.go
│ ├── pinyin/
│ │ ├── example_test.go
│ │ ├── initials_sort.go
│ │ ├── phonetic_symbol.go
│ │ ├── pinyin.go
│ │ ├── pinyin_dict.go
│ │ └── pinyin_test.go
│ ├── pool/
│ │ ├── pool.go
│ │ └── pool_test.go
│ ├── queue/
│ │ ├── queue.go
│ │ └── queue_test.go
│ ├── session/
│ │ ├── README.md
│ │ ├── sess_cookie.go
│ │ ├── sess_cookie_test.go
│ │ ├── sess_file.go
│ │ ├── sess_file_test.go
│ │ ├── sess_mem.go
│ │ ├── sess_mem_test.go
│ │ ├── sess_test.go
│ │ ├── sess_utils.go
│ │ ├── session.go
│ │ └── session_manager_test.go
│ ├── simplejson/
│ │ ├── simplejson.go
│ │ └── simplejson_test.go
│ ├── util/
│ │ ├── util.go
│ │ └── util_test.go
│ ├── websocket/
│ │ ├── client.go
│ │ ├── client_test.go
│ │ ├── hybi.go
│ │ ├── server.go
│ │ ├── server_test.go
│ │ ├── websocket.go
│ │ └── websocket_test.go
│ └── xlsx/
│ ├── cell.go
│ ├── col.go
│ ├── date.go
│ ├── doc.go
│ ├── file.go
│ ├── hsl.go
│ ├── lib.go
│ ├── reftable.go
│ ├── row.go
│ ├── sheet.go
│ ├── style.go
│ ├── templates.go
│ ├── theme.go
│ ├── write.go
│ ├── xlsx_test.go
│ ├── xmlContentTypes.go
│ ├── xmlSharedStrings.go
│ ├── xmlStyle.go
│ ├── xmlTheme.go
│ ├── xmlWorkbook.go
│ └── xmlWorksheet.go
├── config/
│ ├── config.go
│ ├── config_test.go
│ └── setting.go
├── doc/
│ └── GUI编译命令.txt
├── doc.go
├── exec/
│ ├── exec.go
│ ├── exec_darwin.go
│ ├── exec_freebsd.go
│ ├── exec_linux.go
│ ├── exec_test.go
│ └── exec_windows.go
├── go.mod
├── go.sum
├── go.work
├── go.work.sum
├── gui/
│ ├── client.go
│ ├── guimain.manifest
│ ├── logview.go
│ ├── model/
│ │ └── guispider.go
│ ├── offline.go
│ ├── pholcus-gui.go
│ ├── rsrc.syso
│ ├── runmode.go
│ ├── server.go
│ └── var.go
├── logs/
│ ├── logs/
│ │ ├── conn.go
│ │ ├── conn_test.go
│ │ ├── console.go
│ │ ├── console_test.go
│ │ ├── file.go
│ │ ├── file_test.go
│ │ ├── log.go
│ │ ├── log_test.go
│ │ ├── smtp.go
│ │ └── smtp_test.go
│ ├── logs.go
│ └── logs_test.go
├── runtime/
│ ├── cache/
│ │ ├── cache.go
│ │ └── cache_test.go
│ └── status/
│ ├── status.go
│ └── status_test.go
├── sample/
│ ├── dyn_rules/
│ │ ├── baidu_search.pholcus.html
│ │ └── baidu_search.pholcus.xml
│ ├── main.go
│ └── static_rules/
│ ├── IJGUC/
│ │ └── IJGUC.go
│ ├── README.md
│ ├── alibaba/
│ │ └── alibaba.go
│ ├── area_codes/
│ │ └── area_codes.go
│ ├── baidunews/
│ │ └── baidunews.go
│ ├── baidusearch/
│ │ └── baidusearch.go
│ ├── car_home/
│ │ └── car_home.go
│ ├── chinanews/
│ │ ├── chinanews.go
│ │ └── readme.md
│ ├── fang_resell_list/
│ │ ├── fang_resell_list.go
│ │ └── readme.md
│ ├── filetest/
│ │ └── filetest.go
│ ├── ganji_gongsi/
│ │ └── ganji_gongsi.go
│ ├── googlesearch/
│ │ └── googlesearch.go
│ ├── hollandandbarrett/
│ │ └── hollandandbarrett.go
│ ├── jdsearch/
│ │ └── jdsearch.go
│ ├── jiban/
│ │ └── jiban.go
│ ├── jingdong/
│ │ ├── README.md
│ │ └── jdSpider.go
│ ├── kaola/
│ │ └── kaola.go
│ ├── lewa/
│ │ └── lewa.go
│ ├── miyabaobei/
│ │ └── miyabaobei.go
│ ├── people/
│ │ └── people.go
│ ├── pholcus_rules.go
│ ├── qq_avatar/
│ │ ├── README.md
│ │ └── avatar.go
│ ├── shunfenghaitao/
│ │ └── shunfenghaitao.go
│ ├── taobao/
│ │ └── taobao.go
│ ├── taobaosearch/
│ │ └── taobaosearch.go
│ ├── wangyi/
│ │ └── wangyi.go
│ ├── weibo_fans/
│ │ └── weibo_fans.go
│ ├── wukongwenda/
│ │ ├── README.md
│ │ └── wukongwenda.go
│ ├── zhihu_bianji/
│ │ ├── README.md
│ │ └── zhihu_bianji.go
│ ├── zhihu_daily/
│ │ ├── README.md
│ │ └── zhihu_daily.go
│ ├── zolpc/
│ │ └── zolpc.go
│ ├── zolphone/
│ │ └── zolphone.go
│ └── zolslab/
│ └── zolslab.go
└── web/
├── embed.go
├── embed_test.go
├── http_controller.go
├── http_controller_test.go
├── logsocket_controller.go
├── logsocket_controller_test.go
├── pholcus-web.go
├── router.go
├── router_test.go
├── views/
│ ├── bootstrap/
│ │ ├── css/
│ │ │ ├── bootstrap-theme.css
│ │ │ └── bootstrap.css
│ │ └── js/
│ │ ├── bootstrap.js
│ │ └── npm.js
│ ├── css/
│ │ ├── pholcus.css
│ │ └── split.css
│ ├── index.html
│ ├── js/
│ │ ├── app.js
│ │ ├── jquery.githubRepoWidget2.js
│ │ └── tpl.js
│ ├── layer/
│ │ ├── extend/
│ │ │ └── layer.ext.js
│ │ ├── layer.js
│ │ └── skin/
│ │ ├── layer.css
│ │ └── layer.ext.css
│ └── splitjs/
│ └── split.js
├── websocket_controller.go
└── websocket_controller_test.go
================================================
FILE CONTENTS
================================================
================================================
FILE: .gitattributes
================================================
*.* linguist-language=go
================================================
FILE: .gitignore
================================================
*.o
*.a
*.so
_obj
_test
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.exe~
*.test
*.prof
*.rar
*.zip
*.gz
*.psd
*.bmd
*.cfg
*.pptx
*.log
*.out
*.sublime-project
*.sublime-workspace
/openspec
.cursor
.DS_Store
sample/sample
sample/pholcus_pkg/cache
sample/pholcus_pkg/file_out
sample/pholcus_pkg/history
sample/pholcus_pkg/logs
sample/pholcus_pkg/text_out
pholcus_pkg/
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2015 HenryLee
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
<div align="center">
<img src="https://github.com/andeya/pholcus/raw/master/doc/icon.png" width="120" alt="Pholcus Logo"/>
<h1>Pholcus（幽灵蛛）</h1>
<p><strong>纯 Go 语言编写的分布式高并发爬虫框架</strong></p>
[](https://github.com/andeya/pholcus/releases)
[](https://github.com/andeya/pholcus/stargazers)
[](https://pkg.go.dev/github.com/andeya/pholcus)
[](https://goreportcard.com/report/andeya/pholcus)
[](https://github.com/andeya/pholcus/blob/master/LICENSE)
[](https://github.com/andeya/pholcus/issues?q=is%3Aopen+is%3Aissue)
[](https://github.com/andeya/pholcus/issues?q=is%3Aissue+is%3Aclosed)
<p>
<a href="#快速开始">快速开始</a> •
<a href="#核心特性">核心特性</a> •
<a href="#架构设计">架构设计</a> •
<a href="#操作界面">操作界面</a> •
<a href="#规则编写">规则编写</a> •
<a href="#常见问题">FAQ</a>
</p>
</div>
---
## 免责声明
> **本软件仅用于学术研究，使用者需遵守其所在地的相关法律法规，请勿用于非法用途！**
>
> 如在中国大陆频频爆出爬虫开发者涉诉与违规的 [新闻](https://github.com/HiddenStrawberry/Crawler_Illegal_Cases_In_China)。
>
> **郑重声明：因违法违规使用造成的一切后果，使用者自行承担！**
---
## 核心特性
<table>
<tr>
<td width="50%">
**运行模式**
- 单机模式 — 开箱即用
- 服务端模式 — 分发任务
- 客户端模式 — 接收并执行任务
</td>
<td width="50%">
**操作界面**
- Web UI — 跨平台，浏览器操作
- GUI — Windows 原生界面
- Cmd — 命令行批量调度
</td>
</tr>
<tr>
<td>
**数据输出**
- MySQL / MongoDB
- Kafka / Beanstalkd
- CSV / Excel
- 原文件下载
</td>
<td>
**爬虫规则**
- 静态规则（Go）— 高性能，深度定制
- 动态规则（JS/XML）— 热加载，无需编译
- 30+ 内置示例规则
</td>
</tr>
</table>
**更多亮点：**
- 三引擎下载器 [surfer](app/downloader/surfer)：Surf（高并发 HTTP）/ PhantomJS / **Chrome**（Chromium 无头浏览器，自动执行 JS）
- 智能 Cookie 管理：固定 UserAgent 自动保存 cookie，或随机 UserAgent 禁用 cookie
- 模拟登录、自定义 Header、POST 表单提交
- 代理 IP 池，可按频率自动更换
- 随机停歇机制，模拟人工行为
- 采集量与并发协程数可控
- 请求自动去重 + 失败请求自动重试
- 成功记录持久化，支持断点续爬
- 分布式通信全双工 Socket 框架
---
## 架构设计
<details>
<summary><b>模块结构</b></summary>
<br/>
<img src="https://github.com/andeya/pholcus/raw/master/doc/module.png" alt="模块结构" width="700"/>
</details>
<details>
<summary><b>项目架构</b></summary>
<br/>
<img src="https://github.com/andeya/pholcus/raw/master/doc/project.png" alt="项目架构" width="700"/>
</details>
<details>
<summary><b>分布式架构</b></summary>
<br/>
<img src="https://github.com/andeya/pholcus/raw/master/doc/distribute.png" alt="分布式架构" width="700"/>
</details>
### 目录结构
```
pholcus/
├── app/ 核心逻辑
│ ├── crawler/ 爬虫引擎 & 并发池
│ ├── downloader/ 下载器（surfer）
│ ├── pipeline/ 数据管道 & 多种输出后端
│ ├── scheduler/ 请求调度器
│ ├── spider/ 爬虫规则引擎
│ ├── distribute/ 分布式 Master/Slave 通信
│ └── aid/ 辅助模块（历史记录、代理 IP）
├── config/ 配置管理
├── exec/ 启动入口 & 平台适配
├── cmd/ 命令行模式
├── gui/ GUI 模式（Windows）
├── web/ Web UI 模式
├── common/ 公共工具库（DB 驱动、编码、队列等）
├── logs/ 日志模块
├── runtime/ 运行时缓存 & 状态
└── sample/ 示例程序 & 30+ 爬虫规则
```
---
## 快速开始
### 环境要求
- Go 1.18+（推荐 1.22+）
### 获取源码
```bash
git clone https://github.com/andeya/pholcus.git
cd pholcus
```
### 编写入口
创建 `main.go`（或参考 `sample/main.go`）：
```go
package main
import (
"github.com/andeya/pholcus/exec"
_ "github.com/andeya/pholcus/sample/static_rules" // 内置规则库
// _ "yourproject/rules" // 自定义规则库
)
func main() {
// 启动界面：web / gui / cmd
// 可通过 -a_ui 运行参数覆盖
exec.DefaultRun("web")
}
```
### 编译运行
```bash
# 编译（非 Windows 平台自动排除 GUI 包）
go build -o pholcus ./sample/
# 查看所有可选参数
./pholcus -h
```
Windows 下隐藏 cmd 窗口的编译方式：
```bash
go build -ldflags="-H=windowsgui -linkmode=internal" -o pholcus.exe ./sample/
```
### 命令行参数一览
```bash
./pholcus -h
```
---
## 操作界面
### Web UI
启动后访问 `http://localhost:2015`，在浏览器中即可完成蜘蛛选择、参数配置、任务启停等全部操作。
### GUI（仅 Windows）
原生桌面客户端，功能与 Web 版一致。
### Cmd 命令行
适用于服务器部署或 cron 定时任务场景。
```bash
pholcus -_ui=cmd -a_mode=0 -c_spider=3,8 -a_outtype=csv -a_thread=20 \
-a_batchcap=5000 -a_pause=300 -a_proxyminute=0 \
-a_keyins="<pholcus><golang>" -a_limit=10 -a_success=true -a_failure=true
```
---
## 规则编写
Pholcus 支持 **静态规则（Go）** 和 **动态规则（JS/XML）** 两种方式。
### 静态规则（Go）
随软件一同编译，性能最优，适合重量级采集项目。在 `sample/static_rules/` 下新建 Go 文件即可：
```go
package rules
import (
"net/http"
"github.com/andeya/pholcus/app/downloader/request"
"github.com/andeya/pholcus/app/spider"
)
func init() {
mySpider.Register()
}
var mySpider = &spider.Spider{
Name: "示例爬虫",
Description: "示例爬虫 [Auto Page] [http://example.com]",
EnableCookie: true,
RuleTree: &spider.RuleTree{
Root: func(ctx *spider.Context) {
ctx.AddQueue(&request.Request{
URL: "http://example.com",
Rule: "首页",
})
},
Trunk: map[string]*spider.Rule{
"首页": {
ParseFunc: func(ctx *spider.Context) {
ctx.Output(map[int]interface{}{
0: ctx.GetText(),
})
},
},
},
},
}
```
> 更多示例见 [`sample/static_rules/`](sample/static_rules/)，涵盖百度、京东、淘宝、知乎等 30+ 网站。
### 动态规则（JS/XML）
无需编译即可热加载，适合轻量级采集。将 `.pholcus.xml` 文件放入 `dyn_rules/` 目录：
```xml
<Spider>
<Name>百度搜索</Name>
<Description>百度搜索 [Auto Page] [http://www.baidu.com]</Description>
<Pausetime>300</Pausetime>
<EnableLimit>false</EnableLimit>
<EnableCookie>true</EnableCookie>
<EnableKeyin>true</EnableKeyin>
<NotDefaultField>false</NotDefaultField>
<Namespace><Script></Script></Namespace>
<SubNamespace><Script></Script></SubNamespace>
<Root>
<Script param="ctx">
ctx.JsAddQueue({
URL: "http://www.baidu.com/s?wd=" + ctx.GetKeyin(),
Rule: "搜索结果"
});
</Script>
</Root>
<Rule name="搜索结果">
<ParseFunc>
<Script param="ctx">
ctx.Output({
"标题": ctx.GetDom().Find("title").Text(),
"内容": ctx.GetText()
});
</Script>
</ParseFunc>
</Rule>
</Spider>
```
> 同时兼容 `.pholcus.html` 旧格式。`<Script>` 标签内自动包裹 CDATA，无需手动转义特殊字符。
---
## 下载器
Pholcus 内置三种下载引擎，通过 `DownloaderID` 切换：
| ID | 名称 | 说明 |
|----|------|------|
| `0` | **Surf** | 默认引擎。纯 Go HTTP 客户端，高并发，适合大多数静态页面采集 |
| `1` | **PhantomJS** | 基于 PhantomJS 的无头浏览器（已停止维护），可执行 JS，并发能力较低 |
| `2` | **Chrome** | 基于 Chromium（chromedp）的无头浏览器，可执行 JS、绕过安全验证，推荐用于反爬严格的站点 |
### 在静态规则（Go）中使用
```go
import "github.com/andeya/pholcus/app/downloader/request"
// 使用默认 Surf 引擎（可省略 DownloaderID）
ctx.AddQueue(&request.Request{
URL: "https://example.com",
Rule: "页面",
})
// 使用 Chrome 无头浏览器引擎
ctx.AddQueue(&request.Request{
URL: "https://www.baidu.com/s?wd=pholcus",
Rule: "搜索结果",
DownloaderID: request.ChromeID,
})
```
### 在动态规则（JS/XML）中使用
```xml
<Script param="ctx">
ctx.JsAddQueue({
URL: "https://www.baidu.com/s?wd=pholcus",
Rule: "搜索结果",
DownloaderID: 2
});
</Script>
```
### Chrome 引擎说明
Chrome 引擎依赖本机安装的 Chromium / Google Chrome 浏览器，通过 [chromedp](https://github.com/chromedp/chromedp) 驱动。
**适用场景：**
- 目标网站有 JS 渲染的内容（SPA / CSR 页面）
- 目标网站有安全验证（如百度安全验证）需要浏览器执行 JS 后自动跳转
- 需要模拟真实浏览器环境绕过反爬检测
**环境要求：**
- 本机需安装 Chrome / Chromium 浏览器
- macOS: `brew install --cask google-chrome` 或 `brew install chromium`
- Linux: `apt install chromium-browser` 或 `yum install chromium`
- Windows: 安装 Google Chrome 即可
**注意事项：**
- Chrome 引擎每次请求会启动独立的无头浏览器实例，资源消耗高于 Surf
- 建议仅在 Surf 引擎无法获取内容时使用 Chrome
- Chrome 引擎内置了反自动化检测（隐藏 `navigator.webdriver`、禁用自动化标志等）
---
## 配置说明
### 运行时目录
```
├── pholcus 可执行文件
├── dyn_rules/ 动态规则目录（可在 config.ini 中配置）
│ └── xxx.pholcus.xml 动态规则文件
└── pholcus_pkg/ 运行时文件目录
├── config.ini 配置文件
├── proxy.lib 代理 IP 列表
├── phantomjs PhantomJS 程序
├── text_out/ 文本输出目录
├── file_out/ 文件输出目录
├── logs/ 日志目录
├── history/ 历史记录目录
└── cache/ 临时缓存目录
```
### 代理 IP
在 `pholcus_pkg/proxy.lib` 文件中逐行写入代理地址：
```
http://183.141.168.95:3128
https://60.13.146.92:8088
http://59.59.4.22:8090
```
通过界面选择"代理 IP 更换频率"或命令行参数 `-a_proxyminute` 启用。
> **注意：** macOS 下使用代理 IP 功能需要 root 权限，否则无法通过 `ping` 检测可用代理。
---
## 内置爬虫规则
| 分类 | 规则名称 |
| -------- | --------------------------------------------------------- |
| 搜索引擎 | 百度搜索、百度新闻、谷歌搜索、京东搜索、淘宝搜索 |
| 电商平台 | 京东、淘宝、考拉海购、蜜芽宝贝、顺丰海淘、Holland&Barrett |
| 新闻资讯 | 中国新闻网、网易新闻、人民网 |
| 社交问答 | 知乎日报、知乎编辑推荐、悟空问答、微博粉丝 |
| 房产汽车 | 房天下二手房、汽车之家 |
| 数码科技 | ZOL 手机、ZOL 电脑、ZOL 平板、乐蛙 |
| 分类信息 | 赶集公司、全国区号 |
| 社交工具 | QQ 头像 |
| 学术期刊 | IJGUC |
| 其他 | 阿里巴巴、技版、文件下载测试 |
---
## 常见问题
<details>
<summary><b>请求队列中重复的 URL 会自动去重吗？</b></summary>
默认自动去重。如需允许重复请求，设置 `Request.Reloadable = true`。
</details>
<details>
<summary><b>框架能否判断页面内容是否更新？</b></summary>
框架不内置页面变更检测，但可在规则中自定义实现。
</details>
<details>
<summary><b>请求成功的判定标准是什么？</b></summary>
以服务器是否返回响应流为准，而非 HTTP 状态码。即 404 页面也算"请求成功"。
</details>
<details>
<summary><b>请求失败后如何重试？</b></summary>
每个 URL 尝试下载指定次数后，若仍失败则进入 defer 队列。当前任务正常结束后自动重试。再次失败则保存至失败历史记录。下次执行同一规则时，可选择继承历史失败记录进行自动重试。
</details>
---
## 参与贡献
欢迎提交 Issue 和 Pull Request！
1. Fork 本仓库
2. 创建特性分支：`git checkout -b feature/your-feature`
3. 提交更改：`git commit -m 'Add your feature'`
4. 推送分支：`git push origin feature/your-feature`
5. 提交 Pull Request
---
## 开源协议
本项目基于 [Apache License 2.0](LICENSE) 开源。
---
<div align="center">
<sub>Created by <a href="https://github.com/andeya">andeya</a> — 如果觉得有帮助，请给个 Star 支持！</sub>
</div>
================================================
FILE: app/aid/history/failure.go
================================================
package history
import (
"bytes"
"encoding/json"
"os"
"sync"
"github.com/andeya/gust/result"
"github.com/andeya/pholcus/app/downloader/request"
"github.com/andeya/pholcus/common/mgo"
"github.com/andeya/pholcus/common/mysql"
"github.com/andeya/pholcus/common/pool"
"github.com/andeya/pholcus/config"
)
// Failure tracks failed requests for retry.
type Failure struct {
tabName string
fileName string
list map[string]*request.Request
inheritable bool
sync.RWMutex
}
func (f *Failure) PullFailure() map[string]*request.Request {
list := f.list
f.list = make(map[string]*request.Request)
return list
}
// UpsertFailure updates or adds a failure record. Returns true if an insert occurred.
func (f *Failure) UpsertFailure(req *request.Request) bool {
f.RWMutex.Lock()
defer f.RWMutex.Unlock()
if f.list[req.Unique()] != nil {
return false
}
f.list[req.Unique()] = req
return true
}
// DeleteFailure removes a failure record.
func (f *Failure) DeleteFailure(req *request.Request) {
f.RWMutex.Lock()
delete(f.list, req.Unique())
f.RWMutex.Unlock()
}
// flush clears historical failure records first, then updates.
func (f *Failure) flush(provider string) (r result.Result[int]) {
defer r.Catch()
f.RWMutex.Lock()
defer f.RWMutex.Unlock()
fLen := len(f.list)
switch provider {
case "mgo":
result.RetVoid(mgo.Error()).Unwrap()
mgo.Call(func(src pool.Src) error {
c := src.(*mgo.MgoSrc).DB(config.Conf().DBName).C(f.tabName)
c.DropCollection()
if fLen == 0 {
return nil
}
var docs = []interface{}{}
for key, req := range f.list {
docs = append(docs, map[string]interface{}{"_id": key, "failure": req.Serialize().Unwrap()})
}
c.Insert(docs...)
return nil
}).Unwrap()
case "mysql":
_, err := mysql.DB()
result.RetVoid(err).Unwrap()
table, ok := getWriteMysqlTable(f.tabName)
if !ok {
table = mysql.New().Unwrap()
table.SetTableName(f.tabName).CustomPrimaryKey(`id VARCHAR(255) NOT NULL PRIMARY KEY`).AddColumn(`failure MEDIUMTEXT`)
setWriteMysqlTable(f.tabName, table)
table.Create().Unwrap()
} else {
table.Truncate().Unwrap()
}
for key, req := range f.list {
table.AutoInsert([]string{key, req.Serialize().Unwrap()})
table.FlushInsert().Unwrap()
}
default:
os.Remove(f.fileName)
if fLen == 0 {
return result.Ok(0)
}
file, err := os.OpenFile(f.fileName, os.O_CREATE|os.O_WRONLY, 0777)
result.RetVoid(err).Unwrap()
docs := make(map[string]string, len(f.list))
for key, req := range f.list {
docs[key] = req.Serialize().Unwrap()
}
b, _ := json.Marshal(docs)
b = bytes.Replace(b, []byte(`\u0026`), []byte(`&`), -1)
file.Write(b)
file.Close()
}
return result.Ok(fLen)
}
================================================
FILE: app/aid/history/failure_test.go
================================================
package history
import (
"net/http"
"os"
"path/filepath"
"testing"
"github.com/andeya/pholcus/app/downloader/request"
"github.com/andeya/pholcus/common/util"
"github.com/andeya/pholcus/config"
)
func newTestRequest(url string) *request.Request {
r := &request.Request{Spider: "s", URL: url, Rule: "r", Method: "GET", Header: make(http.Header)}
r.Prepare()
return r
}
func TestFailure_PullFailure(t *testing.T) {
req := newTestRequest("http://a.com")
f := &Failure{
tabName: "t",
fileName: "f",
list: map[string]*request.Request{req.Unique(): req},
}
got := f.PullFailure()
if len(got) != 1 {
t.Errorf("PullFailure len = %v, want 1", len(got))
}
if len(f.list) != 0 {
t.Error("PullFailure should clear list")
}
}
func TestFailure_UpsertFailure(t *testing.T) {
req := newTestRequest("http://a.com")
f := &Failure{
tabName: "t",
fileName: "f",
list: make(map[string]*request.Request),
}
tests := []struct {
req *request.Request
want bool
}{
{req, true},
{req, false},
}
for i, tt := range tests {
if got := f.UpsertFailure(tt.req); got != tt.want {
t.Errorf("UpsertFailure #%d = %v, want %v", i, got, tt.want)
}
}
}
func TestFailure_DeleteFailure(t *testing.T) {
req := newTestRequest("http://a.com")
f := &Failure{
tabName: "t",
fileName: "f",
list: map[string]*request.Request{req.Unique(): req},
}
f.DeleteFailure(req)
if len(f.list) != 0 {
t.Error("DeleteFailure should remove from list")
}
}
func TestFailure_Flush_File(t *testing.T) {
tmp := t.TempDir()
dir := filepath.Join(tmp, config.WorkRoot, config.HistoryTag)
if err := os.MkdirAll(dir, 0777); err != nil {
t.Fatalf("MkdirAll: %v", err)
}
orig, _ := os.Getwd()
os.Chdir(tmp)
defer os.Chdir(orig)
fileName := filepath.Join(dir, "history__n__test")
req := newTestRequest("http://b.com")
f := &Failure{
tabName: util.FileNameReplace("history__n__test"),
fileName: fileName,
list: map[string]*request.Request{req.Unique(): req},
}
r := f.flush("file")
if r.IsErr() {
t.Fatalf("flush: %v", r.UnwrapErr())
}
if r.Unwrap() != 1 {
t.Errorf("flush count = %v, want 1", r.Unwrap())
}
if _, err := os.Stat(fileName); err != nil {
t.Errorf("flush file: %v", err)
}
}
func TestFailure_Flush_FileEmpty(t *testing.T) {
tmp := t.TempDir()
dir := filepath.Join(tmp, config.WorkRoot, config.HistoryTag)
if err := os.MkdirAll(dir, 0777); err != nil {
t.Fatalf("MkdirAll: %v", err)
}
orig, _ := os.Getwd()
os.Chdir(tmp)
defer os.Chdir(orig)
fileName := filepath.Join(dir, "history__n__empty")
f := &Failure{
tabName: util.FileNameReplace("history__n__empty"),
fileName: fileName,
list: make(map[string]*request.Request),
}
r := f.flush("file")
if r.IsErr() {
t.Fatalf("flush empty: %v", r.UnwrapErr())
}
if r.Unwrap() != 0 {
t.Errorf("flush count = %v, want 0", r.Unwrap())
}
}
func TestFailure_Flush_FileOverwrite(t *testing.T) {
tmp := t.TempDir()
dir := filepath.Join(tmp, config.WorkRoot, config.HistoryTag)
if err := os.MkdirAll(dir, 0777); err != nil {
t.Fatalf("MkdirAll: %v", err)
}
orig, _ := os.Getwd()
os.Chdir(tmp)
defer os.Chdir(orig)
fileName := filepath.Join(dir, "history__n__overwrite")
if err := os.WriteFile(fileName, []byte("old"), 0644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
req := newTestRequest("http://c.com")
f := &Failure{
tabName: util.FileNameReplace("history__n__overwrite"),
fileName: fileName,
list: map[string]*request.Request{req.Unique(): req},
}
r := f.flush("file")
if r.IsErr() {
t.Fatalf("flush: %v", r.UnwrapErr())
}
data, _ := os.ReadFile(fileName)
if len(data) < 10 {
t.Errorf("flush should overwrite file, got %d bytes", len(data))
}
}
================================================
FILE: app/aid/history/history.go
================================================
// Package history provides persistence and inheritance of success and failure request records.
package history
import (
"encoding/json"
"io"
"os"
"sync"
"gopkg.in/mgo.v2/bson"
"github.com/andeya/gust/result"
"github.com/andeya/pholcus/app/downloader/request"
"github.com/andeya/pholcus/common/closer"
"github.com/andeya/pholcus/common/mgo"
"github.com/andeya/pholcus/common/mysql"
"github.com/andeya/pholcus/common/pool"
"github.com/andeya/pholcus/common/util"
"github.com/andeya/pholcus/config"
"github.com/andeya/pholcus/logs"
)
type (
HistoryStore interface {
ReadSuccess(provider string, inherit bool) result.VoidResult // Read success records
UpsertSuccess(string) bool // Upsert a success record
HasSuccess(string) bool // Check if a success record exists
DeleteSuccess(string) // Delete a success record
FlushSuccess(provider string) result.VoidResult // Flush success records to I/O without clearing cache
ReadFailure(provider string, inherit bool) result.VoidResult // Read failure records
PullFailure() map[string]*request.Request // Pull failure records and clear
UpsertFailure(*request.Request) bool // Upsert a failure record
DeleteFailure(*request.Request) // Delete a failure record
FlushFailure(provider string) result.VoidResult // Flush failure records to I/O without clearing cache
Empty() // Clear cache without output
}
// History stores success and failure records for crawl deduplication.
History struct {
*Success
*Failure
provider string
sync.RWMutex
}
)
const (
SuccessSuffix = config.HistoryTag + "__y"
FailureSuffix = config.HistoryTag + "__n"
SuccessFile = config.HistoryDir + "/" + SuccessSuffix
FailureFile = config.HistoryDir + "/" + FailureSuffix
)
// New creates a HistoryStore for the given spider name and optional subname.
func New(name string, subName string) HistoryStore {
successTabName := SuccessSuffix + "__" + name
successFileName := SuccessFile + "__" + name
failureTabName := FailureSuffix + "__" + name
failureFileName := FailureFile + "__" + name
if subName != "" {
successTabName += "__" + subName
successFileName += "__" + subName
failureTabName += "__" + subName
failureFileName += "__" + subName
}
return &History{
Success: &Success{
tabName: util.FileNameReplace(successTabName),
fileName: successFileName,
new: make(map[string]bool),
old: make(map[string]bool),
},
Failure: &Failure{
tabName: util.FileNameReplace(failureTabName),
fileName: failureFileName,
list: make(map[string]*request.Request),
},
}
}
// ReadSuccess reads success records from the given provider.
func (h *History) ReadSuccess(provider string, inherit bool) result.VoidResult {
h.RWMutex.Lock()
h.provider = provider
h.RWMutex.Unlock()
if !inherit {
// Not inheriting history
h.Success.old = make(map[string]bool)
h.Success.new = make(map[string]bool)
h.Success.inheritable = false
return result.OkVoid()
} else if h.Success.inheritable {
// Both current and previous runs inherit history
return result.OkVoid()
} else {
// Previous run did not inherit, but current run does
h.Success.old = make(map[string]bool)
h.Success.new = make(map[string]bool)
h.Success.inheritable = true
}
switch provider {
case "mgo":
var docs = map[string]interface{}{}
r := mgo.Mgo(&docs, "find", map[string]interface{}{
"Database": config.Conf().DBName,
"Collection": h.Success.tabName,
})
if r.IsErr() {
logs.Log().Error(" * Fail [read success record][mgo]: %v\n", r.UnwrapErr())
return result.OkVoid()
}
for _, v := range docs["Docs"].([]interface{}) {
h.Success.old[v.(bson.M)["_id"].(string)] = true
}
case "mysql":
_, err := mysql.DB()
if err != nil {
logs.Log().Error(" * Fail [read success record][mysql]: %v\n", err)
return result.OkVoid()
}
table, ok := getReadMysqlTable(h.Success.tabName)
if !ok {
table = mysql.New().Unwrap().SetTableName(h.Success.tabName)
setReadMysqlTable(h.Success.tabName, table)
}
r := table.SelectAll()
if r.IsErr() {
return result.OkVoid()
}
rows := r.Unwrap()
for rows.Next() {
var id string
err = rows.Scan(&id)
h.Success.old[id] = true
}
default:
f, err := os.Open(h.Success.fileName)
if err != nil {
return result.OkVoid()
}
defer closer.LogClose(f, logs.Log().Error)
b, _ := io.ReadAll(f)
if len(b) == 0 {
return result.OkVoid()
}
b[0] = '{'
json.Unmarshal(append(b, '}'), &h.Success.old)
}
logs.Log().Informational(" * [read success record]: %v\n", len(h.Success.old))
return result.OkVoid()
}
// ReadFailure reads failure records from the given provider.
func (h *History) ReadFailure(provider string, inherit bool) result.VoidResult {
h.RWMutex.Lock()
h.provider = provider
h.RWMutex.Unlock()
if !inherit {
// Not inheriting history
h.Failure.list = make(map[string]*request.Request)
h.Failure.inheritable = false
return result.OkVoid()
} else if h.Failure.inheritable {
// Both current and previous runs inherit history
return result.OkVoid()
} else {
// Previous run did not inherit, but current run does
h.Failure.list = make(map[string]*request.Request)
h.Failure.inheritable = true
}
var fLen int
switch provider {
case "mgo":
if mgo.Error() != nil {
logs.Log().Error(" * Fail [read failure record][mgo]: %v\n", mgo.Error())
return result.OkVoid()
}
var docs = []interface{}{}
mgo.Call(func(src pool.Src) error {
c := src.(*mgo.MgoSrc).DB(config.Conf().DBName).C(h.Failure.tabName)
return c.Find(nil).All(&docs)
}).Unwrap()
fLen = len(docs)
for _, v := range docs {
key := v.(bson.M)["_id"].(string)
failure := v.(bson.M)["failure"].(string)
reqResult := request.UnSerialize(failure)
if reqResult.IsErr() {
continue
}
h.Failure.list[key] = reqResult.Unwrap()
}
case "mysql":
_, err := mysql.DB()
if err != nil {
logs.Log().Error(" * Fail [read failure record][mysql]: %v\n", err)
return result.OkVoid()
}
table, ok := getReadMysqlTable(h.Failure.tabName)
if !ok {
table = mysql.New().Unwrap().SetTableName(h.Failure.tabName)
setReadMysqlTable(h.Failure.tabName, table)
}
r := table.SelectAll()
if r.IsErr() {
return result.OkVoid()
}
rows := r.Unwrap()
for rows.Next() {
var key, failure string
err = rows.Scan(&key, &failure)
reqResult := request.UnSerialize(failure)
if reqResult.IsErr() {
continue
}
h.Failure.list[key] = reqResult.Unwrap()
fLen++
}
default:
f, err := os.Open(h.Failure.fileName)
if err != nil {
return result.OkVoid()
}
defer closer.LogClose(f, logs.Log().Error)
b, _ := io.ReadAll(f)
if len(b) == 0 {
return result.OkVoid()
}
docs := map[string]string{}
json.Unmarshal(b, &docs)
fLen = len(docs)
for key, s := range docs {
reqResult := request.UnSerialize(s)
if reqResult.IsErr() {
continue
}
h.Failure.list[key] = reqResult.Unwrap()
}
}
logs.Log().Informational(" * [read failure record]: %v\n", fLen)
return result.OkVoid()
}
// Empty clears the cache without output.
func (h *History) Empty() {
h.RWMutex.Lock()
h.Success.new = make(map[string]bool)
h.Success.old = make(map[string]bool)
h.Failure.list = make(map[string]*request.Request)
h.RWMutex.Unlock()
}
// FlushSuccess flushes success records to I/O without clearing cache.
func (h *History) FlushSuccess(provider string) (r result.VoidResult) {
defer r.Catch()
h.RWMutex.Lock()
h.provider = provider
h.RWMutex.Unlock()
sucLen := h.Success.flush(provider).Unwrap()
if sucLen <= 0 {
return result.OkVoid()
}
logs.Log().Informational(" * [add success record]: %v\n", sucLen)
return result.OkVoid()
}
// FlushFailure flushes failure records to I/O without clearing cache.
func (h *History) FlushFailure(provider string) (r result.VoidResult) {
defer r.Catch()
h.RWMutex.Lock()
h.provider = provider
h.RWMutex.Unlock()
failLen := h.Failure.flush(provider).Unwrap()
if failLen <= 0 {
return result.OkVoid()
}
logs.Log().Informational(" * [add failure record]: %v\n", failLen)
return result.OkVoid()
}
var (
readMysqlTable = map[string]*mysql.Table{}
readMysqlTableLock sync.RWMutex
)
func getReadMysqlTable(name string) (*mysql.Table, bool) {
readMysqlTableLock.RLock()
tab, ok := readMysqlTable[name]
readMysqlTableLock.RUnlock()
if ok {
return tab.Clone(), true
}
return nil, false
}
func setReadMysqlTable(name string, tab *mysql.Table) {
readMysqlTableLock.Lock()
readMysqlTable[name] = tab
readMysqlTableLock.Unlock()
}
var (
writeMysqlTable = map[string]*mysql.Table{}
writeMysqlTableLock sync.RWMutex
)
func getWriteMysqlTable(name string) (*mysql.Table, bool) {
writeMysqlTableLock.RLock()
tab, ok := writeMysqlTable[name]
writeMysqlTableLock.RUnlock()
if ok {
return tab.Clone(), true
}
return nil, false
}
func setWriteMysqlTable(name string, tab *mysql.Table) {
writeMysqlTableLock.Lock()
writeMysqlTable[name] = tab
writeMysqlTableLock.Unlock()
}
================================================
FILE: app/aid/history/history_test.go
================================================
package history
import (
"database/sql"
"encoding/json"
"net/http"
"os"
"path/filepath"
"testing"
sqlmock "github.com/DATA-DOG/go-sqlmock"
"github.com/andeya/pholcus/app/downloader/request"
"github.com/andeya/pholcus/common/mysql"
"github.com/andeya/pholcus/config"
)
func setupHistoryDir(t *testing.T) (cleanup func()) {
tmp := t.TempDir()
historyDir := filepath.Join(tmp, config.WorkRoot, config.HistoryTag)
if err := os.MkdirAll(historyDir, 0777); err != nil {
t.Fatalf("MkdirAll: %v", err)
}
orig, _ := os.Getwd()
if err := os.Chdir(tmp); err != nil {
t.Fatalf("Chdir: %v", err)
}
return func() { os.Chdir(orig) }
}
func TestNew(t *testing.T) {
tests := []struct {
name string
subName string
}{
{"spider1", ""},
{"spider2", "sub"},
}
for _, tt := range tests {
t.Run(tt.name+"_"+tt.subName, func(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New(tt.name, tt.subName)
if h == nil {
t.Fatal("New returned nil")
}
if got := h.UpsertSuccess("id1"); !got {
t.Error("UpsertSuccess want true")
}
if got := h.UpsertSuccess("id1"); got {
t.Error("UpsertSuccess duplicate want false")
}
})
}
}
func TestHistory_ReadSuccess_File(t *testing.T) {
tests := []struct {
name string
inherit bool
fileData string
checkOld bool
}{
{"no inherit", false, "", false},
{"inherit no file", true, "", false},
{"inherit with data", true, `,"id1":true,"id2":true`, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New("test", "").(*History)
if tt.fileData != "" {
if err := os.WriteFile(h.Success.fileName, []byte(tt.fileData), 0644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
}
r := h.ReadSuccess("file", tt.inherit)
if r.IsErr() {
t.Errorf("ReadSuccess: %v", r.UnwrapErr())
}
if tt.checkOld {
if len(h.Success.old) != 2 || !h.Success.HasSuccess("id1") || !h.Success.HasSuccess("id2") {
t.Errorf("expected ids in old, got len=%d old=%v", len(h.Success.old), h.Success.old)
}
}
})
}
}
func TestHistory_ReadSuccess_EmptyFile(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New("test", "").(*History)
if err := os.WriteFile(h.Success.fileName, []byte{}, 0644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
r := h.ReadSuccess("file", true)
if r.IsErr() {
t.Errorf("ReadSuccess: %v", r.UnwrapErr())
}
}
func TestHistory_ReadSuccess_InheritPaths(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New("test", "").(*History)
h.Success.inheritable = true
r := h.ReadSuccess("file", true)
if r.IsErr() {
t.Errorf("ReadSuccess inheritable: %v", r.UnwrapErr())
}
h.Success.inheritable = false
h.Success.old["x"] = true
r = h.ReadSuccess("file", true)
if r.IsErr() {
t.Errorf("ReadSuccess: %v", r.UnwrapErr())
}
if len(h.Success.old) != 0 {
t.Error("expected old cleared when switching to inherit")
}
}
func TestHistory_ReadFailure_File(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New("test", "").(*History)
req := &request.Request{Spider: "s", URL: "http://a.com", Rule: "r", Method: "GET", Header: make(http.Header)}
req.Prepare()
ser := req.Serialize().Unwrap()
fileData, _ := json.Marshal(map[string]string{req.Unique(): ser})
tests := []struct {
name string
inherit bool
fileData []byte
}{
{"no inherit", false, nil},
{"inherit no file", true, nil},
{"inherit with data", true, fileData},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.fileData != nil {
if err := os.WriteFile(h.Failure.fileName, tt.fileData, 0644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
}
r := h.ReadFailure("file", tt.inherit)
if r.IsErr() {
t.Errorf("ReadFailure: %v", r.UnwrapErr())
}
})
}
}
func TestHistory_ReadFailure_EmptyFile(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New("test", "").(*History)
if err := os.WriteFile(h.Failure.fileName, []byte{}, 0644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
r := h.ReadFailure("file", true)
if r.IsErr() {
t.Errorf("ReadFailure: %v", r.UnwrapErr())
}
}
func TestHistory_Empty(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New("test", "").(*History)
h.UpsertSuccess("id1")
req := &request.Request{Spider: "s", URL: "http://a.com", Rule: "r", Method: "GET", Header: make(http.Header)}
req.Prepare()
h.UpsertFailure(req)
h.Empty()
if h.HasSuccess("id1") {
t.Error("Empty should clear success")
}
pulled := h.PullFailure()
if len(pulled) != 0 {
t.Error("Empty should clear failure")
}
}
func TestHistory_FlushSuccess_File(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New("test", "").(*History)
h.UpsertSuccess("id1")
h.UpsertSuccess("id2")
r := h.FlushSuccess("file")
if r.IsErr() {
t.Errorf("FlushSuccess: %v", r.UnwrapErr())
}
if _, err := os.Stat(h.Success.fileName); err != nil {
t.Errorf("FlushSuccess file: %v", err)
}
}
func TestHistory_FlushFailure_File(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New("test", "").(*History)
req := &request.Request{Spider: "s", URL: "http://a.com", Rule: "r", Method: "GET", Header: make(http.Header)}
req.Prepare()
h.UpsertFailure(req)
r := h.FlushFailure("file")
if r.IsErr() {
t.Errorf("FlushFailure: %v", r.UnwrapErr())
}
if _, err := os.Stat(h.Failure.fileName); err != nil {
t.Errorf("FlushFailure file: %v", err)
}
}
func TestHistory_FlushSuccess_Empty(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New("test", "").(*History)
r := h.FlushSuccess("file")
if r.IsErr() {
t.Errorf("FlushSuccess empty: %v", r.UnwrapErr())
}
}
func TestHistory_FlushFailure_Empty(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New("test", "").(*History)
r := h.FlushFailure("file")
if r.IsErr() {
t.Errorf("FlushFailure empty: %v", r.UnwrapErr())
}
}
func TestHistory_ReadSuccess_FileNotFound(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New("test", "").(*History)
r := h.ReadSuccess("file", true)
if r.IsErr() {
t.Errorf("ReadSuccess file not found: %v", r.UnwrapErr())
}
}
func TestHistory_ReadSuccess_ReadFailure_MysqlMock(t *testing.T) {
sqlDB, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New: %v", err)
}
defer sqlDB.Close()
cleanup := mysql.SetDBForTest(sqlDB)
defer cleanup()
cleanupDir := setupHistoryDir(t)
defer cleanupDir()
_ = config.Conf()
h := New("test", "").(*History)
rows := sqlmock.NewRows([]string{"id"}).AddRow("id1").AddRow("id2")
mock.ExpectQuery("SELECT \\* FROM").WillReturnRows(rows)
r := h.ReadSuccess("mysql", true)
if r.IsErr() {
t.Errorf("ReadSuccess mysql: %v", r.UnwrapErr())
}
if len(h.Success.old) != 2 {
t.Errorf("ReadSuccess mysql: want 2 old, got %d", len(h.Success.old))
}
req := &request.Request{Spider: "s", URL: "http://a.com", Rule: "r", Method: "GET", Header: make(http.Header)}
req.Prepare()
ser := req.Serialize().Unwrap()
rows2 := sqlmock.NewRows([]string{"id", "failure"}).AddRow(req.Unique(), ser)
mock.ExpectQuery("SELECT \\* FROM").WillReturnRows(rows2)
r = h.ReadFailure("mysql", true)
if r.IsErr() {
t.Errorf("ReadFailure mysql: %v", r.UnwrapErr())
}
if len(h.Failure.list) != 1 {
t.Errorf("ReadFailure mysql: want 1, got %d", len(h.Failure.list))
}
}
func TestHistory_ReadSuccess_MysqlDBError(t *testing.T) {
cleanup := mysql.SetDBForTest(nil)
defer cleanup()
cleanupDir := setupHistoryDir(t)
defer cleanupDir()
_ = config.Conf()
h := New("test", "").(*History)
r := h.ReadSuccess("mysql", true)
if r.IsErr() {
t.Errorf("ReadSuccess mysql no db: %v", r.UnwrapErr())
}
}
func TestHistory_ReadFailure_MysqlDBError(t *testing.T) {
cleanup := mysql.SetDBForTest(nil)
defer cleanup()
cleanupDir := setupHistoryDir(t)
defer cleanupDir()
_ = config.Conf()
h := New("test", "").(*History)
r := h.ReadFailure("mysql", true)
if r.IsErr() {
t.Errorf("ReadFailure mysql no db: %v", r.UnwrapErr())
}
}
func TestHistory_ReadSuccess_MysqlSelectError(t *testing.T) {
sqlDB, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New: %v", err)
}
defer sqlDB.Close()
cleanup := mysql.SetDBForTest(sqlDB)
defer cleanup()
cleanupDir := setupHistoryDir(t)
defer cleanupDir()
_ = config.Conf()
h := New("test", "").(*History)
mock.ExpectQuery("SELECT \\* FROM").WillReturnError(sql.ErrConnDone)
r := h.ReadSuccess("mysql", true)
if r.IsErr() {
t.Errorf("ReadSuccess mysql select err: %v", r.UnwrapErr())
}
}
func TestHistory_FlushSuccess_FlushFailure_MysqlMock(t *testing.T) {
sqlDB, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New: %v", err)
}
defer sqlDB.Close()
cleanup := mysql.SetDBForTest(sqlDB)
defer cleanup()
cleanupDir := setupHistoryDir(t)
defer cleanupDir()
_ = config.Conf()
h := New("test", "").(*History)
h.UpsertSuccess("id1")
h.UpsertSuccess("id2")
mock.ExpectExec("CREATE TABLE IF NOT EXISTS").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("INSERT INTO").WithArgs("id1", "id2").WillReturnResult(sqlmock.NewResult(2, 2))
r := h.FlushSuccess("mysql")
if r.IsErr() {
t.Errorf("FlushSuccess mysql: %v", r.UnwrapErr())
}
req := &request.Request{Spider: "s", URL: "http://a.com", Rule: "r", Method: "GET", Header: make(http.Header)}
req.Prepare()
h.UpsertFailure(req)
mock.ExpectExec("CREATE TABLE IF NOT EXISTS").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("INSERT INTO").WillReturnResult(sqlmock.NewResult(1, 1))
r = h.FlushFailure("mysql")
if r.IsErr() {
t.Errorf("FlushFailure mysql: %v", r.UnwrapErr())
}
}
func TestHistory_ReadFailure_InvalidData(t *testing.T) {
cleanup := setupHistoryDir(t)
defer cleanup()
_ = config.Conf()
h := New("test", "").(*History)
req := &request.Request{Spider: "s", URL: "http://a.com", Rule: "r", Method: "GET", Header: make(http.Header)}
req.Prepare()
ser := req.Serialize().Unwrap()
fileData := map[string]string{req.Unique(): ser, "badkey": "{invalid}"}
data, _ := json.Marshal(fileData)
if err := os.WriteFile(h.Failure.fileName, data, 0644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
r := h.ReadFailure("file", true)
if r.IsErr() {
t.Errorf("ReadFailure: %v", r.UnwrapErr())
}
if len(h.Failure.list) != 1 {
t.Errorf("expected 1 valid record, got %d", len(h.Failure.list))
}
}
================================================
FILE: app/aid/history/success.go
================================================
package history
import (
"encoding/json"
"fmt"
"os"
"sync"
"github.com/andeya/gust/result"
"github.com/andeya/pholcus/common/mgo"
"github.com/andeya/pholcus/common/mysql"
"github.com/andeya/pholcus/config"
)
// Success tracks successfully crawled request IDs for deduplication.
type Success struct {
tabName string
fileName string
new map[string]bool
old map[string]bool
inheritable bool
sync.RWMutex
}
// UpsertSuccess updates or adds a success record. Returns true if an insert occurred.
func (s *Success) UpsertSuccess(reqUnique string) bool {
s.RWMutex.Lock()
defer s.RWMutex.Unlock()
if s.old[reqUnique] {
return false
}
if s.new[reqUnique] {
return false
}
s.new[reqUnique] = true
return true
}
func (s *Success) HasSuccess(reqUnique string) bool {
s.RWMutex.Lock()
has := s.old[reqUnique] || s.new[reqUnique]
s.RWMutex.Unlock()
return has
}
// DeleteSuccess removes a success record.
func (s *Success) DeleteSuccess(reqUnique string) {
s.RWMutex.Lock()
delete(s.new, reqUnique)
s.RWMutex.Unlock()
}
func (s *Success) flush(provider string) result.Result[int] {
s.RWMutex.Lock()
defer s.RWMutex.Unlock()
sLen := len(s.new)
if sLen == 0 {
return result.Ok(0)
}
switch provider {
case "mgo":
if mgo.Error() != nil {
return result.TryErr[int](fmt.Errorf(" * Fail [add success record][mgo]: %v [ERROR] %v\n", sLen, mgo.Error()))
}
var docs = make([]map[string]interface{}, sLen)
var i int
for key := range s.new {
docs[i] = map[string]interface{}{"_id": key}
s.old[key] = true
i++
}
r := mgo.Mgo(nil, "insert", map[string]interface{}{
"Database": config.Conf().DBName,
"Collection": s.tabName,
"Docs": docs,
})
if r.IsErr() {
return result.TryErr[int](fmt.Errorf(" * Fail [add success record][mgo]: %v [ERROR] %v\n", sLen, r.UnwrapErr()))
}
case "mysql":
_, err := mysql.DB()
if err != nil {
return result.TryErr[int](fmt.Errorf(" * Fail [add success record][mysql]: %v [ERROR] %v\n", sLen, err))
}
table, ok := getWriteMysqlTable(s.tabName)
if !ok {
table = mysql.New().Unwrap()
table.SetTableName(s.tabName).CustomPrimaryKey(`id VARCHAR(255) NOT NULL PRIMARY KEY`)
if r := table.Create(); r.IsErr() {
return result.TryErr[int](fmt.Errorf(" * Fail [add success record][mysql]: %v [ERROR] %v\n", sLen, r.UnwrapErr()))
}
setWriteMysqlTable(s.tabName, table)
}
for key := range s.new {
table.AutoInsert([]string{key})
s.old[key] = true
}
if r := table.FlushInsert(); r.IsErr() {
return result.TryErr[int](fmt.Errorf(" * Fail [add success record][mysql]: %v [ERROR] %v\n", sLen, r.UnwrapErr()))
}
default:
f, _ := os.OpenFile(s.fileName, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0777)
b, _ := json.Marshal(s.new)
b[0] = ','
f.Write(b[:len(b)-1])
f.Close()
for key := range s.new {
s.old[key] = true
}
}
s.new = make(map[string]bool)
return result.Ok(sLen)
}
================================================
FILE: app/aid/history/success_test.go
================================================
package history
import (
"encoding/json"
"os"
"path/filepath"
"testing"
"github.com/andeya/pholcus/common/util"
"github.com/andeya/pholcus/config"
)
func TestSuccess_UpsertSuccess(t *testing.T) {
s := &Success{
tabName: "t",
fileName: "f",
new: make(map[string]bool),
old: make(map[string]bool),
}
tests := []struct {
unique string
want bool
}{
{"id1", true},
{"id1", false},
{"id2", true},
{"id2", false},
}
for _, tt := range tests {
if got := s.UpsertSuccess(tt.unique); got != tt.want {
t.Errorf("UpsertSuccess(%q) = %v, want %v", tt.unique, got, tt.want)
}
}
}
func TestSuccess_UpsertSuccess_OldExists(t *testing.T) {
s := &Success{
tabName: "t",
fileName: "f",
new: make(map[string]bool),
old: map[string]bool{"id1": true},
}
if got := s.UpsertSuccess("id1"); got {
t.Error("UpsertSuccess when old exists want false")
}
}
func TestSuccess_HasSuccess(t *testing.T) {
s := &Success{
tabName: "t",
fileName: "f",
new: map[string]bool{"n1": true},
old: map[string]bool{"o1": true},
}
tests := []struct {
unique string
want bool
}{
{"n1", true},
{"o1", true},
{"x", false},
}
for _, tt := range tests {
if got := s.HasSuccess(tt.unique); got != tt.want {
t.Errorf("HasSuccess(%q) = %v, want %v", tt.unique, got, tt.want)
}
}
}
func TestSuccess_DeleteSuccess(t *testing.T) {
s := &Success{
tabName: "t",
fileName: "f",
new: map[string]bool{"id1": true},
old: make(map[string]bool),
}
s.DeleteSuccess("id1")
if s.HasSuccess("id1") {
t.Error("DeleteSuccess should remove from new")
}
}
func TestSuccess_Flush_File(t *testing.T) {
tmp := t.TempDir()
dir := filepath.Join(tmp, config.WorkRoot, config.HistoryTag)
if err := os.MkdirAll(dir, 0777); err != nil {
t.Fatalf("MkdirAll: %v", err)
}
orig, _ := os.Getwd()
os.Chdir(tmp)
defer os.Chdir(orig)
fileName := filepath.Join(dir, "history__y__test")
s := &Success{
tabName: util.FileNameReplace("history__y__test"),
fileName: fileName,
new: map[string]bool{"a": true, "b": true},
old: make(map[string]bool),
}
r := s.flush("file")
if r.IsErr() {
t.Fatalf("flush: %v", r.UnwrapErr())
}
if r.Unwrap() != 2 {
t.Errorf("flush count = %v, want 2", r.Unwrap())
}
if _, err := os.Stat(fileName); err != nil {
t.Errorf("flush file: %v", err)
}
}
func TestSuccess_Flush_Empty(t *testing.T) {
s := &Success{
tabName: "t",
fileName: "/nonexistent",
new: make(map[string]bool),
old: make(map[string]bool),
}
r := s.flush("file")
if r.IsErr() {
t.Fatalf("flush empty: %v", r.UnwrapErr())
}
if r.Unwrap() != 0 {
t.Errorf("flush count = %v, want 0", r.Unwrap())
}
}
func TestSuccess_Flush_FileAppend(t *testing.T) {
tmp := t.TempDir()
dir := filepath.Join(tmp, config.WorkRoot, config.HistoryTag)
if err := os.MkdirAll(dir, 0777); err != nil {
t.Fatalf("MkdirAll: %v", err)
}
orig, _ := os.Getwd()
os.Chdir(tmp)
defer os.Chdir(orig)
fileName := filepath.Join(dir, "history__y__test")
s := &Success{
tabName: util.FileNameReplace("history__y__test"),
fileName: fileName,
new: map[string]bool{"c": true},
old: make(map[string]bool),
}
r := s.flush("file")
if r.IsErr() {
t.Fatalf("flush: %v", r.UnwrapErr())
}
data, _ := os.ReadFile(fileName)
var m map[string]bool
if err := json.Unmarshal(append(append([]byte{'{'}, data[1:]...), '}'), &m); err != nil {
t.Fatalf("unmarshal file: %v, content: %s", err, data)
}
if !m["c"] {
t.Errorf("expected c in file, got %v", m)
}
}
================================================
FILE: app/aid/proxy/host.go
================================================
package proxy
import (
"sync"
"time"
)
// ProxyForHost manages proxy IPs for a host, sorted by response time.
type ProxyForHost struct {
curIndex int // Index of current proxy IP
proxys []string
timedelay []time.Duration
isEcho bool // Whether to print proxy switch info
sync.Mutex
}
// Len implements sort.Interface.
func (ph *ProxyForHost) Len() int {
return len(ph.proxys)
}
func (ph *ProxyForHost) Less(i, j int) bool {
return ph.timedelay[i] < ph.timedelay[j]
}
func (ph *ProxyForHost) Swap(i, j int) {
ph.proxys[i], ph.proxys[j] = ph.proxys[j], ph.proxys[i]
ph.timedelay[i], ph.timedelay[j] = ph.timedelay[j], ph.timedelay[i]
}
================================================
FILE: app/aid/proxy/host_test.go
================================================
package proxy
import (
"testing"
"time"
)
func TestProxyForHost_Len(t *testing.T) {
tests := []struct {
proxys []string
want int
}{
{nil, 0},
{[]string{}, 0},
{[]string{"a"}, 1},
{[]string{"a", "b", "c"}, 3},
}
for _, tt := range tests {
ph := &ProxyForHost{proxys: tt.proxys}
if got := ph.Len(); got != tt.want {
t.Errorf("Len() = %v, want %v", got, tt.want)
}
}
}
func TestProxyForHost_Less(t *testing.T) {
ph := &ProxyForHost{
proxys: []string{"a", "b", "c"},
timedelay: []time.Duration{10 * time.Millisecond, 5 * time.Millisecond, 20 * time.Millisecond},
}
tests := []struct {
i, j int
want bool
}{
{0, 1, false},
{1, 0, true},
{1, 2, true},
{2, 1, false},
}
for _, tt := range tests {
if got := ph.Less(tt.i, tt.j); got != tt.want {
t.Errorf("Less(%d,%d) = %v, want %v", tt.i, tt.j, got, tt.want)
}
}
}
func TestProxyForHost_Swap(t *testing.T) {
ph := &ProxyForHost{
proxys: []string{"a", "b"},
timedelay: []time.Duration{10 * time.Millisecond, 5 * time.Millisecond},
}
ph.Swap(0, 1)
if ph.proxys[0] != "b" || ph.proxys[1] != "a" {
t.Errorf("Swap proxys = %v", ph.proxys)
}
if ph.timedelay[0] != 5*time.Millisecond || ph.timedelay[1] != 10*time.Millisecond {
t.Errorf("Swap timedelay = %v", ph.timedelay)
}
}
================================================
FILE: app/aid/proxy/proxy.go
================================================
// Package proxy provides proxy IP pool management and online filtering.
package proxy
import (
"io"
"log"
"net/http"
"net/url"
"os"
"regexp"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/andeya/gust/option"
"github.com/andeya/gust/result"
"github.com/andeya/pholcus/app/downloader/request"
"github.com/andeya/pholcus/app/downloader/surfer"
"github.com/andeya/pholcus/common/ping"
"github.com/andeya/pholcus/config"
"github.com/andeya/pholcus/logs"
)
// Proxy manages a pool of proxy IPs with online filtering and per-host sorting.
type Proxy struct {
ipRegexp *regexp.Regexp
proxyIPTypeRegexp *regexp.Regexp
proxyUrlTypeRegexp *regexp.Regexp
allIps map[string]string
all map[string]bool
online int32
usable map[string]*ProxyForHost
ticker *time.Ticker
tickMinute int64
threadPool chan bool
surf surfer.Surfer
sync.Mutex
}
const (
CONN_TIMEOUT = 4 //4s
DAIL_TIMEOUT = 4 //4s
TRY_TIMES = 3
// Max concurrency for IP speed testing
MAX_THREAD_NUM = 1000
)
// New creates and starts a Proxy that loads and filters proxy IPs from config.
func New() *Proxy {
p := &Proxy{
ipRegexp: regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+`),
proxyIPTypeRegexp: regexp.MustCompile(`https?://([\w]*:[\w]*@)?[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+`),
proxyUrlTypeRegexp: regexp.MustCompile(`((https?|ftp):\/\/)?(([^:\n\r]+):([^@\n\r]+)@)?((www\.)?([^/\n\r:]+)):?([0-9]{1,5})?\/?([^?\n\r]+)?\??([^#\n\r]*)?#?([^\n\r]*)`),
allIps: map[string]string{},
all: map[string]bool{},
usable: make(map[string]*ProxyForHost),
threadPool: make(chan bool, MAX_THREAD_NUM),
surf: surfer.New(),
}
go p.Update()
return p
}
// Count returns the number of online proxy IPs.
func (p *Proxy) Count() int32 {
return p.online
}
// SetSurfForTest injects a Surfer for testing.
func (p *Proxy) SetSurfForTest(s surfer.Surfer) {
p.surf = s
}
// Update refreshes the proxy IP list.
func (p *Proxy) Update() result.VoidResult {
f, err := os.Open(config.Conf().ProxyFile)
if err != nil {
return result.TryErrVoid(err)
}
b, _ := io.ReadAll(f)
f.Close()
proxysIPType := p.proxyIPTypeRegexp.FindAllString(string(b), -1)
for _, proxy := range proxysIPType {
p.allIps[proxy] = p.ipRegexp.FindString(proxy)
p.all[proxy] = false
}
proxysUrlType := p.proxyUrlTypeRegexp.FindAllString(string(b), -1)
for _, proxy := range proxysUrlType {
gvalue := p.proxyUrlTypeRegexp.FindStringSubmatch(proxy)
p.allIps[proxy] = gvalue[6]
p.all[proxy] = false
}
log.Printf(" * Read proxy IPs: %v\n", len(p.all))
p.findOnline()
return result.OkVoid()
}
// findOnline filters proxy IPs that are online.
func (p *Proxy) findOnline() *Proxy {
log.Printf(" * Filtering online proxy IPs...")
p.online = 0
for proxy := range p.all {
p.threadPool <- true
go func(proxy string) {
alive := ping.Ping(p.allIps[proxy], CONN_TIMEOUT).IsOk()
p.Lock()
p.all[proxy] = alive
p.Unlock()
if alive {
atomic.AddInt32(&p.online, 1)
}
<-p.threadPool
}(proxy)
}
for len(p.threadPool) > 0 {
time.Sleep(0.2e9)
}
p.online = atomic.LoadInt32(&p.online)
log.Printf(" * Online proxy IP filtering complete, total: %v\n", p.online)
return p
}
// UpdateTicker updates the ticker.
func (p *Proxy) UpdateTicker(tickMinute int64) {
p.tickMinute = tickMinute
p.ticker = time.NewTicker(time.Duration(p.tickMinute) * time.Minute)
for _, proxyForHost := range p.usable {
proxyForHost.curIndex++
proxyForHost.isEcho = true
}
}
// GetOne returns an unused proxy IP for this cycle and its response time.
func (p *Proxy) GetOne(u string) option.Option[string] {
if p.online == 0 {
return option.None[string]()
}
u2, _ := url.Parse(u)
if u2.Host == "" {
logs.Log().Informational(" * [%v] Failed to set proxy IP, invalid target URL\n", u)
return option.None[string]()
}
var key = u2.Host
if strings.Count(key, ".") > 1 {
key = key[strings.Index(key, ".")+1:]
}
p.Lock()
defer p.Unlock()
var ok = true
var proxyForHost = p.usable[key]
select {
case <-p.ticker.C:
proxyForHost.curIndex++
if proxyForHost.curIndex >= proxyForHost.Len() {
_, ok = p.testAndSort(key, u2.Scheme+"://"+u2.Host)
}
proxyForHost.isEcho = true
default:
if proxyForHost == nil {
p.usable[key] = &ProxyForHost{
proxys: []string{},
timedelay: []time.Duration{},
isEcho: true,
}
proxyForHost, ok = p.testAndSort(key, u2.Scheme+"://"+u2.Host)
} else if l := proxyForHost.Len(); l == 0 {
ok = false
} else if proxyForHost.curIndex >= l {
_, ok = p.testAndSort(key, u2.Scheme+"://"+u2.Host)
proxyForHost.isEcho = true
}
}
if !ok {
logs.Log().Informational(" * [%v] Failed to set proxy IP, no available proxy IPs\n", key)
return option.None[string]()
}
curProxy := proxyForHost.proxys[proxyForHost.curIndex]
if proxyForHost.isEcho {
logs.Log().Informational(" * Set proxy IP to [%v](%v)\n",
curProxy,
proxyForHost.timedelay[proxyForHost.curIndex],
)
proxyForHost.isEcho = false
}
return option.Some(curProxy)
}
// testAndSort tests and sorts proxy IPs for the given host.
func (p *Proxy) testAndSort(key string, testHost string) (*ProxyForHost, bool) {
logs.Log().Informational(" * [%v] Testing and sorting proxy IPs...", key)
proxyForHost := p.usable[key]
proxyForHost.proxys = []string{}
proxyForHost.timedelay = []time.Duration{}
proxyForHost.curIndex = 0
for proxy, online := range p.all {
if !online {
continue
}
p.threadPool <- true
go func(proxy string) {
alive, timedelay := p.findUsable(proxy, testHost)
if alive {
proxyForHost.Mutex.Lock()
proxyForHost.proxys = append(proxyForHost.proxys, proxy)
proxyForHost.timedelay = append(proxyForHost.timedelay, timedelay)
proxyForHost.Mutex.Unlock()
}
<-p.threadPool
}(proxy)
}
for len(p.threadPool) > 0 {
time.Sleep(0.2e9)
}
if proxyForHost.Len() > 0 {
sort.Sort(proxyForHost)
logs.Log().Informational(" * [%v] Testing and sorting proxy IPs complete, available: %v\n", key, proxyForHost.Len())
return proxyForHost, true
}
logs.Log().Informational(" * [%v] Testing and sorting proxy IPs complete, no available proxy IPs\n", key)
return proxyForHost, false
}
// findUsable tests proxy IP availability.
func (p *Proxy) findUsable(proxy string, testHost string) (alive bool, timedelay time.Duration) {
t0 := time.Now()
req := &request.Request{
URL: testHost,
Method: "HEAD",
Header: make(http.Header),
DialTimeout: time.Second * time.Duration(DAIL_TIMEOUT),
ConnTimeout: time.Second * time.Duration(CONN_TIMEOUT),
TryTimes: TRY_TIMES,
}
req.SetProxy(proxy)
r := p.surf.Download(req)
if r.IsErr() {
return false, 0
}
resp := r.Unwrap()
if resp == nil || resp.StatusCode != http.StatusOK {
return false, 0
}
return true, time.Since(t0)
}
================================================
FILE: app/aid/proxy/proxy_test.go
================================================
package proxy
import (
"net/http"
"os"
"path/filepath"
"regexp"
"testing"
"time"
"github.com/andeya/gust/result"
"github.com/andeya/pholcus/app/downloader/surfer"
"github.com/andeya/pholcus/config"
)
func setupProxyDir(t *testing.T) (cleanup func()) {
tmp := t.TempDir()
configDir := filepath.Join(tmp, config.WorkRoot)
if err := os.MkdirAll(configDir, 0777); err != nil {
t.Fatalf("MkdirAll: %v", err)
}
proxyFile := filepath.Join(configDir, "proxy.lib")
if err := os.WriteFile(proxyFile, []byte(""), 0644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
orig, _ := os.Getwd()
if err := os.Chdir(tmp); err != nil {
t.Fatalf("Chdir: %v", err)
}
return func() { os.Chdir(orig) }
}
func newTestProxy() *Proxy {
return &Proxy{
ipRegexp: regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+`),
proxyIPTypeRegexp: regexp.MustCompile(`https?://([\w]*:[\w]*@)?[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+`),
proxyUrlTypeRegexp: regexp.MustCompile(`((https?|ftp):\/\/)?(([^:\n\r]+):([^@\n\r]+)@)?((www\.)?([^/\n\r:]+)):?([0-9]{1,5})?\/?([^?\n\r]+)?\??([^#\n\r]*)?#?([^\n\r]*)`),
allIps: map[string]string{},
all: map[string]bool{},
usable: make(map[string]*ProxyForHost),
threadPool: make(chan bool, MAX_THREAD_NUM),
surf: surfer.New(),
}
}
func TestProxy_Update_EmptyFile(t *testing.T) {
cleanup := setupProxyDir(t)
defer cleanup()
_ = config.Conf()
p := newTestProxy()
r := p.Update()
if r.IsErr() {
t.Errorf("Update: %v", r.UnwrapErr())
}
if p.Count() != 0 {
t.Errorf("Count = %v, want 0", p.Count())
}
}
func TestProxy_Update_WithIPs(t *testing.T) {
cleanup := setupProxyDir(t)
defer cleanup()
_ = config.Conf()
proxyFile := filepath.Join(config.WorkRoot, "proxy.lib")
content := "http://127.0.0.1:8080\nhttp://user:pass@127.0.0.1:9090"
if err := os.WriteFile(proxyFile, []byte(content), 0644); err != nil {
t.Fatalf("WriteFile: %v", err)
}
p := newTestProxy()
r := p.Update()
if r.IsErr() {
t.Errorf("Update: %v", r.UnwrapErr())
}
}
func TestProxy_GetOne_NoOnline(t *testing.T) {
p := &Proxy{online: 0}
if got := p.GetOne("http://example.com"); got.IsSome() {
t.Error("GetOne with online=0 want None")
}
}
func TestProxy_GetOne_EmptyHost(t *testing.T) {
p := &Proxy{online: 1}
if got := p.GetOne("http://"); got.IsSome() {
t.Error("GetOne with empty host want None")
}
}
func TestProxy_UpdateTicker(t *testing.T) {
p := &Proxy{
usable: make(map[string]*ProxyForHost),
}
p.usable["example.com"] = &ProxyForHost{curIndex: 0, isEcho: false}
p.UpdateTicker(5)
if p.ticker == nil {
t.Error("UpdateTicker should set ticker")
}
if p.tickMinute != 5 {
t.Errorf("tickMinute = %v, want 5", p.tickMinute)
}
}
func TestProxy_New(t *testing.T) {
cleanup := setupProxyDir(t)
defer cleanup()
_ = config.Conf()
p := New()
time.Sleep(100 * time.Millisecond)
if p.Count() != 0 {
t.Errorf("New with empty file Count = %v, want 0", p.Count())
}
}
func TestProxy_GetOne_WithUsable(t *testing.T) {
p := &Proxy{
online: 1,
ticker: time.NewTicker(time.Hour),
usable: map[string]*ProxyForHost{
"example.com": {
proxys: []string{"http://127.0.0.1:8080"},
timedelay: []time.Duration{time.Millisecond},
curIndex: 0,
isEcho: false,
},
},
}
got := p.GetOne("http://www.example.com/path")
if !got.IsSome() {
t.Fatal("GetOne want Some")
}
if got.Unwrap() != "http://127.0.0.1:8080" {
t.Errorf("GetOne = %v, want http://127.0.0.1:8080", got.Unwrap())
}
}
func TestProxy_GetOne_NoUsableForHost(t *testing.T) {
p := &Proxy{
online: 1,
ticker: time.NewTicker(time.Hour),
usable: map[string]*ProxyForHost{
"example.com": {
proxys: []string{},
timedelay: []time.Duration{},
curIndex: 0,
isEcho: false,
},
},
}
got := p.GetOne("http://www.example.com/path")
if got.IsSome() {
t.Error("GetOne with empty proxys want None")
}
}
type mockSurfer struct {
resp *http.Response
}
func (m *mockSurfer) Download(req surfer.Request) result.Result[*http.Response] {
if m.resp != nil {
return result.Ok(m.resp)
}
return result.TryErr[*http.Response](http.ErrHandlerTimeout)
}
func TestProxy_GetOne_TriggersTestAndSort(t *testing.T) {
cleanup := setupProxyDir(t)
defer cleanup()
_ = config.Conf()
p := newTestProxy()
p.SetSurfForTest(&mockSurfer{resp: &http.Response{StatusCode: http.StatusOK}})
p.all = map[string]bool{"http://127.0.0.1:8080": true}
p.allIps = map[string]string{"http://127.0.0.1:8080": "127.0.0.1"}
p.online = 1
p.ticker = time.NewTicker(time.Hour)
p.usable = map[string]*ProxyForHost{
"example.com": {
proxys: []string{"old"}, // curIndex will exceed after tick
timedelay: []time.Duration{time.Millisecond},
curIndex: 1,
isEcho: false,
},
}
got := p.GetOne("http://www.example.com/path")
if !got.IsSome() {
t.Fatal("GetOne want Some")
}
if got.Unwrap() != "http://127.0.0.1:8080" {
t.Errorf("GetOne = %v, want http://127.0.0.1:8080", got.Unwrap())
}
}
func TestProxy_Update_FileNotFound(t *testing.T) {
tmp := t.TempDir()
orig, _ := os.Getwd()
os.Chdir(tmp)
defer os.Chdir(orig)
_ = config.Conf()
p := newTestProxy()
r := p.Update()
if r.IsOk() {
t.Error("Update with missing file want Err")
}
}
================================================
FILE: app/app.go
================================================
// app interface for graphical user interface.
// Basic execution order: New() --> [SetLog(io.Writer) -->] Init() --> SpiderPrepare() --> Run()
// Package app provides the main entry and task scheduling for the crawler application.
package app
import (
"io"
"reflect"
"runtime/debug"
"strconv"
"sync"
"time"
"unicode"
"unicode/utf8"
"github.com/andeya/gust/option"
"github.com/andeya/pholcus/app/crawler"
"github.com/andeya/pholcus/app/distribute"
"github.com/andeya/pholcus/app/distribute/teleport"
"github.com/andeya/pholcus/app/downloader"
"github.com/andeya/pholcus/app/pipeline"
"github.com/andeya/pholcus/app/scheduler"
"github.com/andeya/pholcus/app/spider"
"github.com/andeya/pholcus/logs"
"github.com/andeya/pholcus/runtime/cache"
"github.com/andeya/pholcus/runtime/status"
)
type (
App interface {
SetLog(io.Writer) App // Set global log output to terminal
LogGoOn() App // Resume log output
LogRest() App // Pause log output
Init(mode int, port int, master string, w ...io.Writer) App // Must call Init before using App (except SetLog)
ReInit(mode int, port int, master string, w ...io.Writer) App // Switch run mode and reset log output target
GetAppConf(k ...string) interface{} // Get global config
SetAppConf(k string, v interface{}) App // Set global config (not called in client mode)
SpiderPrepare(original []*spider.Spider) App // Must call after setting global params and before Run() (not called in client mode)
Run() // Block until task completes (call after all config is done)
Stop() // Terminate task mid-run in Offline mode (blocks until current task stops)
IsRunning() bool // Check if task is running
IsPaused() bool // Check if task is paused
IsStopped() bool // Check if task has stopped
PauseRecover() // Pause or resume task in Offline mode
Status() int // Return current status
GetSpiderLib() []*spider.Spider // Get all spider species
GetSpiderByName(string) option.Option[*spider.Spider] // Get spider by name
GetSpiderQueue() crawler.SpiderQueue // Get spider queue interface
GetOutputLib() []string // Get all output methods
GetTaskJar() *distribute.TaskJar // Return task jar
distribute.Distributor // Implements distributed interface
}
Logic struct {
*cache.AppConf // Global config
*spider.SpiderSpecies // All spider species
crawler.SpiderQueue // Spider queue for current task
*distribute.TaskJar // Task storage passed between server and client
crawler.CrawlerPool // Crawler pool
teleport.Teleport // Socket duplex communication, JSON transport
sum [2]uint64 // Execution count
takeTime time.Duration // Execution duration
status int // Run status
finish chan bool
finishOnce sync.Once
canSocketLog bool
sync.RWMutex
}
)
// LogicApp is the global singleton core interface instance.
var LogicApp = New()
func New() App {
return newLogic()
}
func newLogic() *Logic {
return &Logic{
AppConf: cache.Task,
SpiderSpecies: spider.Species,
status: status.STOPPED,
Teleport: teleport.New(),
TaskJar: distribute.NewTaskJar(),
SpiderQueue: crawler.NewSpiderQueue(),
CrawlerPool: crawler.NewCrawlerPool(downloader.SurferDownloader),
}
}
// SetLog sets global log output to the given writer.
func (l *Logic) SetLog(w io.Writer) App {
logs.Log().SetOutput(w)
return l
}
// LogRest pauses log output.
func (l *Logic) LogRest() App {
logs.Log().PauseOutput()
return l
}
// LogGoOn resumes log output.
func (l *Logic) LogGoOn() App {
logs.Log().GoOn()
return l
}
// GetAppConf returns global config value(s).
func (l *Logic) GetAppConf(k ...string) interface{} {
defer func() {
if err := recover(); err != nil {
logs.Log().Error("panic recovered: %v\n%s", err, debug.Stack())
}
}()
if len(k) == 0 {
return l.AppConf
}
key := titleCase(k[0])
acv := reflect.ValueOf(l.AppConf).Elem()
return acv.FieldByName(key).Interface()
}
// SetAppConf sets a global config value.
func (l *Logic) SetAppConf(k string, v interface{}) App {
defer func() {
if err := recover(); err != nil {
logs.Log().Error("panic recovered: %v\n%s", err, debug.Stack())
}
}()
if k == "Limit" && v.(int64) <= 0 {
v = int64(spider.LIMIT)
} else if k == "BatchCap" && v.(int) < 1 {
v = int(1)
}
acv := reflect.ValueOf(l.AppConf).Elem()
key := titleCase(k)
if acv.FieldByName(key).CanSet() {
acv.FieldByName(key).Set(reflect.ValueOf(v))
}
return l
}
// Init initializes the app; must be called before use (except SetLog).
func (l *Logic) Init(mode int, port int, master string, w ...io.Writer) App {
l.AppConf = cache.Task
l.canSocketLog = false
if len(w) > 0 {
l.SetLog(w[0])
}
l.LogGoOn()
l.AppConf.Mode, l.AppConf.Port, l.AppConf.Master = mode, port, master
l.Teleport = teleport.New()
l.TaskJar = distribute.NewTaskJar()
l.SpiderQueue = crawler.NewSpiderQueue()
l.CrawlerPool = crawler.NewCrawlerPool(downloader.SurferDownloader)
switch l.AppConf.Mode {
case status.SERVER:
logs.Log().EnableStealOne(false)
if l.checkPort() {
logs.Log().Informational(" !! Current run mode: [ SERVER ] !!")
l.Teleport.SetAPI(distribute.MasterAPI(l)).Server(":" + strconv.Itoa(l.AppConf.Port))
}
case status.CLIENT:
if l.checkAll() {
logs.Log().Informational(" !! Current run mode: [ CLIENT ] !!")
l.Teleport.SetAPI(distribute.SlaveAPI(l)).Client(l.AppConf.Master, ":"+strconv.Itoa(l.AppConf.Port))
// Enable inter-node log forwarding
l.canSocketLog = true
logs.Log().EnableStealOne(true)
go l.socketLog()
}
case status.OFFLINE:
logs.Log().EnableStealOne(false)
logs.Log().Informational(" !! Current run mode: [ OFFLINE ] !!")
return l
default:
logs.Log().Warning(" * -- Please specify a valid run mode! --")
return l
}
return l
}
// ReInit switches run mode; use when changing mode.
func (l *Logic) ReInit(mode int, port int, master string, w ...io.Writer) App {
if !l.IsStopped() {
l.Stop()
}
l.LogRest()
if l.Teleport != nil {
l.Teleport.Close()
}
// Wait for shutdown
if mode == status.UNSET {
l = newLogic()
l.AppConf.Mode = status.UNSET
return l
}
// Restart
l = newLogic().Init(mode, port, master, w...).(*Logic)
return l
}
// SpiderPrepare must be called after setting global params and immediately before Run().
// original is the raw spider species from spider package without prior assignment.
// Spiders with explicit Keyin are not reassigned.
// Not called in client mode.
func (l *Logic) SpiderPrepare(original []*spider.Spider) App {
l.SpiderQueue.Reset()
for _, sp := range original {
spcopy := sp.Copy()
spcopy.SetPausetime(l.AppConf.Pausetime)
if spcopy.GetLimit() == spider.LIMIT {
spcopy.SetLimit(l.AppConf.Limit)
} else {
spcopy.SetLimit(-1 * l.AppConf.Limit)
}
l.SpiderQueue.Add(spcopy)
}
l.SpiderQueue.AddKeyins(l.AppConf.Keyins)
return l
}
// GetOutputLib returns all output methods.
func (l *Logic) GetOutputLib() []string {
return pipeline.GetOutputLib()
}
// GetSpiderLib returns all spider species.
func (l *Logic) GetSpiderLib() []*spider.Spider {
return l.SpiderSpecies.Get()
}
// GetSpiderByName returns a spider by name.
func (l *Logic) GetSpiderByName(name string) option.Option[*spider.Spider] {
return l.SpiderSpecies.GetByNameOpt(name)
}
// GetMode returns current run mode.
func (l *Logic) GetMode() int {
return l.AppConf.Mode
}
// GetTaskJar returns the task jar.
func (l *Logic) GetTaskJar() *distribute.TaskJar {
return l.TaskJar
}
// CountNodes returns connected node count in server/client mode.
func (l *Logic) CountNodes() int {
return l.Teleport.CountNodes()
}
// GetSpiderQueue returns the spider queue interface.
func (l *Logic) GetSpiderQueue() crawler.SpiderQueue {
return l.SpiderQueue
}
// Run executes the task.
func (l *Logic) Run() {
l.LogGoOn()
if l.AppConf.Mode != status.CLIENT && l.SpiderQueue.Len() == 0 {
logs.Log().Warning(" * -- Task list cannot be empty --")
l.LogRest()
return
}
l.finish = make(chan bool)
l.finishOnce = sync.Once{}
l.sum[0], l.sum[1] = 0, 0
l.takeTime = 0
l.setStatus(status.RUN)
defer l.setStatus(status.STOPPED)
switch l.AppConf.Mode {
case status.OFFLINE:
l.offline()
case status.SERVER:
l.server()
case status.CLIENT:
l.client()
default:
return
}
<-l.finish
}
// PauseRecover pauses or resumes the task in Offline mode.
func (l *Logic) PauseRecover() {
switch l.Status() {
case status.PAUSE:
l.setStatus(status.RUN)
case status.RUN:
l.setStatus(status.PAUSE)
}
scheduler.PauseRecover()
}
// Stop terminates the task mid-run in Offline mode.
func (l *Logic) Stop() {
if l.status == status.STOPPED {
return
}
if l.status != status.STOP {
// Stop order must not be reversed
l.setStatus(status.STOP)
scheduler.Stop()
l.CrawlerPool.Stop()
}
for !l.IsStopped() {
time.Sleep(time.Second)
}
}
// IsRunning reports whether the task is running.
func (l *Logic) IsRunning() bool {
return l.status == status.RUN
}
// IsPaused reports whether the task is paused.
func (l *Logic) IsPaused() bool {
return l.status == status.PAUSE
}
// IsStopped reports whether the task has stopped.
func (l *Logic) IsStopped() bool {
return l.status == status.STOPPED
}
// Status returns current run status.
func (l *Logic) Status() int {
l.RWMutex.RLock()
defer l.RWMutex.RUnlock()
return l.status
}
// setStatus sets the run status.
func (l *Logic) setStatus(status int) {
l.RWMutex.Lock()
defer l.RWMutex.Unlock()
l.status = status
}
// --- Private methods ---
// offline runs in offline mode.
func (l *Logic) offline() {
l.exec()
}
// server runs in server mode; must be called after SpiderPrepare() to add tasks.
// Generated tasks use the same global config.
func (l *Logic) server() {
defer func() {
l.finishOnce.Do(func() { close(l.finish) })
}()
tasksNum, spidersNum := l.addNewTask()
if tasksNum == 0 {
return
}
logs.Log().Informational(" * ")
logs.Log().Informational(` *********************************************************************************************************************************** `)
logs.Log().Informational(" * ")
logs.Log().Informational(" * -- Successfully added %v tasks, %v spider rules in total --", tasksNum, spidersNum)
logs.Log().Informational(" * ")
logs.Log().Informational(` *********************************************************************************************************************************** `)
}
// addNewTask generates tasks and adds them to the jar in server mode.
func (l *Logic) addNewTask() (tasksNum, spidersNum int) {
length := l.SpiderQueue.Len()
t := distribute.Task{}
l.setTask(&t)
for i, sp := range l.SpiderQueue.GetAll() {
t.Spiders = append(t.Spiders, map[string]string{"name": sp.GetName(), "keyin": sp.GetKeyin()})
spidersNum++
if i > 0 && i%10 == 0 && length > 10 {
one := t
l.TaskJar.Push(&one)
tasksNum++
t.Spiders = []map[string]string{}
}
}
if len(t.Spiders) != 0 {
one := t
l.TaskJar.Push(&one)
tasksNum++
}
return
}
// client runs in client mode.
func (l *Logic) client() {
defer func() {
l.finishOnce.Do(func() { close(l.finish) })
}()
for {
t := l.downTask()
if l.Status() == status.STOP || l.Status() == status.STOPPED {
return
}
l.taskToRun(t)
l.sum[0], l.sum[1] = 0, 0
l.takeTime = 0
l.exec()
}
}
// downTask fetches a task from the jar in client mode.
func (l *Logic) downTask() *distribute.Task {
for {
if l.Status() == status.STOP || l.Status() == status.STOPPED {
return nil
}
if l.CountNodes() == 0 && l.TaskJar.Len() == 0 {
time.Sleep(time.Second)
continue
}
if l.TaskJar.Len() == 0 {
l.Request(nil, "task", "")
for l.TaskJar.Len() == 0 {
if l.CountNodes() == 0 {
break
}
time.Sleep(time.Second)
}
if l.TaskJar.Len() == 0 {
continue
}
}
return l.TaskJar.Pull()
}
}
// taskToRun prepares run conditions from a task in client mode.
func (l *Logic) taskToRun(t *distribute.Task) {
l.SpiderQueue.Reset()
l.setAppConf(t)
for _, n := range t.Spiders {
spOpt := l.SpiderSpecies.GetByNameOpt(n["name"])
if spOpt.IsNone() {
continue
}
spcopy := spOpt.Unwrap().Copy()
spcopy.SetPausetime(t.Pausetime)
if spcopy.GetLimit() > 0 {
spcopy.SetLimit(t.Limit)
} else {
spcopy.SetLimit(-1 * t.Limit)
}
if v, ok := n["keyin"]; ok {
spcopy.SetKeyin(v)
}
l.SpiderQueue.Add(spcopy)
}
}
// exec starts task execution.
func (l *Logic) exec() {
count := l.SpiderQueue.Len()
cache.ResetPageCount()
pipeline.RefreshOutput()
scheduler.Init(l.AppConf.ThreadNum, l.AppConf.ProxyMinute)
l.CrawlerPool.SetPipelineConfig(l.AppConf.OutType, l.AppConf.BatchCap)
crawlerCap := l.CrawlerPool.Reset(count)
logs.Log().Informational(" * Total tasks (tasks * custom configs): %v\n", count)
logs.Log().Informational(" * Crawler pool capacity: %v\n", crawlerCap)
logs.Log().Informational(" * Max concurrent goroutines: %v\n", l.AppConf.ThreadNum)
logs.Log().Informational(" * Default random pause: %v~%v ms\n", l.AppConf.Pausetime/2, l.AppConf.Pausetime*2)
logs.Log().App(" * -- Starting crawl, please wait --")
logs.Log().Informational(` *********************************************************************************************************************************** `)
cache.StartTime = time.Now()
if l.AppConf.Mode == status.OFFLINE {
go l.goRun(count)
} else {
l.goRun(count)
}
}
// goRun executes the task.
func (l *Logic) goRun(count int) {
var i int
for i = 0; i < count && l.Status() != status.STOP; i++ {
for l.IsPaused() {
time.Sleep(time.Second)
}
if opt := l.CrawlerPool.UseOpt(); opt.IsSome() {
c := opt.Unwrap()
go func(i int, c crawler.Crawler) {
c.Init(l.SpiderQueue.GetByIndex(i)).Run()
l.RWMutex.RLock()
if l.status != status.STOP {
l.CrawlerPool.Free(c)
}
l.RWMutex.RUnlock()
}(i, c)
}
}
for ii := 0; ii < i; ii++ {
s := <-cache.ReportChan
if (s.DataNum == 0) && (s.FileNum == 0) {
logs.Log().App(" * [Task subtotal: %s | KEYIN: %s] No results, duration %v\n", s.SpiderName, s.Keyin, s.Time)
continue
}
logs.Log().Informational(" * ")
switch {
case s.DataNum > 0 && s.FileNum == 0:
logs.Log().App(" * [Task subtotal: %s | KEYIN: %s] Collected %v data items, duration %v\n",
s.SpiderName, s.Keyin, s.DataNum, s.Time)
case s.DataNum == 0 && s.FileNum > 0:
logs.Log().App(" * [Task subtotal: %s | KEYIN: %s] Downloaded %v files, duration %v\n",
s.SpiderName, s.Keyin, s.FileNum, s.Time)
default:
logs.Log().App(" * [Task subtotal: %s | KEYIN: %s] Collected %v data items + %v files, duration %v\n",
s.SpiderName, s.Keyin, s.DataNum, s.FileNum, s.Time)
}
l.sum[0] += s.DataNum
l.sum[1] += s.FileNum
}
l.takeTime = time.Since(cache.StartTime)
var prefix = func() string {
if l.Status() == status.STOP {
return "Task cancelled: "
}
return "This run: "
}()
logs.Log().Informational(" * ")
logs.Log().Informational(` *********************************************************************************************************************************** `)
logs.Log().Informational(" * ")
switch {
case l.sum[0] > 0 && l.sum[1] == 0:
logs.Log().App(" * -- %sTotal collected [%v data items], crawled [success %v URL + fail %v URL = total %v URL], duration [%v] --",
prefix, l.sum[0], cache.GetPageCount(1), cache.GetPageCount(-1), cache.GetPageCount(0), l.takeTime)
case l.sum[0] == 0 && l.sum[1] > 0:
logs.Log().App(" * -- %sTotal collected [%v files], crawled [success %v URL + fail %v URL = total %v URL], duration [%v] --",
prefix, l.sum[1], cache.GetPageCount(1), cache.GetPageCount(-1), cache.GetPageCount(0), l.takeTime)
case l.sum[0] == 0 && l.sum[1] == 0:
logs.Log().App(" * -- %sNo results, crawled [success %v URL + fail %v URL = total %v URL], duration [%v] --",
prefix, cache.GetPageCount(1), cache.GetPageCount(-1), cache.GetPageCount(0), l.takeTime)
default:
logs.Log().App(" * -- %sTotal collected [%v data items + %v files], crawled [success %v URL + fail %v URL = total %v URL], duration [%v] --",
prefix, l.sum[0], l.sum[1], cache.GetPageCount(1), cache.GetPageCount(-1), cache.GetPageCount(0), l.takeTime)
}
logs.Log().Informational(" * ")
logs.Log().Informational(` *********************************************************************************************************************************** `)
if l.AppConf.Mode == status.OFFLINE {
l.LogRest()
l.finishOnce.Do(func() { close(l.finish) })
}
}
// socketLog forwards client logs to the server.
func (l *Logic) socketLog() {
for l.canSocketLog {
_, msg, ok := logs.Log().StealOne()
if !ok {
return
}
if l.Teleport.CountNodes() == 0 {
continue
}
l.Teleport.Request(msg, "log", "")
}
}
func (l *Logic) checkPort() bool {
if l.AppConf.Port == 0 {
logs.Log().Warning(" * -- Distributed port cannot be empty --")
return false
}
return true
}
func (l *Logic) checkAll() bool {
if l.AppConf.Master == "" || !l.checkPort() {
logs.Log().Warning(" * -- Server address cannot be empty --")
return false
}
return true
}
// setAppConf applies task config to global runtime config.
func (l *Logic) setAppConf(task *distribute.Task) {
l.AppConf.ThreadNum = task.ThreadNum
l.AppConf.Pausetime = task.Pausetime
l.AppConf.OutType = task.OutType
l.AppConf.BatchCap = task.BatchCap
l.AppConf.SuccessInherit = task.SuccessInherit
l.AppConf.FailureInherit = task.FailureInherit
l.AppConf.Limit = task.Limit
l.AppConf.ProxyMinute = task.ProxyMinute
l.AppConf.Keyins = task.Keyins
}
func (l *Logic) setTask(task *distribute.Task) {
task.ThreadNum = l.AppConf.ThreadNum
task.Pausetime = l.AppConf.Pausetime
task.OutType = l.AppConf.OutType
task.BatchCap = l.AppConf.BatchCap
task.SuccessInherit = l.AppConf.SuccessInherit
task.FailureInherit = l.AppConf.FailureInherit
task.Limit = l.AppConf.Limit
task.ProxyMinute = l.AppConf.ProxyMinute
task.Keyins = l.AppConf.Keyins
}
func titleCase(s string) string {
if s == "" {
return s
}
r, size := utf8.DecodeRuneInString(s)
return string(unicode.ToUpper(r)) + s[size:]
}
================================================
FILE: app/app_test.go
================================================
package app
import (
"bytes"
"testing"
"time"
"github.com/andeya/pholcus/app/spider"
"github.com/andeya/pholcus/runtime/status"
)
func TestNew(t *testing.T) {
a := New()
if a == nil {
t.Fatal("New returned nil")
}
}
func TestLogic_SetLog_LogGoOn_LogRest(t *testing.T) {
a := New()
buf := &bytes.Buffer{}
a.SetLog(buf)
a.LogGoOn()
a.LogRest()
}
func TestLogic_GetAppConf(t *testing.T) {
a := New().(*Logic)
tests := []struct {
keys []string
}{
{nil},
{[]string{"Mode"}},
{[]string{"ThreadNum"}},
{[]string{"Limit"}},
}
for _, tt := range tests {
_ = a.GetAppConf(tt.keys...)
}
}
func TestLogic_SetAppConf(t *testing.T) {
a := New().(*Logic)
tests := []struct {
k string
v interface{}
}{
{"Limit", int64(100)},
{"Limit", int64(0)},
{"BatchCap", 50},
{"BatchCap", 0},
{"ThreadNum", 10},
}
for _, tt := range tests {
a.SetAppConf(tt.k, tt.v)
}
}
func TestLogic_GetSpiderLib(t *testing.T) {
a := New()
lib := a.GetSpiderLib()
if lib == nil {
t.Error("GetSpiderLib returned nil")
}
}
func TestLogic_GetSpiderByName(t *testing.T) {
a := New()
opt := a.GetSpiderByName("nonexistent")
if opt.IsSome() {
t.Error("GetSpiderByName(nonexistent) should return None")
}
}
func TestLogic_GetSpiderQueue(t *testing.T) {
a := New()
q := a.GetSpiderQueue()
if q == nil {
t.Fatal("GetSpiderQueue returned nil")
}
if q.Len() != 0 {
t.Errorf("new queue Len() = %d, want 0", q.Len())
}
}
func TestLogic_GetOutputLib(t *testing.T) {
a := New()
lib := a.GetOutputLib()
if len(lib) == 0 {
t.Error("GetOutputLib returned empty")
}
}
func TestLogic_GetTaskJar(t *testing.T) {
a := New()
jar := a.GetTaskJar()
if jar == nil {
t.Fatal("GetTaskJar returned nil")
}
}
func TestLogic_Status_IsRunning_IsPaused_IsStopped(t *testing.T) {
a := New().(*Logic)
if a.Status() != status.STOPPED {
t.Errorf("Status() = %d, want STOPPED", a.Status())
}
if a.IsRunning() {
t.Error("IsRunning() = true, want false")
}
if a.IsPaused() {
t.Error("IsPaused() = true, want false")
}
if !a.IsStopped() {
t.Error("IsStopped() = false, want true")
}
}
func TestLogic_Init_Offline(t *testing.T) {
a := New()
got := a.Init(status.OFFLINE, 2015, "", nil)
if got == nil {
t.Fatal("Init returned nil")
}
}
func TestLogic_Init_Server_invalidPort(t *testing.T) {
a := New()
got := a.Init(status.SERVER, 0, "", nil)
if got == nil {
t.Fatal("Init returned nil")
}
}
func TestLogic_Init_Server_validPort(t *testing.T) {
a := New()
got := a.Init(status.SERVER, 2016, "", nil)
if got == nil {
t.Fatal("Init returned nil")
}
}
func TestLogic_Init_Client_invalidMaster(t *testing.T) {
a := New()
got := a.Init(status.CLIENT, 2015, "", nil)
if got == nil {
t.Fatal("Init returned nil")
}
}
func TestLogic_Init_invalidMode(t *testing.T) {
a := New()
got := a.Init(999, 2015, "", nil)
if got == nil {
t.Fatal("Init returned nil")
}
}
func TestLogic_GetMode(t *testing.T) {
a := New().(*Logic)
a.Init(status.OFFLINE, 2015, "", nil)
if a.GetMode() != status.OFFLINE {
t.Errorf("GetMode() = %d, want OFFLINE", a.GetMode())
}
}
func TestLogic_ReInit(t *testing.T) {
a := New().(*Logic)
a.Init(status.OFFLINE, 2015, "", nil)
got := a.ReInit(status.UNSET, 0, "")
if got == nil {
t.Fatal("ReInit returned nil")
}
}
func TestLogic_GetAppConf_titleCase(t *testing.T) {
a := New().(*Logic)
a.SetAppConf("limit", int64(50))
v := a.GetAppConf("limit")
if v == nil {
t.Fatal("GetAppConf(limit) returned nil")
}
}
func TestLogic_SpiderPrepare(t *testing.T) {
a := New().(*Logic)
a.Init(status.OFFLINE, 2015, "", nil)
sp := &spider.Spider{
Name: "TestSpider",
RuleTree: &spider.RuleTree{Trunk: map[string]*spider.Rule{}},
Limit: spider.LIMIT,
Pausetime: 100,
}
sp.Register()
got := a.SpiderPrepare([]*spider.Spider{sp})
if got == nil {
t.Fatal("SpiderPrepare returned nil")
}
if a.GetSpiderQueue().Len() < 1 {
t.Errorf("SpiderPrepare Len() = %d, want >= 1", a.GetSpiderQueue().Len())
}
}
func TestLogic_Run_emptyQueue(t *testing.T) {
a := New().(*Logic)
a.Init(status.OFFLINE, 2015, "", nil)
a.Run()
}
func TestLogic_Stop_whenStopped(t *testing.T) {
a := New().(*Logic)
a.Stop()
}
func TestLogic_PauseRecover(t *testing.T) {
a := New().(*Logic)
a.Init(status.OFFLINE, 2015, "", nil)
a.PauseRecover()
}
func TestLogic_Run_offline_withSpiders(t *testing.T) {
sp := &spider.Spider{
Name: "AppTestSpider",
RuleTree: &spider.RuleTree{Root: func(_ *spider.Context) {}, Trunk: map[string]*spider.Rule{}},
Limit: spider.LIMIT,
Pausetime: 100,
}
sp.Register()
a := New().(*Logic)
a.Init(status.OFFLINE, 2015, "", nil)
a.SpiderPrepare([]*spider.Spider{sp})
go func() {
time.Sleep(3 * time.Second)
a.Stop()
}()
a.Run()
}
func TestLogic_Run_server_withSpiders(t *testing.T) {
sp := &spider.Spider{
Name: "AppTestSpiderServer",
RuleTree: &spider.RuleTree{Root: func(_ *spider.Context) {}, Trunk: map[string]*spider.Rule{}},
Limit: spider.LIMIT,
Pausetime: 100,
}
sp.Register()
a := New().(*Logic)
a.Init(status.SERVER, 2018, "", nil)
a.SpiderPrepare([]*spider.Spider{sp})
go func() {
time.Sleep(2 * time.Second)
a.Stop()
}()
a.Run()
}
================================================
FILE: app/crawler/crawler.go
================================================
// Package crawler provides the core crawler engine for request scheduling and page downloading.
package crawler
import (
"bytes"
"math/rand"
"runtime"
"time"
"github.com/andeya/gust/result"
"github.com/andeya/pholcus/app/downloader"
"github.com/andeya/pholcus/app/downloader/request"
"github.com/andeya/pholcus/app/pipeline"
"github.com/andeya/pholcus/app/spider"
"github.com/andeya/pholcus/logs"
"github.com/andeya/pholcus/runtime/cache"
)
// Crawler is the core crawler engine.
type (
Crawler interface {
Init(*spider.Spider) Crawler // Init initializes the crawler engine
Run() // Run executes the task
Stop() // Stop terminates the crawler
CanStop() bool // CanStop reports whether the crawler can be stopped
GetID() int // GetID returns the engine ID
}
crawler struct {
*spider.Spider // spider rule being executed
downloader.Downloader // shared downloader
pipeline.Pipeline // result collection and output pipeline
id int // engine ID
outType string // output type for pipeline
batchCap int // batch output capacity for pipeline
pause [2]int64 // [min request interval ms, max additional interval ms]
}
)
// New creates a new Crawler with the given ID, Downloader, and pipeline config.
func New(id int, dl downloader.Downloader, outType string, batchCap int) Crawler {
return &crawler{
id: id,
Downloader: dl,
outType: outType,
batchCap: batchCap,
}
}
// Init initializes the crawler with the given spider.
func (c *crawler) Init(sp *spider.Spider) Crawler {
c.Spider = sp.ReqmatrixInit()
c.Pipeline = pipeline.New(sp, c.outType, c.batchCap)
c.pause[0] = sp.Pausetime / 2
if c.pause[0] > 0 {
c.pause[1] = c.pause[0] * 3
} else {
c.pause[1] = 1
}
return c
}
// Run is the main entry point for task execution.
func (c *crawler) Run() {
c.Pipeline.Start()
done := make(chan bool)
go func() {
c.run()
close(done)
}()
c.Spider.Start()
<-done
c.Pipeline.Stop()
}
// Stop terminates the crawler and its pipeline.
func (c *crawler) Stop() {
c.Spider.Stop()
c.Pipeline.Stop()
}
func (c *crawler) run() {
for {
req := c.GetOne()
if req == nil {
if c.Spider.CanStop() {
break
}
time.Sleep(20 * time.Millisecond)
continue
}
c.UseOne()
go func() {
defer func() {
c.FreeOne()
}()
logs.Log().Debug(" * Start: %v", req.GetURL())
c.Process(req)
}()
c.sleep()
}
c.Spider.Defer()
}
// Process downloads a request, parses the response, and sends results to the pipeline.
func (c *crawler) Process(req *request.Request) {
var (
downUrl = req.GetURL()
sp = c.Spider
)
defer func() {
if p := recover(); p != nil {
if sp.IsStopping() {
return
}
if sp.DoHistory(req, false) {
cache.PageFailCount()
}
stack := make([]byte, 4<<10)
length := runtime.Stack(stack, true)
start := bytes.Index(stack, []byte("/src/runtime/panic.go"))
stack = stack[start:length]
start = bytes.Index(stack, []byte("\n")) + 1
stack = stack[start:]
if end := bytes.Index(stack, []byte("\ngoroutine ")); end != -1 {
stack = stack[:end]
}
stack = bytes.Replace(stack, []byte("\n"), []byte("\r\n"), -1)
logs.Log().Error(" * Panic [process][%s]: %s\r\n[TRACE]\r\n%s", downUrl, p, stack)
}
}()
var ctx = c.Downloader.Download(sp, req) // download page
if r := result.TryErrVoid(ctx.GetError()); r.IsErr() {
if sp.DoHistory(req, false) {
cache.PageFailCount()
}
logs.Log().Error(" * Fail [download][%v]: %v\n", downUrl, r.UnwrapErr())
return
}
ctx.Parse(req.GetRuleName())
if parseErr := ctx.GetError(); parseErr != nil {
if sp.DoHistory(req, false) {
cache.PageFailCount()
}
logs.Log().Error(" * Fail [parse][%v]: %v\n", downUrl, parseErr)
return
}
for _, f := range ctx.PullFiles() {
if c.Pipeline.CollectFile(f).IsErr() {
break
}
}
for _, item := range ctx.PullItems() {
if c.Pipeline.CollectData(item).IsErr() {
break
}
}
sp.DoHistory(req, true)
cache.PageSuccCount()
logs.Log().Informational(" * Success: %v\n", downUrl)
spider.PutContext(ctx)
}
func (c *crawler) sleep() {
sleeptime := c.pause[0] + rand.Int63n(c.pause[1])
time.Sleep(time.Duration(sleeptime) * time.Millisecond)
}
// GetOne pulls one request from the scheduler.
func (c *crawler) GetOne() *request.Request {
return c.Spider.RequestPull()
}
// UseOne acquires one resource slot from the scheduler.
func (c *crawler) UseOne() {
c.Spider.RequestUse()
}
// FreeOne releases one resource slot to the scheduler.
func (c *crawler) FreeOne() {
c.Spider.RequestFree()
}
// SetID sets the crawler ID.
func (c *crawler) SetID(id int) {
c.id = id
}
// GetID returns the crawler engine ID.
func (c *crawler) GetID() int {
return c.id
}
================================================
FILE: app/crawler/crawler_test.go
================================================
package crawler
import (
"fmt"
"net/http"
"testing"
"time"
"github.com/andeya/pholcus/app/downloader/request"
"github.com/andeya/pholcus/app/scheduler"
"github.com/andeya/pholcus/app/spider"
)
func TestNew(t *testing.T) {
c := New(1, &mockDownloader{}, "csv", 10)
if c == nil {
t.Fatal("New returned nil")
}
if c.GetID() != 1 {
t.Errorf("GetID() = %d, want 1", c.GetID())
}
}
func TestCrawler_GetID(t *testing.T) {
c := New(42, &mockDownloader{}, "csv", 10)
if got := c.GetID(); got != 42 {
t.Errorf("GetID() = %d, want 42", got)
}
}
func TestCrawler_Init(t *testing.T) {
scheduler.Init(4, 0)
c := New(0, &mockDownloader{}, "csv", 10)
sp := &spider.Spider{
Name: "TestSpider",
RuleTree: &spider.RuleTree{Trunk: map[string]*spider.Rule{}},
Limit: -5,
}
got := c.Init(sp)
if got != c {
t.Error("Init should return self")
}
}
func TestCrawler_Init_zeroPause(t *testing.T) {
scheduler.Init(4, 0)
c := New(0, &mockDownloader{}, "csv", 10)
sp := &spider.Spider{
Name: "TestSpider",
RuleTree: &spider.RuleTree{Trunk: map[string]*spider.Rule{}},
Limit: -5,
Pausetime: 0,
}
c.Init(sp)
}
func TestCrawler_GetOne_UseOne_FreeOne(t *testing.T) {
scheduler.Init(4, 0)
cr := New(0, &mockDownloader{}, "csv", 10).(*crawler)
sp := &spider.Spider{
Name: "TestSpider",
RuleTree: &spider.RuleTree{Trunk: map[string]*spider.Rule{}},
Limit: -5,
}
cr.Init(sp)
req := cr.GetOne()
if req != nil {
t.Error("GetOne on empty matrix should return nil")
}
cr.UseOne()
cr.FreeOne()
}
func TestCrawler_CanStop(t *testing.T) {
scheduler.Init(4, 0)
c := New(0, &mockDownloader{}, "csv", 10)
sp := &spider.Spider{
Name: "TestSpider",
RuleTree: &spider.RuleTree{Trunk: map[string]*spider.Rule{}},
Limit: -5,
}
c.Init(sp)
if !c.CanStop() {
t.Error("CanStop on empty matrix should be true")
}
}
func TestCrawler_Stop(t *testing.T) {
scheduler.Init(4, 0)
c := New(0, &mockDownloader{}, "csv", 10)
sp := &spider.Spider{
Name: "TestSpider",
RuleTree: &spider.RuleTree{Trunk: map[string]*spider.Rule{}},
Limit: -5,
}
c.Init(sp)
c.Stop()
}
func TestCrawler_SetID(t *testing.T) {
cr := New(0, &mockDownloader{}, "csv", 10).(*crawler)
cr.SetID(99)
if cr.GetID() != 99 {
t.Errorf("GetID() = %d, want 99", cr.GetID())
}
}
type errorDownloader struct{}
func (d *errorDownloader) Download(sp *spider.Spider, req *request.Request) *spider.Context {
ctx := spider.GetContext(sp, req)
ctx.SetError(fmt.Errorf("download failed"))
return ctx
}
func TestCrawler_Process_downloadError(t *testing.T) {
scheduler.Init(4, 0)
cr := New(0, &errorDownloader{}, "csv", 10).(*crawler)
sp := &spider.Spider{
Name: "TestSpider",
RuleTree: &spider.RuleTree{Trunk: map[string]*spider.Rule{}},
Limit: -5,
}
cr.Init(sp)
req := &request.Request{URL: "http://example.com", Rule: "r"}
req.Prepare()
cr.Process(req)
}
func TestCrawler_Run(t *testing.T) {
scheduler.Init(4, 0)
sd := &successDownloader{}
cr := New(0, sd, "csv", 10).(*crawler)
sp := &spider.Spider{
Name: "CrawlerRunTestSpider",
RuleTree: &spider.RuleTree{
Root: func(ctx *spider.Context) {
time.Sleep(50 * time.Millisecond)
req := &request.Request{URL: "http://example.com", Rule: "r"}
req.Prepare()
ctx.AddQueue(req)
},
Trunk: map[string]*spider.Rule{"r": {ParseFunc: func(_ *spider.Context) {}}},
},
Limit: -5,
}
sp.Register()
cr.Init(sp)
cr.Run()
}
type successDownloader struct{}
func (d *successDownloader) Download(sp *spider.Spider, req *request.Request) *spider.Context {
ctx := spider.GetContext(sp, req)
ctx.SetResponse(&http.Response{StatusCode: 200})
return ctx
}
func TestCrawler_Process_success(t *testing.T) {
scheduler.Init(4, 0)
cr := New(0, &successDownloader{}, "csv", 10).(*crawler)
sp := &spider.Spider{
Name: "TestSpider",
RuleTree: &spider.RuleTree{
Root: func(_ *spider.Context) {},
Trunk: map[string]*spider.Rule{"r": {ParseFunc: func(_ *spider.Context) {}}},
},
Limit: -5,
}
cr.Init(sp)
req := &request.Request{URL: "http://example.com", Rule: "r"}
req.Prepare()
cr.Process(req)
}
================================================
FILE: app/crawler/crawlerpool.go
================================================
package crawler
import (
"sync"
"time"
"github.com/andeya/gust/option"
"github.com/andeya/pholcus/app/downloader"
"github.com/andeya/pholcus/config"
"github.com/andeya/pholcus/runtime/status"
)
// CrawlerPool manages a pool of crawler engines.
type (
CrawlerPool interface {
Reset(spiderNum int) int
SetPipelineConfig(outType string, batchCap int)
Use() Crawler
UseOpt() option.Option[Crawler]
Free(Crawler)
Stop()
}
cq struct {
capacity int
count int
usable chan Crawler
all []Crawler
dl downloader.Downloader
outType string
batchCap int
status int
sync.RWMutex
}
)
// NewCrawlerPool creates a new crawler pool with the given Downloader.
func NewCrawlerPool(dl downloader.Downloader) CrawlerPool {
return &cq{
status: status.RUN,
dl: dl,
all: make([]Crawler, 0, config.Conf().CrawlsCap),
}
}
// SetPipelineConfig sets the output type and batch capacity for new crawlers.
func (cq *cq) SetPipelineConfig(outType string, batchCap int) {
cq.Lock()
defer cq.Unlock()
cq.outType = outType
cq.batchCap = batchCap
}
// Reset configures the pool size based on the number of spiders to run.
// When reusing a pool instance, it efficiently resizes to the new capacity.
func (cq *cq) Reset(spiderNum int) int {
cq.Lock()
defer cq.Unlock()
var wantNum int
if spiderNum < config.Conf().CrawlsCap {
wantNum = spiderNum
} else {
wantNum = config.Conf().CrawlsCap
}
if wantNum <= 0 {
wantNum = 1
}
cq.capacity = wantNum
cq.count = 0
cq.usable = make(chan Crawler, wantNum)
for _, crawler := range cq.all {
if cq.count < cq.capacity {
cq.usable <- crawler
cq.count++
}
}
cq.status = status.RUN
return wantNum
}
// Use acquires a crawler from the pool in a concurrency-safe manner.
func (cq *cq) Use() Crawler {
return cq.UseOpt().UnwrapOr(nil)
}
// UseOpt acquires a crawler from the pool; returns None when pool is stopped.
func (cq *cq) UseOpt() option.Option[Crawler] {
var crawler Crawler
for {
cq.Lock()
if cq.status == status.STOP {
cq.Unlock()
return option.None[Crawler]()
}
select {
case crawler = <-cq.usable:
cq.Unlock()
return option.Some(crawler)
default:
if cq.count < cq.capacity {
crawler = New(cq.count, cq.dl, cq.outType, cq.batchCap)
cq.all = append(cq.all, crawler)
cq.count++
cq.Unlock()
return option.Some(crawler)
}
}
cq.Unlock()
time.Sleep(time.Second)
}
}
// Free returns a crawler to the pool.
func (cq *cq) Free(crawler Crawler) {
cq.RLock()
defer cq.RUnlock()
if cq.status == status.STOP || !crawler.CanStop() {
return
}
cq.usable <- crawler
}
// Stop terminates all crawler tasks in the pool.
func (cq *cq) Stop() {
cq.Lock()
if cq.status == status.STOP {
cq.Unlock()
return
}
cq.status = status.STOP
close(cq.usable)
cq.usable = nil
cq.Unlock()
for _, crawler := range cq.all {
crawler.Stop()
}
}
================================================
FILE: app/crawler/crawlerpool_test.go
================================================
package crawler
import (
"testing"
"github.com/andeya/pholcus/app/downloader"
"github.com/andeya/pholcus/app/downloader/request"
"github.com/andeya/pholcus/app/scheduler"
"github.com/andeya/pholcus/app/spider"
"github.com/andeya/pholcus/config"
)
type mockDownloader struct{}
func (d *mockDownloader) Download(_ *spider.Spider, _ *request.Request) *spider.Context {
return nil
}
func TestNewCrawlerPool(t *testing.T) {
dl := &mockDownloader{}
pool := NewCrawlerPool(dl)
if pool == nil {
t.Fatal("NewCrawlerPool returned nil")
}
}
func TestCrawlerPool_SetPipelineConfig(t *testing.T) {
pool := NewCrawlerPool(&mockDownloader{})
pool.SetPipelineConfig("csv", 100)
}
func TestCrawlerPool_Reset(t *testing.T) {
_ = config.Conf()
pool := NewCrawlerPool(&mockDownloader{})
pool.SetPipelineConfig("csv", 10)
tests := []struct {
name string
spiderNum int
wantMinNum int
}{
{"one", 1, 1},
{"five", 5, 5},
{"over_cap", 999, 1},
{"zero", 0, 1},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := pool.Reset(tt.spiderNum)
if got < tt.wantMinNum {
t.Errorf("Reset(%d) = %d, want >= %d", tt.spiderNum, got, tt.wantMinNum)
}
})
}
}
func TestCrawlerPool_Use_UseOpt_Free(t *testing.T) {
scheduler.Init(4, 0)
pool := NewCrawlerPool(downloader.SurferDownloader)
pool.SetPipelineConfig("csv", 10)
pool.Reset(2)
opt := pool.UseOpt()
if !opt.IsSome() {
t.Fatal("UseOpt returned None")
}
c := opt.Unwrap()
if c == nil {
t.Fatal("UseOpt returned nil crawler")
}
sp := &spider.Spider{
Name: "TestSpider",
RuleTree: &spider.RuleTree{Trunk: map[string]*spider.Rule{}},
Limit: -10,
}
c.Init(sp)
pool.Free(c)
c2 := pool.Use()
if c2 == nil {
t.Fatal("Use returned nil")
}
pool.Free(c2)
}
func TestCrawlerPool_UseOpt_returnsSome(t *testing.T) {
scheduler.Init(4, 0)
pool := NewCrawlerPool(&mockDownloader{})
pool.SetPipelineConfig("csv", 10)
pool.Reset(2)
opt := pool.UseOpt()
if !opt.IsSome() {
t.Fatal("UseOpt returned None")
}
c := opt.Unwrap()
if c.GetID() < 0 {
t.Errorf("GetID() = %d, want >= 0", c.GetID())
}
}
func TestCrawlerPool_Stop(t *testing.T) {
pool := NewCrawlerPool(&mockDownloader{})
pool.SetPipelineConfig("csv", 10)
pool.Reset(1)
pool.Stop()
opt := pool.UseOpt()
if opt.IsSome() {
t.Error("UseOpt after Stop should return None")
}
}
func TestCrawlerPool_Reset_reuse(t *testing.T) {
scheduler.Init(4, 0)
_ = config.Conf()
pool := NewCrawlerPool(&mockDownloader{})
pool.SetPipelineConfig("csv", 10)
pool.Reset(2)
c1 := pool.Use()
c2 := pool.Use()
sp := &spider.Spider{
Name: "TestSpider",
RuleTree: &spider.RuleTree{Trunk: map[string]*spider.Rule{}},
Limit: -5,
}
c1.Init(sp)
c2.Init(sp)
pool.Free(c1)
pool.Free(c2)
got := pool.Reset(3)
if got != 3 {
t.Errorf("Reset(3) = %d, want 3", got)
}
}
func TestCrawlerPool_Stop_idempotent(t *testing.T) {
pool := NewCrawlerPool(&mockDownloader{})
pool.Reset(1)
pool.Stop()
pool.Stop()
}
func TestCrawlerPool_Free_whenStopped(t *testing.T) {
scheduler.Init(4, 0)
pool := NewCrawlerPool(downloader.SurferDownloader)
pool.SetPipelineConfig("csv", 10)
pool.Reset(1)
c := pool.Use()
sp := &spider.Spider{
Name: "TestSpider",
RuleTree: &spider.RuleTree{Trunk: map[string]*spider.Rule{}},
Limit: -5,
}
c.Init(sp)
pool.Stop()
pool.Free(c)
}
================================================
FILE: app/crawler/spiderqueue.go
================================================
package crawler
import (
"github.com/andeya/gust/option"
spider "github.com/andeya/pholcus/app/spider"
"github.com/andeya/pholcus/common/util"
"github.com/andeya/pholcus/logs"
)
// SpiderQueue holds the spider rule queue for the crawler engine.
type (
SpiderQueue interface {
Reset() // Reset clears the queue
Add(*spider.Spider)
AddAll([]*spider.Spider)
AddKeyins(string) // AddKeyins assigns Keyin to queue members that have not been assigned yet
GetByIndex(int) *spider.Spider
GetByIndexOpt(int) option.Option[*spider.Spider]
GetByName(string) *spider.Spider
GetByNameOpt(string) option.Option[*spider.Spider]
GetAll() []*spider.Spider
Len() int // Len returns the queue length
}
sq struct {
list []*spider.Spider
}
)
// NewSpiderQueue creates a new spider queue.
func NewSpiderQueue() SpiderQueue {
return &sq{
list: []*spider.Spider{},
}
}
// Reset clears the spider queue.
func (sq *sq) Reset() {
sq.list = []*spider.Spider{}
}
// Add appends a spider to the queue.
func (sq *sq) Add(sp *spider.Spider) {
sp.SetID(sq.Len())
sq.list = append(sq.list, sp)
}
// AddAll appends all spiders in the list to the queue.
func (sq *sq) AddAll(list []*spider.Spider) {
for _, v := range list {
sq.Add(v)
}
}
// AddKeyins iterates over the spider queue and assigns Keyin values.
// Spiders that already have an explicit Keyin are not reassigned.
func (sq *sq) AddKeyins(keyins string) {
keyinSlice := util.KeyinsParse(keyins)
if len(keyinSlice) == 0 {
return
}
unit1 := []*spider.Spider{} // spiders that cannot receive custom config
unit2 := []*spider.Spider{} // spiders that can receive custom config
for _, v := range sq.GetAll() {
if v.GetKeyin() == spider.KEYIN {
unit2 = append(unit2, v)
continue
}
unit1 = append(unit1, v)
}
if len(unit2) == 0 {
logs.Log().Warning("This batch of tasks does not require custom configuration.\n")
return
}
sq.Reset()
for _, keyin := range keyinSlice {
for _, v := range unit2 {
v.Keyin = keyin
sq.Add(v.Copy())
}
}
if sq.Len() == 0 {
sq.AddAll(append(unit1, unit2...))
}
sq.AddAll(unit1)
}
// GetByIndex returns the spider at the given index.
func (sq *sq) GetByIndex(idx int) *spider.Spider {
return sq.GetByIndexOpt(idx).UnwrapOr(nil)
}
// GetByIndexOpt returns the spider at the given index as Option; None if out of range.
func (sq *sq) GetByIndexOpt(idx int) option.Option[*spider.Spider] {
if idx >= 0 && idx < len(sq.list) {
return option.Some(sq.list[idx])
}
return option.None[*spider.Spider]()
}
// GetByName returns the spider with the given name, or nil if not found.
func (sq *sq) GetByName(n string) *spider.Spider {
return sq.GetByNameOpt(n).UnwrapOr(nil)
}
// GetByNameOpt returns the spider with the given name as Option.
func (sq *sq) GetByNameOpt(n string) option.Option[*spider.Spider] {
for _, sp := range sq.list {
if sp.GetName() == n {
return option.Some(sp)
}
}
return option.None[*spider.Spider]()
}
// GetAll returns all spiders in the queue.
func (sq *sq) GetAll() []*spider.Spider {
return sq.list
}
// Len returns the number of spiders in the queue.
func (sq *sq) Len() int {
return len(sq.list)
}
================================================
FILE: app/crawler/spiderqueue_test.go
================================================
package crawler
import (
"testing"
spider "github.com/andeya/pholcus/app/spider"
)
func makeSpider(name string, keyin string) *spider.Spider {
return &spider.Spider{
Name: name,
Keyin: keyin,
RuleTree: &spider.RuleTree{
Trunk: map[string]*spider.Rule{},
},
}
}
func TestNewSpiderQueue(t *testing.T) {
q := NewSpiderQueue()
if q == nil {
t.Fatal("NewSpiderQueue returned nil")
}
if q.Len() != 0 {
t.Errorf("Len() = %d, want 0", q.Len())
}
}
func TestSpiderQueue_Add_Len_Reset(t *testing.T) {
tests := []struct {
name string
adds []*spider.Spider
wantLen int
}{
{"empty", nil, 0},
{"one", []*spider.Spider{makeSpider("a", "")}, 1},
{"two", []*spider.Spider{makeSpider("a", ""), makeSpider("b", "")}, 2},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
q := NewSpiderQueue()
for _, sp := range tt.adds {
q.Add(sp)
}
if got := q.Len(); got != tt.wantLen {
t.Errorf("Len() = %d, want %d", got, tt.wantLen)
}
q.Reset()
if q.Len() != 0 {
t.Errorf("after Reset Len() = %d, want 0", q.Len())
}
})
}
}
func TestSpiderQueue_AddAll(t *testing.T) {
list := []*spider.Spider{
makeSpider("a", ""),
makeSpider("b", ""),
makeSpider("c", ""),
}
q := NewSpiderQueue()
q.AddAll(list)
if got := q.Len(); got != 3 {
t.Errorf("AddAll Len() = %d, want 3", got)
}
all := q.GetAll()
for i := range list {
if all[i].GetName() != list[i].GetName() {
t.Errorf("GetAll()[%d].GetName() = %q, want %q", i, all[i].GetName(), list[i].GetName())
}
}
}
func TestSpiderQueue_GetByIndex_GetByIndexOpt(t *testing.T) {
sp1 := makeSpider("s1", "")
sp2 := makeSpider("s2", "")
q := NewSpiderQueue()
q.Add(sp1)
q.Add(sp2)
tests := []struct {
idx int
want *spider.Spider
optSome bool
}{
{0, sp1, true},
{1, sp2, true},
{-1, nil, false},
{2, nil, false},
{10, nil, false},
}
for _, tt := range tests {
got := q.GetByIndex(tt.idx)
if got != tt.want {
t.Errorf("GetByIndex(%d) = %v, want %v", tt.idx, got, tt.want)
}
opt := q.GetByIndexOpt(tt.idx)
if opt.IsSome() != tt.optSome {
t.Errorf("GetByIndexOpt(%d).IsSome() = %v, want %v", tt.idx, opt.IsSome(), tt.optSome)
}
if opt.IsSome() && opt.Unwrap() != tt.want {
t.Errorf("GetByIndexOpt(%d).Unwrap() = %v, want %v", tt.idx, opt.Unwrap(), tt.want)
}
}
}
func TestSpiderQueue_GetByName_GetByNameOpt(t *testing.T) {
sp1 := makeSpider("alpha", "")
sp2 := makeSpider("beta", "")
q := NewSpiderQueue()
q.Add(sp1)
q.Add(sp2)
tests := []struct {
name string
want *spider.Spider
optSome bool
}{
{"alpha", sp1, true},
{"beta", sp2, true},
{"nonexistent", nil, false},
{"", nil, false},
}
for _, tt := range tests {
got := q.GetByName(tt.name)
if got != tt.want {
t.Errorf("GetByName(%q) = %v, want %v", tt.name, got, tt.want)
}
opt := q.GetByNameOpt(tt.name)
if opt.IsSome() != tt.optSome {
t.Errorf("GetByNameOpt(%q).IsSome() = %v, want %v", tt.name, opt.IsSome(), tt.optSome)
}
if opt.IsSome() && opt.Unwrap() != tt.want {
t.Errorf("GetByNameOpt(%q).Unwrap() = %v, want %v", tt.name, opt.Unwrap(), tt.want)
}
}
}
func TestSpiderQueue_AddKeyins(t *testing.T) {
tests := []struct {
name string
spiders []*spider.Spider
keyins string
wantLen int
}{
{"empty_keyins", []*spider.Spider{makeSpider("a", "")}, "", 1},
{"no_keyin_spiders", []*spider.Spider{makeSpider("a", "x")}, "<k1><k2>", 1},
{"with_keyin_spiders", []*spider.Spider{makeSpider("a", spider.KEYIN)}, "<k1><k2>", 2},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
q := NewSpiderQueue()
for _, sp := range tt.spiders {
q.Add(sp)
}
q.AddKeyins(tt.keyins)
if got := q.Len(); got != tt.wantLen {
t.Errorf("AddKeyins Len() = %d, want %d", got, tt.wantLen)
}
})
}
}
func TestSpiderQueue_Add_setsID(t *testing.T) {
q := NewSpiderQueue()
sp1 := makeSpider("a", "")
sp2 := makeSpider("b", "")
q.Add(sp1)
q.Add(sp2)
if sp1.GetID() != 0 {
t.Errorf("first Add ID = %d, want 0", sp1.GetID())
}
if sp2.GetID() != 1 {
t.Errorf("second Add ID = %d, want 1", sp2.GetID())
}
}
func TestSpiderQueue_GetByIndexOpt(t *testing.T) {
q := NewSpiderQueue()
opt := q.GetByIndexOpt(0)
if opt.IsSome() {
t.Error("GetByIndexOpt(0) on empty queue should be None")
}
}
func TestSpiderQueue_AddKeyins_emptyUnit2(t *testing.T) {
q := NewSpiderQueue()
q.Add(makeSpider("a", "fixed"))
q.AddKeyins("<x><y>")
if q.Len() != 1 {
t.Errorf("AddKeyins with no KEYIN spiders Len() = %d, want 1", q.Len())
}
}
================================================
FILE: app/distribute/integration_test.go
================================================
package distribute
import (
"net"
"strconv"
"sync"
"testing"
"time"
"github.com/andeya/pholcus/app/distribute/teleport"
)
func freePort(t *testing.T) string {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("freePort: %v", err)
}
defer l.Close()
return strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
}
func TestTP_ServerClient_Request(t *testing.T) {
if testing.Short() {
t.Skip("skipping network test in short mode")
}
port := freePort(t)
portStr := ":" + port
tj := NewTaskJar()
tj.Push(&Task{ID: 0, Limit: 100})
serverTP := teleport.New().SetUID("server").SetAPI(MasterAPI(tj)).SetTimeout(100 * time.Millisecond)
serverTP.Server(portStr)
time.Sleep(50 * time.Millisecond)
clientTP := teleport.New().SetUID("client").SetAPI(SlaveAPI(NewTaskJar())).SetTimeout(100 * time.Millisecond)
clientTP.Client("127.0.0.1", portStr)
time.Sleep(100 * time.Millisecond)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
clientTP.Request("", "task", "", "")
}()
time.Sleep(200 * time.Millisecond)
serverTP.Close()
clientTP.Close()
wg.Wait()
}
================================================
FILE: app/distribute/interface.go
================================================
package distribute
// Distributor is the distributed interface.
type Distributor interface {
// Send sends a task from the master to the jar.
Send(clientNum int) Task
// Receive receives a task into the jar on a slave node.
Receive(task *Task)
// CountNodes returns the number of connected nodes.
CountNodes() int
}
================================================
FILE: app/distribute/master_api.go
================================================
package distribute
import (
"encoding/json"
"github.com/andeya/gust/result"
"github.com/andeya/pholcus/app/distribute/teleport"
"github.com/andeya/pholcus/logs"
)
// MasterAPI creates the master node API.
func MasterAPI(n Distributor) teleport.API {
return teleport.API{
"task": &masterTaskHandle{n},
"log": &masterLogHandle{},
}
}
// masterTaskHandle assigns tasks to clients.
type masterTaskHandle struct {
Distributor
}
func (mth *masterTaskHandle) Process(receive *teleport.NetData) *teleport.NetData {
b := result.Ret(json.Marshal(mth.Send(mth.CountNodes())))
if b.IsErr() {
return teleport.ReturnError(receive, teleport.FAILURE, "marshal error: "+b.UnwrapErr().Error(), receive.From)
}
return teleport.ReturnData(string(b.Unwrap()))
}
// masterLogHandle receives and prints log messages from slave nodes.
type masterLogHandle struct{}
func (*masterLogHandle) Process(receive *teleport.NetData) *teleport.NetData {
logs.Log().Informational(" * ")
logs.Log().Informational(" * [ %s ] %s", receive.From, receive.Body)
logs.Log().Informational(" * ")
return nil
}
================================================
FILE: app/distribute/master_api_test.go
================================================
package distribute
import (
"encoding/json"
"testing"
"github.com/andeya/pholcus/app/distribute/teleport"
)
type mockDistributor struct {
sendCount int
countNodes int
sendTask Task
receiveTask *Task
}
func (m *mockDistributor) Send(clientNum int) Task {
m.sendCount++
return m.sendTask
}
func (m *mockDistributor) Receive(task *Task) {
m.receiveTask = task
}
func (m *mockDistributor) CountNodes() int {
return m.countNodes
}
func TestMasterAPI(t *testing.T) {
d := &mockDistributor{countNodes: 2, sendTask: Task{ID: 1, Limit: 100}}
api := MasterAPI(d)
if api == nil {
t.Fatal("MasterAPI returned nil")
}
if _, ok := api["task"]; !ok {
t.Error("API missing task handler")
}
if _, ok := api["log"]; !ok {
t.Error("API missing log handler")
}
}
func TestMasterTaskHandle_Process(t *testing.T) {
task := Task{ID: 1, Limit: 50, OutType: "mgo"}
d := &mockDistributor{countNodes: 1, sendTask: task}
handle := &masterTaskHandle{d}
req := &teleport.NetData{From: "client1", To: "server", Operation: "task", Body: ""}
resp := handle.Process(req)
if resp == nil {
t.Fatal("Process returned nil")
}
if resp.Status != teleport.SUCCESS {
t.Errorf("Status = %d, want SUCCESS", resp.Status)
}
bodyStr, ok := resp.Body.(string)
if !ok {
t.Fatalf("Body type = %T, want string", resp.Body)
}
var got Task
if err := json.Unmarshal([]byte(bodyStr), &got); err != nil {
t.Fatalf("json.Unmarshal: %v", err)
}
if got.ID != task.ID || got.Limit != task.Limit {
t.Errorf("got Task %+v, want %+v", got, task)
}
}
func TestMasterLogHandle_Process(t *testing.T) {
handle := &masterLogHandle{}
req := &teleport.NetData{From: "slave1", Body: "test log message"}
resp := handle.Process(req)
if resp != nil {
t.Errorf("Process returned %v, want nil", resp)
}
}
================================================
FILE: app/distribute/slave_api.go
================================================
package distribute
import (
"encoding/json"
"github.com/andeya/gust/result"
"github.com/andeya/pholcus/app/distribute/teleport"
"github.com/andeya/pholcus/logs"
)
// SlaveAPI creates the slave node API.
func SlaveAPI(n Distributor) teleport.API {
return teleport.API{
"task": &slaveTaskHandle{n},
}
}
// slaveTaskHandle receives tasks from the master and adds them to the task jar.
type slaveTaskHandle struct {
Distributor
}
func (sth *slaveTaskHandle) Process(receive *teleport.NetData) *teleport.NetData {
t := &Task{}
if r := result.RetVoid(json.Unmarshal([]byte(receive.Body.(string)), t)); r.IsErr() {
logs.Log().Error("JSON decode failed: %v", receive.Body)
return nil
}
sth.Receive(t)
return nil
}
================================================
FILE: app/distribute/slave_api_test.go
================================================
package distribute
import (
"encoding/json"
"testing"
"github.com/andeya/pholcus/app/distribute/teleport"
)
func TestSlaveAPI(t *testing.T) {
tj := NewTaskJar()
api := SlaveAPI(tj)
if api == nil {
t.Fatal("SlaveAPI returned nil")
}
if _, ok := api["task"]; !ok {
t.Error("API missing task handler")
}
}
func TestSlaveTaskHandle_Process(t *testing.T) {
tj := NewTaskJar()
task := Task{ID: 2, Limit: 200, OutType: "csv"}
body, _ := json.Marshal(task)
handle := &slaveTaskHandle{tj}
req := &teleport.NetData{From: "master", Body: string(body)}
resp := handle.Process(req)
if resp != nil {
t.Errorf("Process returned %v, want nil", resp)
}
got := tj.Pull()
if got.ID != task.ID || got.Limit != task.Limit {
t.Errorf("got Task %+v, want %+v", got, task)
}
}
func TestSlaveTaskHandle_Process_InvalidJSON(t *testing.T) {
tj := NewTaskJar()
handle := &slaveTaskHandle{tj}
req := &teleport.NetData{From: "master", Body: "invalid json {"}
resp := handle.Process(req)
if resp != nil {
t.Errorf("Process returned %v, want nil", resp)
}
if tj.Len() != 0 {
t.Errorf("Len() = %d, want 0", tj.Len())
}
}
================================================
FILE: app/distribute/task.go
================================================
// Package distribute provides distributed task scheduling and master-slave node communication.
package distribute
// Task is used for distributed task dispatch.
type Task struct {
ID int
Spiders []map[string]string // Spider rule name and keyin, format: map[string]string{"name":"baidu","keyin":"henry"}
ThreadNum int // Global max concurrency
Pausetime int64 // Pause duration in ms (random: Pausetime/2 ~ Pausetime*2)
OutType string // Output method
BatchCap int // Batch output capacity per flush
BatchQueueCap int // Batch output pool capacity, >= 2
SuccessInherit bool // Inherit historical success records
FailureInherit bool // Inherit historical failure records
Limit int64 // Collection limit, 0=unlimited; if rule sets LIMIT then custom limit
ProxyMinute int64 // Proxy IP rotation interval in minutes
Keyins string // Custom input, later split into Keyin config for multiple tasks
}
================================================
FILE: app/distribute/task_test.go
================================================
package distribute
import (
"testing"
)
func TestTask_Fields(t *testing.T) {
tests := []struct {
name string
task Task
wantID int
wantLimit int64
wantOutType string
}{
{"zero", Task{}, 0, 0, ""},
{"with_values", Task{
ID: 1,
Limit: 100,
OutType: "mgo",
ThreadNum: 10,
}, 1, 100, "mgo"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.task.ID != tt.wantID {
t.Errorf("ID = %d, want %d", tt.task.ID, tt.wantID)
}
if tt.task.Limit != tt.wantLimit {
t.Errorf("Limit = %d, want %d", tt.task.Limit, tt.wantLimit)
}
if tt.task.OutType != tt.wantOutType {
t.Errorf("OutType = %q, want %q", tt.task.OutType, tt.wantOutType)
}
})
}
}
================================================
FILE: app/distribute/taskjar.go
================================================
package distribute
// TaskJar is the task storage.
type TaskJar struct {
Tasks chan *Task
}
// NewTaskJar creates a task storage instance.
func NewTaskJar() *TaskJar {
return &TaskJar{
Tasks: make(chan *Task, 1024),
}
}
// Push adds a task to the jar (server side).
func (tj *TaskJar) Push(task *Task) {
id := len(tj.Tasks)
task.ID = id
tj.Tasks <- task
}
// Pull gets a task from the local jar (client side).
func (tj *TaskJar) Pull() *Task {
return <-tj.Tasks
}
// Len returns number of tasks in the jar.
func (tj *TaskJar) Len() int {
return len(tj.Tasks)
}
// Send sends a task from the jar (master side).
func (tj *TaskJar) Send(clientNum int) Task {
return *<-tj.Tasks
}
// Receive receives a task into the jar (slave side).
func (tj *TaskJar) Receive(task *Task) {
tj.Tasks <- task
}
// CountNodes returns 0; TaskJar does not track connected nodes.
func (tj *TaskJar) CountNodes() int {
return 0
}
================================================
FILE: app/distribute/taskjar_test.go
================================================
package distribute
import (
"sync"
"testing"
)
func TestNewTaskJar(t *testing.T) {
tj := NewTaskJar()
if tj == nil {
t.Fatal("NewTaskJar() returned nil")
}
if tj.Tasks == nil {
t.Fatal("Tasks channel is nil")
}
if cap(tj.Tasks) != 1024 {
t.Errorf("cap(Tasks) = %d, want 1024", cap(tj.Tasks))
}
}
func TestTaskJar_PushPull(t *testing.T) {
tj := NewTaskJar()
task := &Task{ID: 0, Limit: 10}
tj.Push(task)
if tj.Len() != 1 {
t.Errorf("Len() = %d, want 1", tj.Len())
}
got := tj.Pull()
if got != task {
t.Errorf("Pull() = %p, want %p", got, task)
}
if tj.Len() != 0 {
t.Errorf("Len() after Pull = %d, want 0", tj.Len())
}
}
func TestTaskJar_SendReceive(t *testing.T) {
tests := []struct {
name string
clientNum int
}{
{"single", 1},
{"multi", 3},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tj := NewTaskJar()
task := &Task{ID: 0, Limit: 5}
tj.Receive(task)
if tj.Len() != 1 {
t.Errorf("Len() = %d, want 1", tj.Len())
}
got := tj.Send(tt.clientNum)
if got.Limit != 5 {
t.Errorf("Send() Limit = %d, want 5", got.Limit)
}
})
}
}
func TestTaskJar_PushAssignsID(t *testing.T) {
tj := NewTaskJar()
t1 := &Task{Limit: 1}
t2 := &Task{Limit: 2}
tj.Push(t1)
tj.Push(t2)
got1 := tj.Pull()
got2 := tj.Pull()
if got1.ID != 0 {
t.Errorf("first task ID = %d, want 0", got1.ID)
}
if got2.ID != 1 {
t.Errorf("second task ID = %d, want 1", got2.ID)
}
}
func TestTaskJar_CountNodes(t *testing.T) {
tj := NewTaskJar()
if got := tj.CountNodes(); got != 0 {
t.Errorf("CountNodes() = %d, want 0", got)
}
}
func TestTaskJar_Concurrent(t *testing.T) {
tj := NewTaskJar()
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
tj.Push(&Task{ID: id, Limit: int64(id)})
}(i)
}
wg.Wait()
if tj.Len() != 10 {
t.Errorf("Len() = %d, want 10", tj.Len())
}
for tj.Len() > 0 {
_ = tj.Pull()
}
}
================================================
FILE: app/distribute/teleport/client.go
================================================
package teleport
import (
"log"
"net"
"time"
"github.com/andeya/gust/result"
)
// tpClient holds client-only state.
type tpClient struct {
short bool
mustClose bool
serverUID string
}
// Client starts client mode.
func (tp *TP) Client(serverAddr string, port string, isShort ...bool) {
if len(isShort) > 0 && isShort[0] {
tp.tpClient.short = true
} else if tp.timeout == 0 {
tp.timeout = DEFAULT_TIMEOUT_C
}
if tp.tpClient.serverUID == "" {
tp.tpClient.serverUID = DEFAULT_SERVER_UID
}
tp.reserveAPI()
tp.mode = CLIENT
if port != "" {
tp.port = port
} else {
tp.port = DEFAULT_PORT
}
tp.serverAddr = serverAddr
tp.tpClient.mustClose = false
go tp.apiHandle()
go tp.client()
}
// --- Client implementation ---
func (tp *TP) client() {
if !tp.short {
log.Println(" * -- Connecting to server... --")
}
RetryLabel:
connRes := result.Ret(net.Dial("tcp", tp.serverAddr+tp.port))
if connRes.IsErr() {
if tp.tpClient.mustClose {
tp.tpClient.mustClose = false
return
}
time.Sleep(LOOP_TIMEOUT)
goto RetryLabel
}
conn := connRes.Unwrap()
debugPrintf("Debug: connected to server: %v", conn.RemoteAddr().String())
tp.cGoConn(conn)
if !tp.short {
for tp.CountNodes() > 0 {
time.Sleep(LOOP_TIMEOUT)
}
if _, ok := tp.connPool[tp.tpClient.serverUID]; ok {
goto RetryLabel
}
}
}
// cGoConn starts read/write goroutines for the connection.
func (tp *TP) cGoConn(conn net.Conn) {
remoteAddr, connect := NewConnect(conn, tp.connBufferLen, tp.connWChanCap)
tp.connPool[tp.tpClient.serverUID] = connect
if tp.uid == "" {
tp.uid = conn.LocalAddr().String()
}
if !tp.short {
tp.send(NewNetData(tp.uid, tp.tpClient.serverUID, IDENTITY, "", nil))
log.Printf(" * -- Connected to server: %v --", remoteAddr)
} else {
connect.Short = true
}
tp.connPool[tp.tpClient.serverUID].Usable = true
go tp.cReader(tp.tpClient.serverUID)
go tp.cWriter(tp.tpClient.serverUID)
}
// cReader reads data on the client side.
func (tp *TP) cReader(nodeuid string) {
defer func() {
tp.closeConn(nodeuid, true)
}()
var conn = tp.getConn(nodeuid)
for {
if !tp.read(conn) {
break
}
}
}
// cWriter sends data on the client side.
func (tp *TP) cWriter(nodeuid string) {
defer func() {
tp.closeConn(nodeuid, true)
}()
var conn = tp.getConn(nodeuid)
for conn != nil {
if tp.short {
tp.send(<-conn.WriteChan)
continue
}
timing := time.After(tp.timeout)
data := new(NetData)
select {
case data = <-conn.WriteChan:
case <-timing:
data = NewNetData(tp.uid, nodeuid, HEARTBEAT, "", nil)
}
tp.send(data)
}
}
================================================
FILE: app/distribute/teleport/conn.go
================================================
package teleport
import (
"net"
)
// Connect wraps a network connection.
type Connect struct {
net.Conn
Usable bool
Short bool
WriteChan chan *NetData
Buffer []byte
TmpBuffer []byte
}
// NewConnect creates a Connect instance; defaults to long connection (Short=false).
func NewConnect(conn net.Conn, bufferLen int, wChanCap int) (k string, v *Connect) {
k = conn.RemoteAddr().String()
v = &Connect{
WriteChan: make(chan *NetData, wChanCap),
Buffer: make([]byte, bufferLen),
TmpBuffer: make([]byte, 0),
Conn: conn,
}
return k, v
}
// Addr returns the remote node address.
func (conn *Connect) Addr() string {
return conn.Conn.RemoteAddr().String()
}
================================================
FILE: app/distribute/teleport/conn_test.go
================================================
package teleport
import (
"net"
"testing"
)
func TestNewConnect(t *testing.T) {
client, server := net.Pipe()
defer client.Close()
defer server.Close()
k, v := NewConnect(client, 1024, 256)
if k != client.RemoteAddr().String() {
t.Errorf("key = %q, want %q", k, client.RemoteAddr().String())
}
if v == nil {
t.Fatal("Connect is nil")
}
if v.WriteChan == nil {
t.Error("WriteChan is nil")
}
if len(v.Buffer) != 1024 {
t.Errorf("Buffer len = %d, want 1024", len(v.Buffer))
}
if cap(v.WriteChan) != 256 {
t.Errorf("WriteChan cap = %d, want 256", cap(v.WriteChan))
}
}
func TestConnect_Addr(t *testing.T) {
client, server := net.Pipe()
defer client.Close()
defer server.Close()
_, conn := NewConnect(client, 64, 16)
addr := conn.Addr()
want := client.RemoteAddr().String()
if addr != want {
t.Errorf("Addr() = %q, want %q", addr, want)
}
}
================================================
FILE: app/distribute/teleport/debug.go
================================================
package teleport
import (
"log"
)
var Debug bool
func debugPrintf(format string, v ...interface{}) {
if !Debug {
return
}
log.Printf(format, v...)
}
func debugPrintln(v ...interface{}) {
if !Debug {
return
}
log.Println(v...)
}
func debugFatal(v ...interface{}) {
if !Debug {
return
}
log.Fatal(v...)
}
================================================
FILE: app/distribute/teleport/netdata.go
================================================
package teleport
const (
SUCCESS = 0
FAILURE = -1
LLLEGAL = -2
)
// NetData is the data transfer structure.
type NetData struct {
Body interface{}
Operation string
From string
To string
Status int
Flag string
}
// NewNetData creates a network data transfer structure.
func NewNetData(from, to, operation string, flag string, body interface{}) *NetData {
return &NetData{
From: from,
To: to,
Body: body,
Operation: operation,
Status: SUCCESS,
Flag: flag,
}
}
================================================
FILE: app/distribute/teleport/netdata_test.go
================================================
package teleport
import (
"testing"
)
func TestNewNetData(t *testing.T) {
tests := []struct {
from, to, op, flag string
body interface{}
}{
{"a", "b", "task", "", "body"},
{"", "", "heartbeat", "f", nil},
}
for i, tt := range tests {
t.Run("", func(t *testing.T) {
d := NewNetData(tt.from, tt.to, tt.op, tt.flag, tt.body)
if d == nil {
t.Fatal("NewNetData returned nil")
}
if d.From != tt.from {
t.Errorf("From = %q, want %q", d.From, tt.from)
}
if d.To != tt.to {
t.Errorf("To = %q, want %q", d.To, tt.to)
}
if d.Operation != tt.op {
t.Errorf("Operation = %q, want %q", d.Operation, tt.op)
}
if d.Flag != tt.flag {
t.Errorf("Flag = %q, want %q", d.Flag, tt.flag)
}
if d.Status != SUCCESS {
t.Errorf("Status = %d, want SUCCESS", d.Status)
}
_ = i
})
}
}
================================================
FILE: app/distribute/teleport/protocol.go
================================================
package teleport
import (
"bytes"
"encoding/binary"
)
const (
DataLengthOfLenth = 4
)
// Protocol handles packet framing (pack/unpack).
type Protocol struct {
header string
headerLen int
}
// NewProtocol creates a protocol instance; packetHeader is the packet header identifier.
func NewProtocol(packetHeader string) *Protocol {
return &Protocol{
header: packetHeader,
headerLen: len([]byte(packetHeader)),
}
}
func (p *Protocol) ReSet(header string) {
p.header = header
p.headerLen = len([]byte(header))
}
// Packet frames a message for transmission.
func (p *Protocol) Packet(message []byte) []byte {
return append(append([]byte(p.header), IntToBytes(len(message))...), message...)
}
// Unpack extracts messages from the buffer.
func (p *Protocol) Unpack(buffer []byte) (readerSlice [][]byte, bufferOver []byte) {
length := len(buffer)
var i int
for i = 0; i < length; i = i + 1 {
if length < i+p.headerLen+DataLengthOfLenth {
break
}
if string(buffer[i:i+p.headerLen]) == p.header {
messageLength := BytesToInt(buffer[i+p.headerLen : i+p.headerLen+DataLengthOfLenth])
if length < i+p.headerLen+DataLengthOfLenth+messageLength {
break
}
data := buffer[i+p.headerLen+DataLengthOfLenth : i+p.headerLen+DataLengthOfLenth+messageLength]
readerSlice = append(readerSlice, data)
i += p.headerLen + DataLengthOfLenth + messageLength - 1
}
}
if i == length {
bufferOver = make([]byte, 0)
return
}
bufferOver = buffer[i:]
return
}
// IntToBytes converts int to bytes.
func IntToBytes(n int) []byte {
x := int32(n)
bytesBuffer := bytes.NewBuffer([]byte{})
binary.Write(bytesBuffer, binary.LittleEndian, x)
return bytesBuffer.Bytes()
}
// BytesToInt converts bytes to int.
func BytesToInt(b []byte) int {
bytesBuffer := bytes.NewBuffer(b)
var x int32
binary.Read(bytesBuffer, binary.LittleEndian, &x)
return int(x)
}
================================================
FILE: app/distribute/teleport/protocol_test.go
================================================
package teleport
import (
"bytes"
"testing"
)
func TestNewProtocol(t *testing.T) {
tests := []struct {
header string
}{
{""},
{"andeya"},
{"custom-header"},
}
for _, tt := range tests {
t.Run(tt.header, func(t *testing.T) {
p := NewProtocol(tt.header)
if p == nil {
t.Fatal("NewProtocol returned nil")
}
if p.header != tt.header {
t.Errorf("header = %q, want %q", p.header, tt.header)
}
wantLen := len([]byte(tt.header))
if p.headerLen != wantLen {
t.Errorf("headerLen = %d, want %d", p.headerLen, wantLen)
}
})
}
}
func TestProtocol_ReSet(t *testing.T) {
p := NewProtocol("old")
p.ReSet("new")
if p.header != "new" {
t.Errorf("header = %q, want new", p.header)
}
if p.headerLen != 3 {
t.Errorf("headerLen = %d, want 3", p.headerLen)
}
}
func TestProtocol_Packet(t *testing.T) {
p := NewProtocol("andeya")
msg := []byte("hello")
got := p.Packet(msg)
want := append(append([]byte("andeya"), IntToBytes(len(msg))...), msg...)
if !bytes.Equal(got, want) {
t.Errorf("Packet() = %v, want %v", got, want)
}
}
func TestProtocol_Unpack(t *testing.T) {
p := NewProtocol("andeya")
msg := []byte("hello")
packed := p.Packet(msg)
slice, rest := p.Unpack(packed)
if len(slice) != 1 {
t.Fatalf("len(slice) = %d, want 1", len(slice))
}
if !bytes.Equal(slice[0], msg) {
t.Errorf("Unpack()[0] = %v, want %v", slice[0], msg)
}
if len(rest) != 0 {
t.Errorf("rest = %v, want empty", rest)
}
}
func TestProtocol_Unpack_Multiple(t *testing.T) {
p := NewProtocol("ab")
m1 := []byte("x")
m2 := []byte("yz")
packed := append(p.Packet(m1), p.Packet(m2)...)
slice, rest := p.Unpack(packed)
if len(slice) != 2 {
t.Fatalf("len(slice) = %d, want 2", len(slice))
}
if !bytes.Equal(slice[0], m1) {
t.Errorf("slice[0] = %v, want %v", slice[0], m1)
}
if !bytes.Equal(slice[1], m2) {
t.Errorf("slice[1] = %v, want %v", slice[1], m2)
}
if len(rest) != 0 {
t.Errorf("rest len = %d, want 0", len(rest))
}
}
func TestProtocol_Unpack_Partial(t *testing.T) {
p := NewProtocol("ab")
msg := []byte("full")
packed := p.Packet(msg)
partial := packed[:len(packed)-2]
slice, rest := p.Unpack(partial)
if len(slice) != 0 {
t.Errorf("len(slice) = %d, want 0", len(slice))
}
if !bytes.Equal(rest, partial) {
t.Errorf("rest = %v, want %v", rest, partial)
}
}
func TestProtocol_Unpack_GarbageBeforeHeader(t *testing.T) {
p := NewProtocol("ab")
msg := []byte("x")
packed := p.Packet(msg)
buf := append([]byte("xx"), packed...)
slice, rest := p.Unpack(buf)
if len(slice) != 1 {
t.Fatalf("len(slice) = %d, want 1", len(slice))
}
if !bytes.Equal(slice[0], msg) {
t.Errorf("slice[0] = %v, want %v", slice[0], msg)
}
if len(rest) != 0 {
t.Errorf("rest len = %d, want 0", len(rest))
}
}
func TestProtocol_Unpack_EmptyBuffer(t *testing.T) {
p := NewProtocol("ab")
slice, rest := p.Unpack([]byte{})
if len(slice) != 0 {
t.Errorf("len(slice) = %d, want 0", len(slice))
}
if len(rest) != 0 {
t.Errorf("rest = %v, want empty", rest)
}
}
func TestProtocol_Unpack_TooShort(t *testing.T) {
p := NewProtocol("andeya")
slice, rest := p.Unpack([]byte("and"))
if len(slice) != 0 {
t.Errorf("len(slice) = %d, want 0", len(slice))
}
if !bytes.Equal(rest, []byte("and")) {
t.Errorf("rest = %v", rest)
}
}
func TestIntToBytes_BytesToInt(t *testing.T) {
tests := []int{0, 1, 42, 1024, -1}
for _, n := range tests {
b := IntToBytes(n)
got := BytesToInt(b)
if got != n {
t.Errorf("BytesToInt(IntToBytes(%d)) = %d", n, got)
}
}
}
================================================
FILE: app/distribute/teleport/return_func.go
================================================
package teleport
// ReturnData builds an API response. If OpAndToAndFrom[0] is empty, use peer operation; if [1] is empty, peer is receiver; if [2] is empty, self is sender.
func ReturnData(body interface{}, OpAndToAndFrom ...string) *NetData {
data := &NetData{
Status: SUCCESS,
Body: body,
}
if len(OpAndToAndFrom) > 0 {
data.Operation = OpAndToAndFrom[0]
}
if len(OpAndToAndFrom) > 1 {
data.To = OpAndToAndFrom[1]
}
if len(OpAndToAndFrom) > 2 {
data.From = OpAndToAndFrom[2]
}
return data
}
// ReturnError returns an error response; receive should be the original *NetData.
func ReturnError(receive *NetData, status int, msg string, nodeuid ...string) *NetData {
receive.Status = status
receive.Body = msg
receive.From = ""
if len(nodeuid) > 0 {
receive.To = nodeuid[0]
} else {
receive.To = ""
}
return receive
}
================================================
FILE: app/distribute/teleport/return_func_test.go
================================================
package teleport
import (
"testing"
)
func TestReturnData(t *testing.T) {
tests := []struct {
body string
args []string
wantOp, wantTo, wantFrom string
}{
{"ok", nil, "", "", ""},
{"x", []string{"op1"}, "op1", "", ""},
{"y", []string{"op2", "to2"}, "op2", "to2", ""},
{"z", []string{"op3", "to3", "from3"}, "op3", "to3", "from3"},
}
for _, tt := range tests {
t.Run(tt.body, func(t *testing.T) {
var d *NetData
if len(tt.args) == 0 {
d = ReturnData(tt.body)
} else {
d = ReturnData(tt.body, tt.args...)
}
if d == nil {
t.Fatal("ReturnData returned nil")
}
if d.Status != SUCCESS {
t.Errorf("Status = %d, want SUCCESS", d.Status)
}
if d.Body != tt.body {
t.Errorf("Body = %v, want %v", d.Body, tt.body)
}
if d.Operation != tt.wantOp {
t.Errorf("Operation = %q, want %q", d.Operation, tt.wantOp)
}
if d.To != tt.wantTo {
t.Errorf("To = %q, want %q", d.To, tt.wantTo)
}
if d.From != tt.wantFrom {
t.Errorf("From = %q, want %q", d.From, tt.wantFrom)
}
})
}
}
func TestReturnError(t *testing.T) {
req := &NetData{From: "a", To: "b", Operation: "task", Body: "orig"}
resp := ReturnError(req, FAILURE, "err msg", "target")
if resp != req {
t.Error("ReturnError should return same pointer")
}
if req.Status != FAILURE {
t.Errorf("Status = %d, want FAILURE", req.Status)
}
if req.Body != "err msg" {
t.Errorf("Body = %q, want err msg", req.Body)
}
if req.From != "" {
t.Errorf("From = %q, want empty", req.From)
}
if req.To != "target" {
t.Errorf("To = %q, want target", req.To)
}
}
func TestReturnError_NoNodeUID(t *testing.T) {
req := &NetData{To: "x"}
ReturnError(req, LLLEGAL, "bad")
if req.To != "" {
t.Errorf("To = %q, want empty", req.To)
}
}
================================================
FILE: app/distribute/teleport/server.go
================================================
package teleport
import (
"encoding/json"
"log"
"net"
"time"
"github.com/andeya/gust/result"
)
// tpServer holds server-only state.
type tpServer struct {
listener net.Listener
}
// Server starts server mode; port defaults to DEFAULT_PORT.
func (tp *TP) Server(port ...string) {
tp.reserveAPI()
tp.mode = SERVER
if len(port) > 0 {
tp.port = port[0]
} else {
tp.port = DEFAULT_PORT
}
if tp.uid == "" {
tp.uid = DEFAULT_SERVER_UID
}
if tp.timeout == 0 {
tp.timeout = DEFAULT_TIMEOUT_S
}
go tp.apiHandle()
go tp.server()
}
// --- Server implementation ---
func (tp *TP) server() {
retry:
listenerRes := result.Ret(net.Listen("tcp", tp.port))
if listenerRes.IsErr() {
debugPrintf("Debug: listen port error: %v", listenerRes.UnwrapErr())
time.Sleep(LOOP_TIMEOUT)
goto retry
}
tp.listener = listenerRes.Unwrap()
log.Printf(" * -- Server listening (port %v) --", tp.port)
for tp.listener != nil {
connRes := result.Ret(tp.listener.Accept())
if connRes.IsErr() {
return
}
conn := connRes.Unwrap()
debugPrintf("Debug: client %v connected, identity not yet verified", conn.RemoteAddr().String())
tp.sGoConn(conn)
}
}
// sGoConn starts read/write goroutines for each connection.
func (tp *TP) sGoConn(conn net.Conn) {
remoteAddr, connect := NewConnect(conn, tp.connBufferLen, tp.connWChanCap)
nodeuid, ok := tp.sInitConn(connect, remoteAddr)
if !ok {
conn.Close()
return
}
go tp.sReader(nodeuid)
go tp.sWriter(nodeuid)
}
// sInitConn initializes connection and binds node to conn; default key is node IP.
func (tp *TP) sInitConn(conn *Connect, remoteAddr string) (nodeuid string, usable bool) {
readLen, err := conn.Read(conn.Buffer)
if result.TryErrVoid(err).IsErr() || readLen == 0 {
return
}
conn.TmpBuffer = append(conn.TmpBuffer, conn.Buffer[:readLen]...)
dataSlice := make([][]byte, 10)
dataSlice, conn.TmpBuffer = tp.Unpack(conn.TmpBuffer)
for i, data := range dataSlice {
debugPrintln("Debug: received data batch 1 before decode: ", string(data))
d := new(NetData)
if result.RetVoid(json.Unmarshal(data, d)).IsErr() {
if i == 0 {
return
}
continue
}
if d.From == "" {
d.From = remoteAddr
}
if i == 0 {
debugPrintf("Debug: received data item 1 NetData: %+v", d)
if !tp.checkRights(d, remoteAddr) {
return
}
nodeuid = d.From
tp.connPool[nodeuid] = conn
if d.Operation != IDENTITY {
conn.Short = true
} else {
log.Printf(" * -- Client %v (%v) connected --", nodeuid, remoteAddr)
}
conn.Usable = true
}
tp.apiReadChan <- d
}
return nodeuid, true
}
// sReader reads data on the server side.
func (tp *TP) sReader(nodeuid string) {
defer func() {
tp.closeConn(nodeuid, false)
}()
var conn = tp.getConn(nodeuid)
for conn != nil {
if !conn.Short {
conn.SetReadDeadline(time.Now().Add(tp.timeout))
}
if !tp.read(conn) {
return
}
}
}
// sWriter sends data on the server side.
func (tp *TP) sWriter(nodeuid string) {
defer func() {
tp.closeConn(nodeuid, false)
}()
var conn = tp.getConn(nodeuid)
for conn != nil {
data := <-conn.WriteChan
tp.send(data)
if conn.Short {
return
}
}
}
================================================
FILE: app/distribute/teleport/teleport.go
================================================
// Package teleport provides a high-concurrency API framework for distributed systems.
// It uses socket duplex communication for peer-to-peer S/C, supports long and short connections,
// auto-reconnect after disconnect, and JSON for data transport.
package teleport
import (
"encoding/json"
"log"
"time"
"github.com/andeya/gust/result"
)
// Run mode constants.
const (
SERVER = iota + 1
CLIENT
)
// Reserved operation names for API handlers.
const (
IDENTITY = "+identity+"
HEARTBEAT = "+heartbeat+"
DEFAULT_PACK_HEADER = "andeya"
DEFAULT_SERVER_UID = "server"
DEFAULT_PORT = ":8080"
DEFAULT_TIMEOUT_S = 20e9
DEFAULT_TIMEOUT_C = 15e9
LOOP_TIMEOUT = 1e9
)
type Teleport interface {
Server(port ...string)
Client(serverAddr string, port string, isShort ...bool)
Request(body interface{}, operation string, flag string, nodeuid ...string)
SetAPI(api API) Teleport
Close(nodeuid ...string)
SetUID(mine string, server ...string) Teleport
SetPackHeader(string) Teleport
SetApiRChan(int) Teleport
SetConnWChan(int) Teleport
SetConnBuffer(int) Teleport
SetTimeout(time.Duration) Teleport
GetMode() int
CountNodes() int
}
type TP struct {
uid string
mode int
port string
serverAddr string
connPool map[string]*Connect
timeout time.Duration
*Protocol
apiReadChan chan *NetData
connWChanCap int
connBufferLen int
api API
*tpServer
*tpClient
}
type API map[string]Handle
// Handle processes requests.
type Handle interface {
Process(*NetData) *NetData
}
// New creates a Teleport instance.
func New() Teleport {
return &TP{
connPool: make(map[string]*Connect),
api: API{},
Protocol: NewProtocol(DEFAULT_PACK_HEADER),
apiReadChan: make(chan *NetData, 4096),
connWChanCap: 2048,
connBufferLen: 1024,
tpServer: new(tpServer),
tpClient: new(tpClient),
}
}
// --- Interface implementation ---
func (tp *TP) SetUID(mine string, server ...string) Teleport {
if len(server) > 0 {
tp.tpClient.serverUID = server[0]
}
tp.uid = mine
return tp
}
// SetAPI sets the application API.
func (tp *TP) SetAPI(api API) Teleport {
tp.api = api
return tp
}
// Request pushes data; blocks until a connection exists; empty nodeuid sends to a random node.
func (tp *TP) Request(body interface{}, operation string, flag string, nodeuid ...string) {
var conn *Connect
var uid string
if len(nodeuid) == 0 {
for {
if tp.CountNodes() > 0 {
break
}
time.Sleep(LOOP_TIMEOUT)
}
for uid, conn = range tp.connPool {
if conn.Usable {
nodeuid = append(nodeuid, uid)
break
}
}
}
conn = tp.getConn(nodeuid[0])
for conn == nil || !conn.Usable {
conn = tp.getConn(nodeuid[0])
time.Sleep(LOOP_TIMEOUT)
}
conn.WriteChan <- NewNetData(tp.uid, nodeuid[0], operation, flag, body)
}
// Close disconnects; empty nodeuid closes all; in server mode also stops listening.
func (tp *TP) Close(nodeuid ...string) {
if tp.mode == CLIENT {
tp.tpClient.mustClose = true
} else if tp.mode == SERVER && tp.tpServer.listener != nil {
tp.tpServer.listener.Close()
log.Printf(" * -- Server stopped listening on %v --", tp.port)
}
if len(nodeuid) == 0 {
uids := make([]string, 0, len(tp.connPool))
for uid := range tp.connPool {
uids = append(uids, uid)
}
for _, uid := range uids {
conn := tp.connPool[uid]
delete(tp.connPool, uid)
if conn != nil {
conn.Close()
tp.closeMsg(uid, conn.Addr(), conn.Short)
}
}
return
}
for _, uid := range nodeuid {
conn := tp.connPool[uid]
delete(tp.connPool, uid)
if conn != nil {
conn.Close()
tp.closeMsg(uid, conn.Addr(), conn.Short)
}
}
}
// SetPackHeader sets the packet header string.
func (tp *TP) SetPackHeader(header string) Teleport {
tp.Protocol.ReSet(header)
return tp
}
// SetApiRChan sets the global receive channel length.
func (tp *TP) SetApiRChan(length int) Teleport {
tp.apiReadChan = make(chan *NetData, length)
return tp
}
// SetConnWChan sets per-connection write channel length.
func (tp *TP) SetConnWChan(length int) Teleport {
tp.connWChanCap = length
return tp
}
// SetConnBuffer sets per-connection receive buffer size.
func (tp *TP) SetConnBuffer(length int) Teleport {
tp.connBufferLen = length
return tp
}
// SetTimeout sets connection timeout (heartbeat interval).
func (tp *TP) SetTimeout(long time.Duration) Teleport {
tp.timeout = long
return tp
}
// GetMode returns run mode.
func (tp *TP) GetMode() int {
return tp.mode
}
// CountNodes returns the number of active connections.
func (tp *TP) CountNodes() int {
count := 0
for _, conn := range tp.connPool {
if conn != nil && conn.Usable {
count++
}
}
return count
}
func (tp *TP) read(conn *Connect) bool {
readLen, err := conn.Read(conn.Buffer)
if result.TryErrVoid(err).IsErr() || readLen == 0 {
return false
}
conn.TmpBuffer = append(conn.TmpBuffer, conn.Buffer[:readLen]...)
tp.save(conn)
return true
}
// getConn returns the connection for the given node UID.
func (tp *TP) getConn(nodeuid string) *Connect {
return tp.connPool[nodeuid]
}
// getConnAddr returns the address of the connection for the given node UID.
func (tp *TP) getConnAddr(nodeuid string) string {
conn := tp.getConn(nodeuid)
if conn == nil {
return ""
}
return conn.Addr()
}
// closeConn closes the connection and exits the goroutine.
func (tp *TP) closeConn(nodeuid string, reconnect bool) {
conn, ok := tp.connPool[nodeuid]
if !ok {
return
}
if reconnect {
tp.connPool[nodeuid] = nil
} else {
delete(tp.connPool, nodeuid)
}
if conn == nil {
return
}
conn.Close()
tp.closeMsg(nodeuid, conn.Addr(), conn.Short)
}
// closeMsg logs connection close.
func (tp *TP) closeMsg(uid, addr string, short bool) {
if short {
return
}
switch tp.mode {
case SERVER:
log.Printf(" * -- Disconnected from client %v (%v) --", uid, addr)
case CLIENT:
log.Printf(" * -- Disconnected from server %v --", addr)
}
}
// send encodes and sends data.
func (tp *TP) send(data *NetData) {
if data.From == "" {
data.From = tp.uid
}
d := result.Ret(json.Marshal(*data)).UnwrapOrElse(func(err error) []byte {
debugPrintln("Debug: send data encode error", err)
return nil
})
if d == nil {
return
}
conn := tp.getConn(data.To)
if conn == nil {
debugPrintf("Debug: send data connection closed: %+v", data)
return
}
end := tp.Packet(d)
conn.Write(end)
debugPrintf("Debug: send data success: %+v", data)
}
// save decodes received data and stores it in the cache.
func (tp *TP) save(conn *Connect) {
debugPrintf("Debug: received data bytes: %v", conn.TmpBuffer)
dataSlice := make([][]byte, 10)
dataSlice, conn.TmpBuffer = tp.Unpack(conn.TmpBuffer)
for _, data := range dataSlice {
debugPrintf("Debug: received data before decode: %v", string(data))
d := new(NetData)
if r := result.RetVoid(json.Unmarshal(data, d)); r.IsErr() {
debugPrintf("Debug: received data decode error: %v", r.UnwrapErr())
continue
}
if d.From == "" {
d.From = conn.Addr()
}
tp.apiReadChan <- d
debugPrintf("Debug: received data NetData: %+v", d)
}
}
// apiHandle processes requests concurrently via the API.
func (tp *TP) apiHandle() {
for {
req := <-tp.apiReadChan
go func(req *NetData) {
var conn *Connect
operation, from, to, flag := req.Operation, req.To, req.From, req.Flag
handle, ok := tp.api[operation]
if !ok {
peerUID := from
peerConn := tp.getConn(peerUID)
addrStr := ""
if peerConn != nil {
addrStr = peerConn.LocalAddr().String()
}
if tp.mode == SERVER {
tp.autoErrorHandle(req, LLLEGAL, "Server ("+addrStr+") has no API: "+req.Operation, peerUID)
log.Printf("Client %v (%v) requesting non-existent API: %v", from, tp.getConnAddr(peerUID), req.Operation)
} else {
tp.autoErrorHandle(req, LLLEGAL, "Client "+from+" ("+addrStr+") has no API: "+req.Operation, peerUID)
log.Printf("Server (%v) requesting non-existent API: %v", tp.getConnAddr(peerUID), req.Operation)
}
return
}
resp := handle.Process(req)
if resp == nil {
if conn = tp.getConn(to); conn != nil && tp.getConn(to).Short {
tp.closeConn(to, false)
}
return //continue
}
if resp.To == "" {
resp.To = to
}
if conn = tp.getConn(resp.To); conn == nil {
tp.autoErrorHandle(req, FAILURE, "", to)
return
}
if resp.Operation == "" {
resp.Operation = operation
}
if resp.From == "" {
resp.From = from
}
if resp.Flag == "" {
resp.Flag = flag
}
conn.WriteChan <- resp
}(req)
}
}
func (tp *TP) autoErrorHandle(data *NetData, status int, msg string, reqFrom string) bool {
oldConn := tp.getConn(reqFrom)
if oldConn == nil {
return false
}
respErr := ReturnError(data, status, msg)
respErr.From = tp.uid
respErr.To = reqFrom
oldConn.WriteChan <- respErr
return true
}
// checkRights validates connection permissions.
func (tp *TP) checkRights(data *NetData, addr string) bool {
if data.To != tp.uid {
log.Printf("Unknown connection (%v) provided wrong server identifier, request rejected", addr)
return false
}
return true
}
// reserveAPI sets system-reserved API handlers.
func (tp *TP) reserveAPI() {
tp.api[IDENTITY] = identi
tp.api[HEARTBEAT] = beat
}
var identi, beat = new(identity), new(heartbeat)
type identity struct{}
func (*identity) Process(receive *NetData) *NetData {
return nil
}
type heartbeat struct{}
func (*heartbeat) Process(receive *NetData) *NetData {
return nil
}
================================================
FILE: app/distribute/teleport/teleport_test.go
================================================
package teleport
import (
"encoding/json"
"net"
"strconv"
"sync"
"testing"
"time"
)
func freePort(t *testing.T) string {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("freePort: %v", err)
}
defer l.Close()
return strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
}
func TestNew(t *testing.T) {
tp := New()
if tp == nil {
t.Fatal("New returned nil")
}
if tp.GetMode() != 0 {
t.Errorf("GetMode = %d, want 0", tp.GetMode())
}
if tp.CountNodes() != 0 {
t.Errorf("CountNodes = %d, want 0", tp.CountNodes())
}
}
func TestTP_SetUID(t *testing.T) {
tp := New().(*TP)
tp.SetUID("mine")
if tp.uid != "mine" {
t.Errorf("uid = %q, want mine", tp.uid)
}
tp.SetUID("client", "server")
if tp.tpClient.serverUID != "server" {
t.Errorf("serverUID = %q, want server", tp.tpClient.serverUID)
}
}
func TestTP_SetPackHeader(t *testing.T) {
tp := New().(*TP)
tp.SetPackHeader("custom")
if tp.Protocol.header != "custom" {
t.Errorf("header = %q, want custom", tp.Protocol.header)
}
}
func TestTP_SetApiRChan(t *testing.T) {
tp := New().(*TP)
tp.SetApiRChan(100)
if cap(tp.apiReadChan) != 100 {
t.Errorf("apiReadChan cap = %d, want 100", cap(tp.apiReadChan))
}
}
func TestTP_SetConnWChan(t *testing.T) {
tp := New().(*TP)
tp.SetConnWChan(512)
if tp.connWChanCap != 512 {
t.Errorf("connWChanCap = %d, want 512", tp.connWChanCap)
}
}
func TestTP_SetConnBuffer(t *testing.T) {
tp := New().(*TP)
tp.SetConnBuffer(2048)
if tp.connBufferLen != 2048 {
t.Errorf("connBufferLen = %d, want 2048", tp.connBufferLen)
}
}
func TestTP_SetTimeout(t *testing.T) {
tp := New().(*TP)
d := 5 * time.Second
tp.SetTimeout(d)
if tp.timeout != d {
t.Errorf("timeout = %v, want %v", tp.timeout, d)
}
}
func TestTP_SetAPI(t *testing.T) {
tp := New().(*TP)
api := API{"test": &identity{}}
tp.SetAPI(api)
if tp.api["test"] == nil {
t.Error("SetAPI did not set handler")
}
}
func TestTP_ServerClient_Pipe(t *testing.T) {
port := freePort(t)
portStr := ":" + port
serverTP := New().(*TP)
serverTP.SetUID("server").SetTimeout(100 * time.Millisecond)
serverTP.api["echo"] = &echoHandle{}
serverTP.Server(portStr)
time.Sleep(50 * time.Millisecond)
clientTP := New().(*TP)
clientTP.SetUID("client1").SetTimeout(100 * time.Millisecond)
clientTP.api["echo"] = &echoHandle{}
clientTP.Client("127.0.0.1", portStr)
time.Sleep(100 * time.Millisecond)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
clientTP.Request("hello", "echo", "", "server")
}()
time.Sleep(200 * time.Millisecond)
serverTP.Close()
clientTP.Close()
wg.Wait()
}
type echoHandle struct{}
func (*echoHandle) Process(receive *NetData) *NetData {
return ReturnData(receive.Body, receive.Operation, receive.From, receive.To)
}
func TestTP_CloseSpecificNode(t *testing.T) {
port := freePort(t)
portStr := ":" + port
serverTP := New().(*TP)
serverTP.SetUID("server").SetTimeout(100 * time.Millisecond)
serverTP.api["echo"] = &echoHandle{}
serverTP.Server(portStr)
time.Sleep(50 * time.Millisecond)
clientTP := New().(*TP)
clientTP.SetUID("client1").SetTimeout(100 * time.Millisecond)
clientTP.api["echo"] = &echoHandle{}
clientTP.Client("127.0.0.1", portStr)
time.Sleep(100 * time.Millisecond)
serverTP.Close("client1")
clientTP.Close("server")
}
func TestConnect_Close(t *testing.T) {
client, server := net.Pipe()
defer server.Close()
_, conn := NewConnect(client, 64, 16)
if err := conn.Close(); err != nil {
t.Errorf("Close() = %v", err)
}
}
func TestTP_CheckRightsReject(t *testing.T) {
port := freePort(t)
portStr := ":" + port
serverTP := New().(*TP)
serverTP.SetUID("server").SetTimeout(100 * time.Millisecond)
serverTP.api["echo"] = &echoHandle{}
serverTP.Server(portStr)
time.Sleep(50 * time.Millisecond)
conn, err := net.Dial("tcp", "127.0.0.1"+portStr)
if err != nil {
t.Fatalf("Dial: %v", err)
}
defer conn.Close()
nd := &NetData{From: "evil", To: "wrongserver", Operation: IDENTITY, Body: nil}
data, _ := json.Marshal(nd)
p := NewProtocol(DEFAULT_PACK_HEADER)
packed := p.Packet(data)
conn.Write(packed)
conn.Close()
time.Sleep(100 * time.Millisecond)
serverTP.Close()
}
func TestDebugPrint(t *testing.T) {
Debug = true
defer func() { Debug = false }()
debugPrintf("test %v", 1)
debugPrintln("test")
}
func TestTP_GetConnAddr(t *testing.T) {
tp := New().(*TP)
if got := tp.getConnAddr("x"); got != "" {
t.Errorf("getConnAddr(\"x\") = %q, want empty", got)
}
client, server := net.Pipe()
defer client.Close()
defer server.Close()
_, c := NewConnect(client, 64, 16)
c.Usable = true
tp.connPool["node1"] = c
if got := tp.getConnAddr("node1"); got == "" {
t.Error("getConnAddr(\"node1\") = empty")
}
}
================================================
FILE: app/distribute/teleport/util.go
================================================
package teleport
import (
"crypto/md5"
"encoding/hex"
"encoding/json"
"fmt"
"hash/crc32"
"hash/fnv"
"strconv"
)
// MakeHash converts a string to a hash value.
func MakeHash(s string) string {
const IEEE = 0xedb88320
var IEEETable = crc32.MakeTable(IEEE)
hash := fmt.Sprintf("%x", crc32.Checksum([]byte(s), IEEETable))
return hash
}
// HashString encodes a string to a 64-bit hash value.
func HashString(encode string) uint64 {
hash := fnv.New64()
hash.Write([]byte(encode))
return hash.Sum64()
}
// MakeUnique generates a unique fingerprint for an object (method 1).
func MakeUnique(obj interface{}) string {
baseString, _ := json.Marshal(obj)
return strconv.FormatUint(HashString(string(baseString)), 10)
}
// MakeMd5 generates an MD5 fingerprint for an object (method 2).
func MakeMd5(obj interface{}, length int) string {
if length > 32 {
length = 32
}
h := md5.New()
baseString, _ := json.Marshal(obj)
h.Write([]byte(baseString))
s := hex.EncodeToString(h.Sum(nil))
return s[:length]
}
================================================
FILE: app/distribute/teleport/util_test.go
================================================
package teleport
import (
"testing"
)
func TestMakeHash(t *testing.T) {
tests := []struct {
s string
want string
}{
{"", "0"},
{"a", "e8b7be43"},
{"hello", "3610a686"},
}
for _, tt := range tests {
got := MakeHash(tt.s)
if got != tt.want {
t.Errorf("MakeHash(%q) = %q, want %q", tt.s, got, tt.want)
}
}
}
func TestHashString(t *testing.T) {
tests := []struct {
s string
}{
{""},
{"x"},
{"hello world"},
}
for _, tt := range tests {
got := HashString(tt.s)
if tt.s != "" && got == 0 {
t.Errorf("HashString(%q) = 0", tt.s)
}
}
}
func TestMakeUnique(t *testing.T) {
tests := []struct {
obj interface{}
}{
{nil},
{"s"},
{map[string]int{"a": 1}},
}
for _, tt := range tests {
got := MakeUnique(tt.obj)
if got == "" {
t.Errorf("MakeUnique(%v) = empty", tt.obj)
}
}
}
func TestMakeMd5(t *testing.T) {
tests := []struct {
obj interface{}
length int
}{
{"x", 8},
{123, 16},
{[]int{1, 2}, 32},
{"y", 64},
}
for _, tt := range tests {
got := MakeMd5(tt.obj, tt.length)
wantLen := tt.length
if wantLen > 32 {
wantLen = 32
}
if len(got) != wantLen {
t.Errorf("MakeMd5(%v, %d) len = %d, want %d", tt.obj, tt.length, len(got), wantLen)
}
}
}
================================================
FILE: app/downloader/downloader.go
================================================
// Package downloader defines the page downloader interface.
package downloader
import (
"github.com/andeya/pholcus/app/downloader/request"
"github.com/andeya/pholcus/app/spider"
)
// The Downloader interface.
// You can implement the interface by implement function Download.
// Function Download need to return Page instance pointer that has request result downloaded from Request.
type Downloader interface {
Download(*spider.Spider, *request.Request) *spider.Context
}
================================================
FILE: app/downloader/downloader_surfer.go
================================================
package downloader
import (
"errors"
"net/http"
"net/http/cookiejar"
"github.com/andeya/gust/result"
"github.com/andeya/gust/syncutil"
"github.com/andeya/pholcus/app/downloader/request"
"github.com/andeya/pholcus/app/downloader/surfer"
"github.com/andeya/pholcus/app/spider"
"github.com/andeya/pholcus/config"
)
type Surfer struct {
surf surfer.Surfer
}
var (
cookieJar, _ = cookiejar.New(nil)
SurferDownloader = &Surfer{
surf: surfer.New(cookieJar),
}
)
var lazyPhantom = syncutil.NewLazyValueWithFunc(func() result.Result[surfer.Surfer] {
return result.Ok[surfer.Surfer](surfer.NewPhantom(config.Conf().PhantomJS, config.PhantomJSTemp, cookieJar))
})
var lazyChrome = syncutil.NewLazyValueWithFunc(func() result.Result[surfer.Surfer] {
return result.Ok[surfer.Surfer](surfer.NewChrome(cookieJar))
})
func (s *Surfer) Download(sp *spider.Spider, cReq *request.Request) *spider.Context {
ctx := spider.GetContext(sp, cReq)
var resp *http.Response
var err error
switch cReq.GetDownloaderID() {
case request.SurfID:
r := s.surf.Download(cReq)
if r.IsErr() {
err = r.UnwrapErr()
} else {
resp = r.Unwrap()
}
case request.PhantomID:
r := lazyPhantom.TryGetValue().Unwrap().Download(cReq)
if r.IsErr() {
err = r.UnwrapErr()
} else {
resp = r.Unwrap()
}
case request.ChromeID:
r := lazyChrome.TryGetValue().Unwrap().Download(cReq)
if r.IsErr() {
err = r.UnwrapErr()
} else {
resp = r.Unwrap()
}
}
if resp != nil && resp.StatusCode >= 400 {
err = errors.New("response status " + resp.Status)
}
ctx.SetResponse(resp).SetError(err)
return ctx
}
================================================
FILE: app/downloader/downloader_test.go
================================================
package downloader
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/andeya/pholcus/app/downloader/request"
"github.com/andeya/pholcus/app/spider"
)
func TestSurferDownloader_implementsInterface(t *testing.T) {
var _ Downloader = SurferDownloader
}
func makeSpiderNotStopping(name string) *spider.Spider {
sp := &spider.Spider{
Name: name,
RuleTree: &spider.RuleTree{Trunk: map[string]*spider.Rule{}},
}
sp.Register()
return sp
}
func TestSurferDownloader_Download_SurfID(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}))
defer ts.Close()
sp := makeSpiderNotStopping("DownloaderTestSpider1")
req := &request.Request{URL: ts.URL, Rule: "r"}
req.Prepare()
ctx := SurferDownloader.Download(sp, req)
if ctx == nil {
t.Fatal("Download returned nil context")
}
if err := ctx.GetError(); err != nil {
t.Errorf("GetError() = %v, want nil", err)
}
if ctx.Response == nil {
t.Fatal("Response is nil")
}
if ctx.Response.StatusCode != 200 {
t.Errorf("StatusCode = %d, want 200", ctx.Response.StatusCode)
}
}
func TestSurferDownloader_Download_SurfID_error(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
defer ts.Close()
sp := makeSpiderNotStopping("DownloaderTestSpider2")
req := &request.Request{URL: ts.URL, Rule: "r"}
req.Prepare()
ctx := SurferDownloader.Download(sp, req)
if ctx == nil {
t.Fatal("Download returned nil context")
}
if err := ctx.GetError(); err == nil {
t.Error("GetError() = nil, want error for 5xx")
}
}
func TestSurferDownloader_Download_SurfID_4xx(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer ts.Close()
sp := makeSpiderNotStopping("DownloaderTestSpider4xx")
req := &request.Request{URL: ts.URL, Rule: "r"}
req.Prepare()
ctx := SurferDownloader.Download(sp, req)
if ctx == nil {
t.Fatal("Download returned nil context")
}
if err := ctx.GetError(); err == nil {
t.Error("GetError() = nil, want error for 4xx")
}
}
func TestSurferDownloader_Download_SurfID_badURL(t *testing.T) {
sp := makeSpiderNotStopping("DownloaderTestSpider3")
req := &request.Request{URL: "http://localhost:0/nonexistent", Rule: "r"}
req.Prepare()
ctx := SurferDownloader.Download(sp, req)
if ctx == nil {
t.Fatal("Download returned nil context")
}
if err := ctx.GetError(); err == nil {
t.Error("GetError() = nil, want error for failed request")
}
}
================================================
FILE: app/downloader/request/request.go
================================================
// Package request provides encapsulation and deduplication of crawl requests.
package request
import (
"crypto/md5"
"encoding/hex"
"encoding/json"
"net/http"
"net/url"
"strings"
"sync"
"time"
"github.com/andeya/gust/option"
"github.com/andeya/gust/result"
"github.com/andeya/pholcus/common/util"
)
// Request represents object waiting for being crawled.
type Request struct {
Spider string // spider name, auto-set, do not set manually
URL string // target URL, required
Rule string // rule node name for parsing response, required
Method string // GET POST POST-M HEAD
Header http.Header // request headers
EnableCookie bool // whether to use cookies, set in Spider.EnableCookie
PostData string // POST values
DialTimeout time.Duration // dial timeout (dial tcp: i/o timeout)
ConnTimeout time.Duration // connection timeout (WSARecv tcp: i/o timeout)
TryTimes int // max download retry attempts
RetryPause time.Duration // wait time before retry after download failure
RedirectTimes int // max redirects; 0=unlimited, <0=no redirects
Temp Temp // temporary data
TempIsJSON map[string]bool // marks Temp fields stored as JSON; auto-set, do not set manually
Priority int // scheduling priority, default 0 (min priority)
Reloadable bool // whether the link can be re-downloaded
// DownloaderID: 0=Surf (high concurrency, full features), 1=PhantomJS (strong anti-block, slow, low concurrency)
DownloaderID int
proxy string // proxy, auto-set when UI enables proxy
unique string // unique ID
lock sync.RWMutex
}
const (
DefaultDialTimeout = 2 * time.Minute // default server request timeout
DefaultConnTimeout = 2 * time.Minute // default download timeout
DefaultTryTimes = 3 // default max download attempts
DefaultRetryPause = 2 * time.Second // default pause before retry
)
const (
SurfID = 0 // Surf downloader (native Go), do not change
PhantomID = 1 // PhantomJS downloader (fallback, rarely used)
ChromeID = 2 // Chromium headless browser downloader
)
// Prepare sets default values before sending a request.
// Request.URL and Request.Rule must be set.
// Request.Spider is auto-set by the system.
// Request.EnableCookie is set in Spider; per-request values are ignored.
// Optional fields with defaults: Method (GET), DialTimeout, ConnTimeout, TryTimes,
// RedirectTimes, RetryPause, DownloaderID (0=Surf, 1=PhantomJS).
func (r *Request) Prepare() result.VoidResult {
URL, err := url.Parse(r.URL)
if err != nil {
return result.TryErrVoid(err)
}
r.URL = URL.String()
if r.Method == "" {
r.Method = "GET"
} else {
r.Method = strings.ToUpper(r.Method)
}
if r.Header == nil {
r.Header = make(http.Header)
}
if r.DialTimeout < 0 {
r.DialTimeout = 0
} else if r.DialTimeout == 0 {
r.DialTimeout = DefaultDialTimeout
}
if r.ConnTimeout < 0 {
r.ConnTimeout = 0
} else if r.ConnTimeout == 0 {
r.ConnTimeout = DefaultConnTimeout
}
if r.TryTimes == 0 {
r.TryTimes = DefaultTryTimes
}
if r.RetryPause <= 0 {
r.RetryPause = DefaultRetryPause
}
if r.Priority < 0 {
r.Priority = 0
}
if r.DownloaderID < SurfID || r.DownloaderID > ChromeID {
r.DownloaderID = SurfID
}
if r.TempIsJSON == nil {
r.TempIsJSON = make(map[string]bool)
}
if r.Temp == nil {
r.Temp = make(Temp)
}
return result.OkVoid()
}
// UnSerialize deserializes a Request from JSON string.
func UnSerialize(s string) result.Result[*Request] {
req := new(Request)
return result.Ret(req, json.Unmarshal([]byte(s), req))
}
// Serialize serializes the Request to JSON string.
func (r *Request) Serialize() result.Result[string] {
for k, v := range r.Temp {
r.Temp.set(k, v)
r.TempIsJSON[k] = true
}
b, err := json.Marshal(r)
if err != nil {
return result.TryErr[string](err)
}
return result.Ok(strings.ReplaceAll(util.Bytes2String(b), `\u0026`, `&`))
}
// Unique returns the unique identifier for the request.
func (r *Request) Unique() string {
if r.unique == "" {
block := md5.Sum([]byte(r.Spider + r.Rule + r.URL + r.Method))
r.unique = hex.EncodeToString(block[:])
}
return r.unique
}
// Copy returns a deep copy of the request.
func (r *Request) Copy() result.Result[*Request] {
reqcopy := new(Request)
b, err := json.Marshal(r)
if err != nil {
return result.TryErr[*Request](err)
}
return result.Ret(reqcopy, json.Unmarshal(b, reqcopy))
}
// GetURL returns the request URL.
func (r *Request) GetURL() string {
return r.URL
}
// GetMethod returns the HTTP method name (e.g. GET, POST).
func (r *Request) GetMethod() string {
return r.Method
}
// SetMethod sets the HTTP method.
func (r *Request) SetMethod(method string) *Request {
r.Method = strings.ToUpper(method)
return r
}
func (r *Request) SetURL(url string) *Request {
r.URL = url
return r
}
func (r *Request) GetReferer() string {
return r.Header.Get("Referer")
}
func (r *Request) SetReferer(referer string) *Request {
r.Header.Set("Referer", referer)
return r
}
func (r *Request) GetPostData() string {
return r.PostData
}
func (r *Request) GetHeader() http.Header {
return r.Header
}
func (r *Request) SetHeader(key, value string) *Request {
r.Header.Set(key, value)
return r
}
func (r *Request) AddHeader(key, value string) *Request {
r.Header.Add(key, value)
return r
}
func (r *Request) GetEnableCookie() bool {
return r.EnableCookie
}
func (r *Request) SetEnableCookie(enableCookie bool) *Request {
r.EnableCookie = enableCookie
return r
}
func (r *Request) GetCookies() string {
return r.Header.Get("Cookie")
}
func (r *Request) SetCookies(cookie string) *Request {
r.Header.Set("Cookie", cookie)
return r
}
func (r *Request) GetDialTimeout() time.Duration {
return r.DialTimeout
}
func (r *Request) GetConnTimeout() time.Duration {
return r.ConnTimeout
}
func (r *Request) GetTryTimes() int {
return r.TryTimes
}
func (r *Request) GetRetryPause() time.Duration {
return r.RetryPause
}
func (r *Request) GetProxy() string {
return r.proxy
}
func (r *Request) SetProxy(proxy string) *Request {
r.proxy = proxy
return r
}
func (r *Request) GetRedirectTimes() int {
return r.RedirectTimes
}
func (r *Request) GetRuleName() string {
return r.Rule
}
func (r *Request) SetRuleName(ruleName string) *Request {
r.Rule = ruleName
return r
}
func (r *Request) GetSpiderName() string {
return r.Spider
}
func (r *Request) SetSpiderName(spiderName string) *Request {
r.Spider = spiderName
return r
}
func (r *Request) IsReloadable() bool {
return r.Reloadable
}
func (r *Request) SetReloadable(can bool) *Request {
r.Reloadable = can
return r
}
// GetTemp returns temporary cached data. defaultValue must not be nil.
func (r *Request) GetTemp(key string, defaultValue interface{}) interface{} {
if defaultValue == nil {
panic("*Request.GetTemp() defaultValue must not be nil, key=" + key)
}
r.lock.RLock()
defer r.lock.RUnlock()
if r.Temp[key] == nil {
return defaultValue
}
if r.TempIsJSON[key] {
return r.Temp.get(key, defaultValue)
}
return r.Temp[key]
}
// GetTempOpt returns temporary cached data as Option. None when key is missing.
func (r *Request) GetTempOpt(key string) option.Option[interface{}] {
r.lock.RLock()
defer r.lock.RUnlock()
if _, ok := r.Temp[key]; !ok {
return option.None[interface{}]()
}
if r.TempIsJSON[key] {
var v interface{}
r.Temp.get(key, &v)
return option.Some(v)
}
return option.Some(r.Temp[key])
}
func (r *Request) GetTemps() Temp {
return r.Temp
}
func (r *Request) SetTemp(key string, value interface{}) *Request {
r.lock.Lock()
r.Temp[key] = value
delete(r.TempIsJSON, key)
r.lock.Unlock()
return r
}
func (r *Request) SetTemps(temp map[string]interface{}) *Request {
r.lock.Lock()
r.Temp = temp
r.TempIsJSON = make(map[string]bool)
r.lock.Unlock()
return r
}
func (r *Request) GetPriority() int {
return r.Priority
}
func (r *Request) SetPriority(priority int) *Request {
r.Priority = priority
return r
}
func (r *Request) GetDownloaderID() int {
return r.DownloaderID
}
func (r *Request) SetDownloaderID(id int) *Request {
r.DownloaderID = id
return r
}
func (r *Request) MarshalJSON() ([]byte, error) {
for k, v := range r.Temp {
if r.TempIsJSON[k] {
continue
}
r.Temp.set(k, v)
r.TempIsJSON[k] = true
}
// Marshal a struct without the mutex to avoid copying sync.RWMutex
j := struct {
Spider string
URL string
Rule string
Method string
Header http.Header
EnableCookie bool
PostData string
DialTimeout time.Duration
ConnTimeout time.Duration
TryTimes int
RetryPause time.Duration
RedirectTimes int
Temp Temp
TempIsJSON map[string]bool
Priority int
Reloadable bool
DownloaderID int
}{
Spider: r.Spider,
URL: r.URL,
Rule: r.Rule,
Method: r.Method,
Header: r.Header,
EnableCookie: r.EnableCookie,
PostData: r.PostData,
DialTimeout: r.DialTimeout,
ConnTimeout: r.ConnTimeout,
TryTimes: r.TryTimes,
RetryPause: r.RetryPause,
RedirectTimes: r.RedirectTimes,
Temp: r.Temp,
TempIsJSON: r.TempIsJSON,
Priority: r.Priority,
Reloadable: r.Reloadable,
DownloaderID: r.DownloaderID,
}
return json.Marshal(j)
}
================================================
FILE: app/downloader/request/request_test.go
================================================
package request
import (
"encoding/json"
"net/http"
"testing"
"time"
)
func TestReqTemp(t *testing.T) {
var a = &Request{
Temp: Temp{"3": map[string]int{"33": 33}},
}
a.Prepare()
a.SetTemp("6", 66)
c, _ := json.Marshal(&a)
var b = Request{}
json.Unmarshal(c, &b)
b.SetTemp("1", map[string]int{"11": 11})
b.SetTemp("2", []int{22})
b.SetTemp("4", 44)
b.SetTemp("5", "55")
b.SetTemp("x", x{"henry"})
t.Logf("%#v", b.TempIsJSON)
t.Logf("%#v", b.Temp)
t.Logf("1：%#v\n", b.GetTemp("1", map[string]int{}))
t.Logf("2：%#v\n", b.GetTemp("2", []int{}))
t.Logf("3：%#v\n", b.GetTemp("3", map[string]int{}))
t.Logf("4：%v\n", b.GetTemp("4", 0))
t.Logf("5：%#v\n", b.GetTemp("5", ""))
t.Logf("6：%v\n", b.GetTemp("6", 0))
t.Logf("x：%v\n", b.GetTemp("x", x{}))
_b := b.Copy().Unwrap()
_b.SetTemp("6", 666)
t.Logf("%#v", _b.TempIsJSON)
t.Logf("%#v", _b.Temp)
t.Logf("5：%#v\n", _b.GetTemp("5", 1.0))
t.Logf("5：%#v\n", _b.GetTemp("5", ""))
t.Logf("6：%#v\n", _b.GetTemp("6", 0))
t.Logf("x：%v\n", b.GetTemp("x", &x{}))
t.Logf("10000：%#v\n", _b.GetTemp("10000", 999))
}
type x struct {
Name string
}
func TestPrepare(t *testing.T) {
t.Run("invalid URL", func(t *testing.T) {
r := &Request{URL: "://invalid"}
res := r.Prepare()
if res.IsOk() {
t.Error("expected Prepare to fail for invalid URL")
}
})
t.Run("edge cases", func(t *testing.T) {
tests := []struct {
name string
req *Request
chk func(*Request)
}{
{
name: "negative DialTimeout",
req: &Request{URL: "http://a.com", Rule: "r", DialTimeout: -1},
chk: func(r *Request) {
r.Prepare()
if r.DialTimeout != 0 {
t.Errorf("DialTimeout=%v", r.DialTimeout)
}
},
},
{
name: "negative ConnTimeout",
req: &Request{URL: "http://a.com", Rule: "r", ConnTimeout: -1},
chk: func(r *Request) {
r.Prepare()
if r.ConnTimeout != 0 {
t.Errorf("ConnTimeout=%v", r.ConnTimeout)
}
},
},
{
name: "negative Priority",
req: &Request{URL: "http://a.com", Rule: "r", Priority: -5},
chk: func(r *Request) {
r.Prepare()
if r.Priority != 0 {
t.Errorf("Priority=%v", r.Priority)
}
},
},
{
name: "DownloaderID out of range low",
req: &Request{URL: "http://a.com", Rule: "r", DownloaderID: -1},
chk: func(r *Request) {
r.Prepare()
if r.DownloaderID != SurfID {
t.Errorf("DownloaderID=%v", r.DownloaderID)
}
},
},
{
name: "DownloaderID out of range high",
req: &Request{URL: "http://a.com", Rule: "r", DownloaderID: 99},
chk: func(r *Request) {
r.Prepare()
if r.DownloaderID != SurfID {
t.Errorf("DownloaderID=%v", r.DownloaderID)
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.chk(tt.req)
})
}
})
}
func TestSerializeUnSerialize(t *testing.T) {
r := &Request{
Spider: "s", URL: "http://example.com", Rule: "r",
Method: "POST", PostData: "a=1",
Header: http.Header{"X-Custom": {"v"}},
EnableCookie: true,
Temp: Temp{"k": "v"},
}
r.Prepare()
res := r.Serialize()
if res.IsErr() {
t.Fatalf("Serialize: %v", res.Err())
}
s := res.Unwrap()
if s == "" {
t.Error("Serialize returned empty string")
}
ures := UnSerialize(s)
if ures.IsErr() {
t.Fatalf("UnSerialize: %v", ures.Err())
}
req := ures.Unwrap()
if req.URL != r.URL || req.Method != r.Method || req.Spider != r.Spider {
t.Errorf("UnSerialize mismatch: got %+v", req)
}
}
func TestUnSerializeInvalid(t *testing.T) {
res := UnSerialize("invalid json {{{")
if res.IsOk() {
t.Error("expected UnSerialize to fail")
}
}
func TestUnique(t *testing.T) {
r := &Request{Spider: "s", Rule: "r", URL: "http://a.com", Method: "GET"}
r.Prepare()
u1 := r.Unique()
u2 := r.Unique()
if u1 != u2 || len(u1) != 32 {
t.Errorf("Unique: %q vs %q", u1, u2)
}
}
func TestCopy(t *testing.T) {
r := &Request{Spider: "s", URL: "http://a.com", Rule: "r"}
r.Prepare()
r.SetTemp("x", 1)
cres := r.Copy()
if cres.IsErr() {
t.Fatal(cres.Err())
}
c := cres.Unwrap()
if c.URL != r.URL || c.Spider != r.Spider {
t.Errorf("Copy mismatch")
}
if v, ok := c.GetTemp("x", 0).(float64); !ok || v != 1 {
t.Errorf("Copy Temp mismatch: got %v", c.GetTemp("x", 0))
}
}
func TestGettersSetters(t *testing.T) {
r := &Request{URL: "http://a.com", Rule: "r"}
r.Prepare()
tests := []struct {
name string
fn func()
}{
{"GetURL", func() {
r.SetURL("http://u.com")
if r.GetURL() != "http://u.com" {
t.Error("GetURL")
}
}},
{"GetMethod", func() {
r.SetMethod("post")
if r.GetMethod() != "POST" {
t.Error("GetMethod")
}
}},
{"GetReferer", func() {
r.SetReferer("http://ref")
if r.GetReferer() != "http://ref" {
t.Error("GetReferer")
}
}},
{"GetPostData", func() {
r.PostData = "p=1"
if r.GetPostData() != "p=1" {
t.Error("GetPostData")
}
}},
{"GetHeader", func() {
r.SetHeader("A", "1")
if r.GetHeader().Get("A") != "1" {
t.Error("GetHeader")
}
}},
{"AddHeader", func() {
r.AddHeader("B", "2")
if r.GetHeader().Get("B") != "2" {
t.Error("AddHeader")
}
}},
{"GetEnableCookie", func() {
r.SetEnableCookie(true)
if !r.GetEnableCookie() {
t.Error("GetEnableCookie")
}
}},
{"GetCookies", func() {
r.SetCookies("c=1")
if r.GetCookies() != "c=1" {
t.Error("GetCookies")
}
}},
{"GetDialTimeout", func() {
r.DialTimeout = 5 * time.Second
if r.GetDialTimeout() != 5*time.Second {
t.Error("GetDialTimeout")
}
}},
{"GetConnTimeout", func() {
r.ConnTimeout = 10 * time.Second
if r.GetConnTimeout() != 10*time.Second {
t.Error("GetConnTimeout")
}
}},
{"GetTryTimes", func() {
r.TryTimes = 5
if r.GetTryTimes() != 5 {
t.Error("GetTryTimes")
}
}},
{"GetRetryPause", func() {
r.RetryPause = 3 * time.Second
if r.GetRetryPause() != 3*time.Second {
t.Error("GetRetryPause")
}
}},
{"GetProxy", func() {
r.SetProxy("http://p")
if r.GetProxy() != "http://p" {
t.Error("GetProxy")
}
}},
{"GetRedirectTimes", func() {
r.RedirectTimes = 2
if r.GetRedirectTimes() != 2 {
t.Error("GetRedirectTimes")
}
}},
{"GetRuleName", func() {
r.SetRuleName("r1")
if r.GetRuleName() != "r1" {
t.Error("GetRuleName")
}
}},
{"GetSpiderName", func() {
r.SetSpiderName("sp")
if r.GetSpiderName() != "sp" {
t.Error("GetSpiderName")
}
}},
{"IsReloadable", func() {
r.SetReloadable(true)
if !r.IsReloadable() {
t.Error("IsReloadable")
}
}},
{"GetPriority", func() {
r.SetPriority(3)
if r.GetPriority() != 3 {
t.Error("GetPriority")
}
}},
{"GetDownloaderID", func() {
r.SetDownloaderID(PhantomID)
if r.GetDownloaderID() != PhantomID {
t.Error("GetDownloaderID")
}
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tt.fn()
})
}
}
func TestGetTempOpt(t *testing.T) {
r := &Request{URL: "http://a.com", Rule: "r", Temp: Temp{"a": 1}}
r.Prepare()
if opt := r.GetTempOpt("missing"); opt.IsSome() {
t.Error("expected None for missing key")
}
if opt := r.GetTempOpt("a"); !opt.IsSome() || opt.Unwrap() != 1 {
t.Errorf("GetTempOpt(a)=%v", opt)
}
r.SetTemp("j", map[string]int{"x": 1})
sres := r.Serialize()
if sres.IsErr() {
t.Fatal(sres.Err())
}
ures := UnSerialize(sres.Unwrap())
if ures.IsErr() {
t.Fatal(ures.Err())
}
req := ures.Unwrap()
if opt := req.GetTempOpt("j"); !opt.IsSome() {
t.Error("GetTempOpt(j) expected Some")
}
}
func TestGetTemps(t *testing.T) {
r := &Request{URL: "http://a.com", Rule: "r", Temp: Temp{"k": "v"}}
r.Prepare()
temps := r.GetTemps()
if temps["k"] != "v" {
t.Errorf("GetTemps=%v", temps)
}
}
func TestSetTemps(t *testing.T) {
r := &Request{URL: "http://a.com", Rule: "r"}
r.Prepare()
r.SetTemps(map[string]interface{}{"x": 1, "y": "2"})
if r.Temp["x"] != 1 || r.Temp["y"] != "2" {
t.Errorf("SetTemps=%v", r.Temp)
}
}
func TestGetTempPanic(t *testing.T) {
defer func() {
if recover() == nil {
t.Error("expected panic for nil defaultValue")
}
}()
r := &Request{URL: "http://a.com", Rule: "r"}
r.Prepare()
r.GetTemp("k", nil)
}
================================================
FILE: app/downloader/request/temp.go
================================================
package request
import (
"encoding/json"
"reflect"
"github.com/andeya/pholcus/common/util"
"github.com/andeya/pholcus/logs"
)
type Temp map[string]interface{}
// get returns temporary cached data by deserializing from JSON.
func (t Temp) get(key string, defaultValue interface{}) interface{} {
defer func() {
if p := recover(); p != nil {
logs.Log().Error(" * Request.Temp.Get(%v): %v", key, p)
}
}()
var (
err error
b = util.String2Bytes(t[key].(string))
)
if reflect.TypeOf(defaultValue).Kind() == reflect.Ptr {
err = json.Unmarshal(b, defaultValue)
} else {
err = json.Unmarshal(b, &defaultValue)
}
if err != nil {
logs.Log().Error(" * Request.Temp.Get(%v): %v", key, err)
}
return defaultValue
}
func (t Temp) set(key string, value interface{}) Temp {
b, err := json.Marshal(value)
if err != nil {
logs.Log().Error(" * Request.Temp.Set(%v): %v", key, err)
}
t[key] = util.Bytes2String(b)
return t
}
================================================
FILE: app/downloader/surfer/agent/agent.go
================================================
// Package agent generates user agents strings for well known browsers
// and for custom browsers.
//
// When submitting patches to add user agents formats, please *always* include
// "{{.Coms}}" between the opening ( and closing ) braces, even if you're
// sure the browser would never have additional comments.
package agent
import (
"bytes"
"math/rand"
"runtime"
"strings"
"text/template"
"time"
)
// TemplateData structure for template data.
type TemplateData struct {
Name string
Ver string
OSN string
OSV string
Coms string
}
// OSAttributes stores OS attributes.
type OSAttributes struct {
// OSName is the operating system name.
OSName string
// OSVersion is the operating system version.
OSVersion string
// Comments are additional comments to add to a user agent string.
Comments []string
}
const (
// Windows operating system.
Windows int = iota
// Linux based operating system.
Linux
// Macintosh/OS X operating system.
Macintosh
)
// DefaultOSAttributes stores default OS attributes.
var DefaultOSAttributes = map[int]OSAttributes{
Windows: {"Windows NT", "10.0", []string{"Win64", "x64"}},
Linux: {"Linux", "x86_64", []string{}},
Macintosh: {"Intel Mac OS X", "10_15_7", []string{}},
}
type (
// Formats is a collection of UA format strings.
// key is the browser version.
// value is the browser info.
Formats map[string]string
// UAData stores information on a browser user agent.
UAData struct {
TopVersion string
DefaultOS int
Formats Formats
}
// UATable is a collection of UAData values.
// key is the name of the browser.
UATable map[string]UAData
)
// Database is the "database" of user agents.
var Database = UATable{
"chrome": {
"127.0.6533.73",
Windows,
Formats{
"127": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{{.Ver}} Safari/537.36",
"126": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{{.Ver}} Safari/537.36",
"125": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{{.Ver}} Safari/537.36",
"124": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{{.Ver}} Safari/537.36",
"123": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{{.Ver}} Safari/537.36",
"122": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{{.Ver}} Safari/537.36",
"121": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{{.Ver}} Safari/537.36",
"120": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{{.Ver}} Safari/537.36",
},
},
"firefox": {
"127.0",
Windows,
Formats{
"127": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:127.0) Gecko/20100101 Firefox/{{.Ver}}",
"126": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:126.0) Gecko/20100101 Firefox/{{.Ver}}",
"125": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:125.0) Gecko/20100101 Firefox/{{.Ver}}",
"124": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:124.0) Gecko/20100101 Firefox/{{.Ver}}",
"123": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:123.0) Gecko/20100101 Firefox/{{.Ver}}",
"122": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:122.0) Gecko/20100101 Firefox/{{.Ver}}",
"121": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:121.0) Gecko/20100101 Firefox/{{.Ver}}",
"120": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}; rv:120.0) Gecko/20100101 Firefox/{{.Ver}}",
},
},
"edge": {
"127.0.2651.74",
Windows,
Formats{
"127": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/{{.Ver}}",
"126": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/{{.Ver}}",
"125": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/{{.Ver}}",
"124": "Mozilla/5.0 ({{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/{{.Ver}}",
},
},
"safari": {
"17.5",
Macintosh,
Formats{
"17": "Mozilla/5.0 (Macintosh; {{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/{{.Ver}} Safari/605.1.15",
"16": "Mozilla/5.0 (Macintosh; {{.OSN}} {{.OSV}}{{.Coms}}) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/{{.Ver}} Safari/605.1.15",
},
},
"googlebot": {
"2.1",
Linux,
Formats{
"2": "Mozilla/5.0 (compatible; Googlebot/{{.Ver}}; +http://www.google.com/bot.html{{.Coms}})",
"1": "Googlebot/{{.Ver}} (+http://www.google.com/bot.html{{.Coms}})",
},
},
"bingbot": {
"2.0",
Windows,
Formats{
"2": "Mozilla/5.0 (compatible; bingbot/{{.Ver}}; +http://www.bing.com/bingbot.htm{{.Coms}})",
},
},
"yahoobot": {
"2.0",
Linux,
Formats{
"2": "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp{{.Coms}})",
},
},
"default": {
"1.0",
Linux,
Formats{
"1": "{{.Name}}/{{.Ver}} ({{.OSN}} {{.OSV}}{{.Coms}})",
},
},
}
// UserAgents holds all generated user agent strings.
var UserAgents = map[string][]string{}
func init() {
for browser, userAgentData := range Database {
if browser == "default" {
continue
}
os := userAgentData.DefaultOS
osAttribs := DefaultOSAttributes[os]
for version, _ := range userAgentData.Formats {
ua := createFromDetails(
browser,
version,
osAttribs.OSName,
osAttribs.OSVersion,
osAttribs.Comments)
UserAgents["all"] = append(UserAgents["all"], ua)
if browser != "googlebot" && browser != "bingbot" && browser != "yahoobot" {
UserAgents["common"] = append(UserAgents["common"], ua)
}
}
}
l := len(UserAgents["common"])
r := rand.New(rand.NewSource(time.Now().UnixNano()))
idx := r.Intn(l)
UserAgents["all"][0], UserAgents["all"][idx] = UserAgents["all"][idx], UserAgents["all"][0]
UserAgents["common"][0], UserAgents["common"][idx] = UserAgents["common"][idx], UserAgents["common"][0]
}
// Create generates and returns a complete user agent string.
func CreateReal() string {
return createFromD
gitextract_kq_lvwwa/
├── .gitattributes
├── .gitignore
├── LICENSE
├── README.md
├── app/
│ ├── aid/
│ │ ├── history/
│ │ │ ├── failure.go
│ │ │ ├── failure_test.go
│ │ │ ├── history.go
│ │ │ ├── history_test.go
│ │ │ ├── success.go
│ │ │ └── success_test.go
│ │ └── proxy/
│ │ ├── host.go
│ │ ├── host_test.go
│ │ ├── proxy.go
│ │ └── proxy_test.go
│ ├── app.go
│ ├── app_test.go
│ ├── crawler/
│ │ ├── crawler.go
│ │ ├── crawler_test.go
│ │ ├── crawlerpool.go
│ │ ├── crawlerpool_test.go
│ │ ├── spiderqueue.go
│ │ └── spiderqueue_test.go
│ ├── distribute/
│ │ ├── integration_test.go
│ │ ├── interface.go
│ │ ├── master_api.go
│ │ ├── master_api_test.go
│ │ ├── slave_api.go
│ │ ├── slave_api_test.go
│ │ ├── task.go
│ │ ├── task_test.go
│ │ ├── taskjar.go
│ │ ├── taskjar_test.go
│ │ └── teleport/
│ │ ├── client.go
│ │ ├── conn.go
│ │ ├── conn_test.go
│ │ ├── debug.go
│ │ ├── netdata.go
│ │ ├── netdata_test.go
│ │ ├── protocol.go
│ │ ├── protocol_test.go
│ │ ├── return_func.go
│ │ ├── return_func_test.go
│ │ ├── server.go
│ │ ├── teleport.go
│ │ ├── teleport_test.go
│ │ ├── util.go
│ │ └── util_test.go
│ ├── downloader/
│ │ ├── downloader.go
│ │ ├── downloader_surfer.go
│ │ ├── downloader_test.go
│ │ ├── request/
│ │ │ ├── request.go
│ │ │ ├── request_test.go
│ │ │ └── temp.go
│ │ └── surfer/
│ │ ├── agent/
│ │ │ ├── agent.go
│ │ │ ├── agent_bsd.go
│ │ │ ├── agent_linux.go
│ │ │ ├── agent_linux_arm.go
│ │ │ ├── agent_test.go
│ │ │ └── agent_windows.go
│ │ ├── chrome.go
│ │ ├── chrome_stub.go
│ │ ├── chrome_test.go
│ │ ├── example/
│ │ │ └── example.go
│ │ ├── param.go
│ │ ├── param_test.go
│ │ ├── phantom.go
│ │ ├── phantom_stub.go
│ │ ├── request.go
│ │ ├── request_test.go
│ │ ├── surf.go
│ │ ├── surf_stub_test.go
│ │ ├── surf_test.go
│ │ ├── surfer.go
│ │ ├── util.go
│ │ └── util_test.go
│ ├── pipeline/
│ │ ├── collector/
│ │ │ ├── collector.go
│ │ │ ├── collector_test.go
│ │ │ ├── data/
│ │ │ │ ├── data.go
│ │ │ │ └── data_test.go
│ │ │ ├── output_beanstalkd.go
│ │ │ ├── output_beanstalkd_stub.go
│ │ │ ├── output_csv.go
│ │ │ ├── output_data.go
│ │ │ ├── output_data_test.go
│ │ │ ├── output_excel.go
│ │ │ ├── output_file.go
│ │ │ ├── output_kafka.go
│ │ │ ├── output_kafka_stub.go
│ │ │ ├── output_mgo.go
│ │ │ ├── output_mgo_stub.go
│ │ │ ├── output_mysql.go
│ │ │ ├── output_mysql_stub.go
│ │ │ ├── output_util.go
│ │ │ └── output_util_test.go
│ │ ├── output.go
│ │ ├── pipeline.go
│ │ └── pipeline_test.go
│ ├── scheduler/
│ │ ├── matrix.go
│ │ ├── scheduler.go
│ │ └── scheduler_test.go
│ └── spider/
│ ├── common/
│ │ ├── common.go
│ │ ├── common_test.go
│ │ ├── form.go
│ │ └── form_test.go
│ ├── context.go
│ ├── parsejs.go
│ ├── species.go
│ ├── species_test.go
│ ├── spider.go
│ ├── timer.go
│ └── timer_test.go
├── cmd/
│ ├── cmd_test.go
│ └── pholcus-cmd.go
├── common/
│ ├── beanstalkd/
│ │ ├── beanstalkd.go
│ │ └── beanstalkd_test.go
│ ├── bytes/
│ │ ├── bytes.go
│ │ └── bytes_test.go
│ ├── closer/
│ │ ├── closer.go
│ │ └── closer_test.go
│ ├── gc/
│ │ ├── gc.go
│ │ └── gc_test.go
│ ├── goquery/
│ │ ├── .gitattributes
│ │ ├── .gitignore
│ │ ├── .travis.yml
│ │ ├── LICENSE
│ │ ├── README.md
│ │ ├── array.go
│ │ ├── array_test.go
│ │ ├── bench/
│ │ │ ├── v0.1.0
│ │ │ ├── v0.1.1
│ │ │ ├── v0.2.0
│ │ │ ├── v0.2.1-go1.1rc1
│ │ │ ├── v0.3.0
│ │ │ ├── v0.3.2-go1.2
│ │ │ ├── v0.3.2-go1.2-take2
│ │ │ ├── v0.3.2-go1.2rc1
│ │ │ ├── v1.0.0-go1.7
│ │ │ ├── v1.0.1a-go1.7
│ │ │ ├── v1.0.1b-go1.7
│ │ │ └── v1.0.1c-go1.7
│ │ ├── bench_array_test.go
│ │ ├── bench_example_test.go
│ │ ├── bench_expand_test.go
│ │ ├── bench_filter_test.go
│ │ ├── bench_iteration_test.go
│ │ ├── bench_property_test.go
│ │ ├── bench_query_test.go
│ │ ├── bench_traversal_test.go
│ │ ├── doc/
│ │ │ └── tips.md
│ │ ├── doc.go
│ │ ├── example_test.go
│ │ ├── expand.go
│ │ ├── expand_test.go
│ │ ├── filter.go
│ │ ├── filter_test.go
│ │ ├── iteration.go
│ │ ├── iteration_test.go
│ │ ├── manipulation.go
│ │ ├── manipulation_test.go
│ │ ├── misc/
│ │ │ └── git/
│ │ │ └── pre-commit
│ │ ├── property.go
│ │ ├── property_test.go
│ │ ├── query.go
│ │ ├── query_test.go
│ │ ├── testdata/
│ │ │ ├── gotesting.html
│ │ │ ├── gowiki.html
│ │ │ ├── metalreview.html
│ │ │ ├── page.html
│ │ │ ├── page2.html
│ │ │ └── page3.html
│ │ ├── traversal.go
│ │ ├── traversal_test.go
│ │ ├── type.go
│ │ ├── type_test.go
│ │ ├── utilities.go
│ │ └── utilities_test.go
│ ├── kafka/
│ │ ├── kafka.go
│ │ └── kafka_test.go
│ ├── mahonia/
│ │ ├── 8bit.go
│ │ ├── ASCII.go
│ │ ├── README.md
│ │ ├── big5-data.go
│ │ ├── big5.go
│ │ ├── charset.go
│ │ ├── convert_string.go
│ │ ├── cp51932.go
│ │ ├── entity.go
│ │ ├── entity_data.go
│ │ ├── euc-jp.go
│ │ ├── euc-kr-data.go
│ │ ├── euc-kr.go
│ │ ├── fallback.go
│ │ ├── gb18030-data.go
│ │ ├── gb18030.go
│ │ ├── gbk-data.go
│ │ ├── gbk.go
│ │ ├── iso2022jp.go
│ │ ├── jis0201-data.go
│ │ ├── jis0208-data.go
│ │ ├── jis0212-data.go
│ │ ├── kuten.go
│ │ ├── mahonia_test.go
│ │ ├── mahoniconv/
│ │ │ └── mahoniconv.go
│ │ ├── mbcs.go
│ │ ├── ms-jis-data.go
│ │ ├── reader.go
│ │ ├── shiftjis-data.go
│ │ ├── shiftjis.go
│ │ ├── tcvn3.go
│ │ ├── translate.go
│ │ ├── utf16.go
│ │ ├── utf8.go
│ │ └── writer.go
│ ├── mgo/
│ │ ├── count.go
│ │ ├── find.go
│ │ ├── insert.go
│ │ ├── list.go
│ │ ├── mgo.go
│ │ ├── mgo_test.go
│ │ ├── operator.go
│ │ ├── remove.go
│ │ ├── update.go
│ │ ├── update_all.go
│ │ └── upsert.go
│ ├── mysql/
│ │ ├── mysql.go
│ │ └── mysql_test.go
│ ├── ping/
│ │ ├── ping.go
│ │ └── ping_test.go
│ ├── pinyin/
│ │ ├── example_test.go
│ │ ├── initials_sort.go
│ │ ├── phonetic_symbol.go
│ │ ├── pinyin.go
│ │ ├── pinyin_dict.go
│ │ └── pinyin_test.go
│ ├── pool/
│ │ ├── pool.go
│ │ └── pool_test.go
│ ├── queue/
│ │ ├── queue.go
│ │ └── queue_test.go
│ ├── session/
│ │ ├── README.md
│ │ ├── sess_cookie.go
│ │ ├── sess_cookie_test.go
│ │ ├── sess_file.go
│ │ ├── sess_file_test.go
│ │ ├── sess_mem.go
│ │ ├── sess_mem_test.go
│ │ ├── sess_test.go
│ │ ├── sess_utils.go
│ │ ├── session.go
│ │ └── session_manager_test.go
│ ├── simplejson/
│ │ ├── simplejson.go
│ │ └── simplejson_test.go
│ ├── util/
│ │ ├── util.go
│ │ └── util_test.go
│ ├── websocket/
│ │ ├── client.go
│ │ ├── client_test.go
│ │ ├── hybi.go
│ │ ├── server.go
│ │ ├── server_test.go
│ │ ├── websocket.go
│ │ └── websocket_test.go
│ └── xlsx/
│ ├── cell.go
│ ├── col.go
│ ├── date.go
│ ├── doc.go
│ ├── file.go
│ ├── hsl.go
│ ├── lib.go
│ ├── reftable.go
│ ├── row.go
│ ├── sheet.go
│ ├── style.go
│ ├── templates.go
│ ├── theme.go
│ ├── write.go
│ ├── xlsx_test.go
│ ├── xmlContentTypes.go
│ ├── xmlSharedStrings.go
│ ├── xmlStyle.go
│ ├── xmlTheme.go
│ ├── xmlWorkbook.go
│ └── xmlWorksheet.go
├── config/
│ ├── config.go
│ ├── config_test.go
│ └── setting.go
├── doc/
│ └── GUI编译命令.txt
├── doc.go
├── exec/
│ ├── exec.go
│ ├── exec_darwin.go
│ ├── exec_freebsd.go
│ ├── exec_linux.go
│ ├── exec_test.go
│ └── exec_windows.go
├── go.mod
├── go.sum
├── go.work
├── go.work.sum
├── gui/
│ ├── client.go
│ ├── guimain.manifest
│ ├── logview.go
│ ├── model/
│ │ └── guispider.go
│ ├── offline.go
│ ├── pholcus-gui.go
│ ├── rsrc.syso
│ ├── runmode.go
│ ├── server.go
│ └── var.go
├── logs/
│ ├── logs/
│ │ ├── conn.go
│ │ ├── conn_test.go
│ │ ├── console.go
│ │ ├── console_test.go
│ │ ├── file.go
│ │ ├── file_test.go
│ │ ├── log.go
│ │ ├── log_test.go
│ │ ├── smtp.go
│ │ └── smtp_test.go
│ ├── logs.go
│ └── logs_test.go
├── runtime/
│ ├── cache/
│ │ ├── cache.go
│ │ └── cache_test.go
│ └── status/
│ ├── status.go
│ └── status_test.go
├── sample/
│ ├── dyn_rules/
│ │ ├── baidu_search.pholcus.html
│ │ └── baidu_search.pholcus.xml
│ ├── main.go
│ └── static_rules/
│ ├── IJGUC/
│ │ └── IJGUC.go
│ ├── README.md
│ ├── alibaba/
│ │ └── alibaba.go
│ ├── area_codes/
│ │ └── area_codes.go
│ ├── baidunews/
│ │ └── baidunews.go
│ ├── baidusearch/
│ │ └── baidusearch.go
│ ├── car_home/
│ │ └── car_home.go
│ ├── chinanews/
│ │ ├── chinanews.go
│ │ └── readme.md
│ ├── fang_resell_list/
│ │ ├── fang_resell_list.go
│ │ └── readme.md
│ ├── filetest/
│ │ └── filetest.go
│ ├── ganji_gongsi/
│ │ └── ganji_gongsi.go
│ ├── googlesearch/
│ │ └── googlesearch.go
│ ├── hollandandbarrett/
│ │ └── hollandandbarrett.go
│ ├── jdsearch/
│ │ └── jdsearch.go
│ ├── jiban/
│ │ └── jiban.go
│ ├── jingdong/
│ │ ├── README.md
│ │ └── jdSpider.go
│ ├── kaola/
│ │ └── kaola.go
│ ├── lewa/
│ │ └── lewa.go
│ ├── miyabaobei/
│ │ └── miyabaobei.go
│ ├── people/
│ │ └── people.go
│ ├── pholcus_rules.go
│ ├── qq_avatar/
│ │ ├── README.md
│ │ └── avatar.go
│ ├── shunfenghaitao/
│ │ └── shunfenghaitao.go
│ ├── taobao/
│ │ └── taobao.go
│ ├── taobaosearch/
│ │ └── taobaosearch.go
│ ├── wangyi/
│ │ └── wangyi.go
│ ├── weibo_fans/
│ │ └── weibo_fans.go
│ ├── wukongwenda/
│ │ ├── README.md
│ │ └── wukongwenda.go
│ ├── zhihu_bianji/
│ │ ├── README.md
│ │ └── zhihu_bianji.go
│ ├── zhihu_daily/
│ │ ├── README.md
│ │ └── zhihu_daily.go
│ ├── zolpc/
│ │ └── zolpc.go
│ ├── zolphone/
│ │ └── zolphone.go
│ └── zolslab/
│ └── zolslab.go
└── web/
├── embed.go
├── embed_test.go
├── http_controller.go
├── http_controller_test.go
├── logsocket_controller.go
├── logsocket_controller_test.go
├── pholcus-web.go
├── router.go
├── router_test.go
├── views/
│ ├── bootstrap/
│ │ ├── css/
│ │ │ ├── bootstrap-theme.css
│ │ │ └── bootstrap.css
│ │ └── js/
│ │ ├── bootstrap.js
│ │ └── npm.js
│ ├── css/
│ │ ├── pholcus.css
│ │ └── split.css
│ ├── index.html
│ ├── js/
│ │ ├── app.js
│ │ ├── jquery.githubRepoWidget2.js
│ │ └── tpl.js
│ ├── layer/
│ │ ├── extend/
│ │ │ └── layer.ext.js
│ │ ├── layer.js
│ │ └── skin/
│ │ ├── layer.css
│ │ └── layer.ext.css
│ └── splitjs/
│ └── split.js
├── websocket_controller.go
└── websocket_controller_test.go
Showing preview only (263K chars total). Download the full file or copy to clipboard to get everything.
SYMBOL INDEX (3148 symbols across 320 files)
FILE: app/aid/history/failure.go
type Failure (line 18) | type Failure struct
method PullFailure (line 26) | func (f *Failure) PullFailure() map[string]*request.Request {
method UpsertFailure (line 33) | func (f *Failure) UpsertFailure(req *request.Request) bool {
method DeleteFailure (line 44) | func (f *Failure) DeleteFailure(req *request.Request) {
method flush (line 51) | func (f *Failure) flush(provider string) (r result.Result[int]) {
FILE: app/aid/history/failure_test.go
function newTestRequest (line 14) | func newTestRequest(url string) *request.Request {
function TestFailure_PullFailure (line 20) | func TestFailure_PullFailure(t *testing.T) {
function TestFailure_UpsertFailure (line 36) | func TestFailure_UpsertFailure(t *testing.T) {
function TestFailure_DeleteFailure (line 57) | func TestFailure_DeleteFailure(t *testing.T) {
function TestFailure_Flush_File (line 70) | func TestFailure_Flush_File(t *testing.T) {
function TestFailure_Flush_FileEmpty (line 99) | func TestFailure_Flush_FileEmpty(t *testing.T) {
function TestFailure_Flush_FileOverwrite (line 124) | func TestFailure_Flush_FileOverwrite(t *testing.T) {
FILE: app/aid/history/history.go
type HistoryStore (line 24) | type HistoryStore interface
type History (line 40) | type History struct
method ReadSuccess (line 83) | func (h *History) ReadSuccess(provider string, inherit bool) result.Vo...
method ReadFailure (line 162) | func (h *History) ReadFailure(provider string, inherit bool) result.Vo...
method Empty (line 267) | func (h *History) Empty() {
method FlushSuccess (line 276) | func (h *History) FlushSuccess(provider string) (r result.VoidResult) {
method FlushFailure (line 290) | func (h *History) FlushFailure(provider string) (r result.VoidResult) {
constant SuccessSuffix (line 49) | SuccessSuffix = config.HistoryTag + "__y"
constant FailureSuffix (line 50) | FailureSuffix = config.HistoryTag + "__n"
constant SuccessFile (line 51) | SuccessFile = config.HistoryDir + "/" + SuccessSuffix
constant FailureFile (line 52) | FailureFile = config.HistoryDir + "/" + FailureSuffix
function New (line 56) | func New(name string, subName string) HistoryStore {
function getReadMysqlTable (line 308) | func getReadMysqlTable(name string) (*mysql.Table, bool) {
function setReadMysqlTable (line 318) | func setReadMysqlTable(name string, tab *mysql.Table) {
function getWriteMysqlTable (line 329) | func getWriteMysqlTable(name string) (*mysql.Table, bool) {
function setWriteMysqlTable (line 339) | func setWriteMysqlTable(name string, tab *mysql.Table) {
FILE: app/aid/history/history_test.go
function setupHistoryDir (line 17) | func setupHistoryDir(t *testing.T) (cleanup func()) {
function TestNew (line 30) | func TestNew(t *testing.T) {
function TestHistory_ReadSuccess_File (line 57) | func TestHistory_ReadSuccess_File(t *testing.T) {
function TestHistory_ReadSuccess_EmptyFile (line 92) | func TestHistory_ReadSuccess_EmptyFile(t *testing.T) {
function TestHistory_ReadSuccess_InheritPaths (line 107) | func TestHistory_ReadSuccess_InheritPaths(t *testing.T) {
function TestHistory_ReadFailure_File (line 130) | func TestHistory_ReadFailure_File(t *testing.T) {
function TestHistory_ReadFailure_EmptyFile (line 165) | func TestHistory_ReadFailure_EmptyFile(t *testing.T) {
function TestHistory_Empty (line 180) | func TestHistory_Empty(t *testing.T) {
function TestHistory_FlushSuccess_File (line 202) | func TestHistory_FlushSuccess_File(t *testing.T) {
function TestHistory_FlushFailure_File (line 220) | func TestHistory_FlushFailure_File(t *testing.T) {
function TestHistory_FlushSuccess_Empty (line 239) | func TestHistory_FlushSuccess_Empty(t *testing.T) {
function TestHistory_FlushFailure_Empty (line 251) | func TestHistory_FlushFailure_Empty(t *testing.T) {
function TestHistory_ReadSuccess_FileNotFound (line 263) | func TestHistory_ReadSuccess_FileNotFound(t *testing.T) {
function TestHistory_ReadSuccess_ReadFailure_MysqlMock (line 275) | func TestHistory_ReadSuccess_ReadFailure_MysqlMock(t *testing.T) {
function TestHistory_ReadSuccess_MysqlDBError (line 314) | func TestHistory_ReadSuccess_MysqlDBError(t *testing.T) {
function TestHistory_ReadFailure_MysqlDBError (line 329) | func TestHistory_ReadFailure_MysqlDBError(t *testing.T) {
function TestHistory_ReadSuccess_MysqlSelectError (line 344) | func TestHistory_ReadSuccess_MysqlSelectError(t *testing.T) {
function TestHistory_FlushSuccess_FlushFailure_MysqlMock (line 365) | func TestHistory_FlushSuccess_FlushFailure_MysqlMock(t *testing.T) {
function TestHistory_ReadFailure_InvalidData (line 401) | func TestHistory_ReadFailure_InvalidData(t *testing.T) {
FILE: app/aid/history/success.go
type Success (line 16) | type Success struct
method UpsertSuccess (line 26) | func (s *Success) UpsertSuccess(reqUnique string) bool {
method HasSuccess (line 40) | func (s *Success) HasSuccess(reqUnique string) bool {
method DeleteSuccess (line 48) | func (s *Success) DeleteSuccess(reqUnique string) {
method flush (line 54) | func (s *Success) flush(provider string) result.Result[int] {
FILE: app/aid/history/success_test.go
function TestSuccess_UpsertSuccess (line 13) | func TestSuccess_UpsertSuccess(t *testing.T) {
function TestSuccess_UpsertSuccess_OldExists (line 36) | func TestSuccess_UpsertSuccess_OldExists(t *testing.T) {
function TestSuccess_HasSuccess (line 48) | func TestSuccess_HasSuccess(t *testing.T) {
function TestSuccess_DeleteSuccess (line 70) | func TestSuccess_DeleteSuccess(t *testing.T) {
function TestSuccess_Flush_File (line 83) | func TestSuccess_Flush_File(t *testing.T) {
function TestSuccess_Flush_Empty (line 112) | func TestSuccess_Flush_Empty(t *testing.T) {
function TestSuccess_Flush_FileAppend (line 128) | func TestSuccess_Flush_FileAppend(t *testing.T) {
FILE: app/aid/proxy/host.go
type ProxyForHost (line 9) | type ProxyForHost struct
method Len (line 18) | func (ph *ProxyForHost) Len() int {
method Less (line 22) | func (ph *ProxyForHost) Less(i, j int) bool {
method Swap (line 26) | func (ph *ProxyForHost) Swap(i, j int) {
FILE: app/aid/proxy/host_test.go
function TestProxyForHost_Len (line 8) | func TestProxyForHost_Len(t *testing.T) {
function TestProxyForHost_Less (line 26) | func TestProxyForHost_Less(t *testing.T) {
function TestProxyForHost_Swap (line 47) | func TestProxyForHost_Swap(t *testing.T) {
FILE: app/aid/proxy/proxy.go
type Proxy (line 27) | type Proxy struct
method Count (line 67) | func (p *Proxy) Count() int32 {
method SetSurfForTest (line 72) | func (p *Proxy) SetSurfForTest(s surfer.Surfer) {
method Update (line 77) | func (p *Proxy) Update() result.VoidResult {
method findOnline (line 105) | func (p *Proxy) findOnline() *Proxy {
method UpdateTicker (line 131) | func (p *Proxy) UpdateTicker(tickMinute int64) {
method GetOne (line 141) | func (p *Proxy) GetOne(u string) option.Option[string] {
method testAndSort (line 200) | func (p *Proxy) testAndSort(key string, testHost string) (*ProxyForHos...
method findUsable (line 235) | func (p *Proxy) findUsable(proxy string, testHost string) (alive bool,...
constant CONN_TIMEOUT (line 43) | CONN_TIMEOUT = 4
constant DAIL_TIMEOUT (line 44) | DAIL_TIMEOUT = 4
constant TRY_TIMES (line 45) | TRY_TIMES = 3
constant MAX_THREAD_NUM (line 47) | MAX_THREAD_NUM = 1000
function New (line 51) | func New() *Proxy {
FILE: app/aid/proxy/proxy_test.go
function setupProxyDir (line 16) | func setupProxyDir(t *testing.T) (cleanup func()) {
function newTestProxy (line 33) | func newTestProxy() *Proxy {
function TestProxy_Update_EmptyFile (line 46) | func TestProxy_Update_EmptyFile(t *testing.T) {
function TestProxy_Update_WithIPs (line 61) | func TestProxy_Update_WithIPs(t *testing.T) {
function TestProxy_GetOne_NoOnline (line 79) | func TestProxy_GetOne_NoOnline(t *testing.T) {
function TestProxy_GetOne_EmptyHost (line 86) | func TestProxy_GetOne_EmptyHost(t *testing.T) {
function TestProxy_UpdateTicker (line 93) | func TestProxy_UpdateTicker(t *testing.T) {
function TestProxy_New (line 107) | func TestProxy_New(t *testing.T) {
function TestProxy_GetOne_WithUsable (line 119) | func TestProxy_GetOne_WithUsable(t *testing.T) {
function TestProxy_GetOne_NoUsableForHost (line 141) | func TestProxy_GetOne_NoUsableForHost(t *testing.T) {
type mockSurfer (line 160) | type mockSurfer struct
method Download (line 164) | func (m *mockSurfer) Download(req surfer.Request) result.Result[*http....
function TestProxy_GetOne_TriggersTestAndSort (line 171) | func TestProxy_GetOne_TriggersTestAndSort(t *testing.T) {
function TestProxy_Update_FileNotFound (line 200) | func TestProxy_Update_FileNotFound(t *testing.T) {
FILE: app/app.go
type App (line 31) | type App interface
type Logic (line 54) | type Logic struct
method SetLog (line 91) | func (l *Logic) SetLog(w io.Writer) App {
method LogRest (line 97) | func (l *Logic) LogRest() App {
method LogGoOn (line 103) | func (l *Logic) LogGoOn() App {
method GetAppConf (line 109) | func (l *Logic) GetAppConf(k ...string) interface{} {
method SetAppConf (line 124) | func (l *Logic) SetAppConf(k string, v interface{}) App {
method Init (line 145) | func (l *Logic) Init(mode int, port int, master string, w ...io.Writer...
method ReInit (line 189) | func (l *Logic) ReInit(mode int, port int, master string, w ...io.Writ...
method SpiderPrepare (line 212) | func (l *Logic) SpiderPrepare(original []*spider.Spider) App {
method GetOutputLib (line 229) | func (l *Logic) GetOutputLib() []string {
method GetSpiderLib (line 234) | func (l *Logic) GetSpiderLib() []*spider.Spider {
method GetSpiderByName (line 239) | func (l *Logic) GetSpiderByName(name string) option.Option[*spider.Spi...
method GetMode (line 244) | func (l *Logic) GetMode() int {
method GetTaskJar (line 249) | func (l *Logic) GetTaskJar() *distribute.TaskJar {
method CountNodes (line 254) | func (l *Logic) CountNodes() int {
method GetSpiderQueue (line 259) | func (l *Logic) GetSpiderQueue() crawler.SpiderQueue {
method Run (line 264) | func (l *Logic) Run() {
method PauseRecover (line 291) | func (l *Logic) PauseRecover() {
method Stop (line 303) | func (l *Logic) Stop() {
method IsRunning (line 319) | func (l *Logic) IsRunning() bool {
method IsPaused (line 324) | func (l *Logic) IsPaused() bool {
method IsStopped (line 329) | func (l *Logic) IsStopped() bool {
method Status (line 334) | func (l *Logic) Status() int {
method setStatus (line 341) | func (l *Logic) setStatus(status int) {
method offline (line 350) | func (l *Logic) offline() {
method server (line 356) | func (l *Logic) server() {
method addNewTask (line 376) | func (l *Logic) addNewTask() (tasksNum, spidersNum int) {
method client (line 403) | func (l *Logic) client() {
method downTask (line 421) | func (l *Logic) downTask() *distribute.Task {
method taskToRun (line 448) | func (l *Logic) taskToRun(t *distribute.Task) {
method exec (line 472) | func (l *Logic) exec() {
method goRun (line 497) | func (l *Logic) goRun(count int) {
method socketLog (line 572) | func (l *Logic) socketLog() {
method checkPort (line 585) | func (l *Logic) checkPort() bool {
method checkAll (line 593) | func (l *Logic) checkAll() bool {
method setAppConf (line 602) | func (l *Logic) setAppConf(task *distribute.Task) {
method setTask (line 613) | func (l *Logic) setTask(task *distribute.Task) {
function New (line 74) | func New() App {
function newLogic (line 78) | func newLogic() *Logic {
function titleCase (line 625) | func titleCase(s string) string {
FILE: app/app_test.go
function TestNew (line 12) | func TestNew(t *testing.T) {
function TestLogic_SetLog_LogGoOn_LogRest (line 19) | func TestLogic_SetLog_LogGoOn_LogRest(t *testing.T) {
function TestLogic_GetAppConf (line 27) | func TestLogic_GetAppConf(t *testing.T) {
function TestLogic_SetAppConf (line 42) | func TestLogic_SetAppConf(t *testing.T) {
function TestLogic_GetSpiderLib (line 59) | func TestLogic_GetSpiderLib(t *testing.T) {
function TestLogic_GetSpiderByName (line 67) | func TestLogic_GetSpiderByName(t *testing.T) {
function TestLogic_GetSpiderQueue (line 75) | func TestLogic_GetSpiderQueue(t *testing.T) {
function TestLogic_GetOutputLib (line 86) | func TestLogic_GetOutputLib(t *testing.T) {
function TestLogic_GetTaskJar (line 94) | func TestLogic_GetTaskJar(t *testing.T) {
function TestLogic_Status_IsRunning_IsPaused_IsStopped (line 102) | func TestLogic_Status_IsRunning_IsPaused_IsStopped(t *testing.T) {
function TestLogic_Init_Offline (line 118) | func TestLogic_Init_Offline(t *testing.T) {
function TestLogic_Init_Server_invalidPort (line 126) | func TestLogic_Init_Server_invalidPort(t *testing.T) {
function TestLogic_Init_Server_validPort (line 134) | func TestLogic_Init_Server_validPort(t *testing.T) {
function TestLogic_Init_Client_invalidMaster (line 142) | func TestLogic_Init_Client_invalidMaster(t *testing.T) {
function TestLogic_Init_invalidMode (line 150) | func TestLogic_Init_invalidMode(t *testing.T) {
function TestLogic_GetMode (line 158) | func TestLogic_GetMode(t *testing.T) {
function TestLogic_ReInit (line 166) | func TestLogic_ReInit(t *testing.T) {
function TestLogic_GetAppConf_titleCase (line 175) | func TestLogic_GetAppConf_titleCase(t *testing.T) {
function TestLogic_SpiderPrepare (line 184) | func TestLogic_SpiderPrepare(t *testing.T) {
function TestLogic_Run_emptyQueue (line 203) | func TestLogic_Run_emptyQueue(t *testing.T) {
function TestLogic_Stop_whenStopped (line 209) | func TestLogic_Stop_whenStopped(t *testing.T) {
function TestLogic_PauseRecover (line 214) | func TestLogic_PauseRecover(t *testing.T) {
function TestLogic_Run_offline_withSpiders (line 220) | func TestLogic_Run_offline_withSpiders(t *testing.T) {
function TestLogic_Run_server_withSpiders (line 238) | func TestLogic_Run_server_withSpiders(t *testing.T) {
FILE: app/crawler/crawler.go
type Crawler (line 21) | type Crawler interface
type crawler (line 28) | type crawler struct
method Init (line 50) | func (c *crawler) Init(sp *spider.Spider) Crawler {
method Run (line 63) | func (c *crawler) Run() {
method Stop (line 80) | func (c *crawler) Stop() {
method run (line 85) | func (c *crawler) run() {
method Process (line 112) | func (c *crawler) Process(req *request.Request) {
method sleep (line 176) | func (c *crawler) sleep() {
method GetOne (line 182) | func (c *crawler) GetOne() *request.Request {
method UseOne (line 187) | func (c *crawler) UseOne() {
method FreeOne (line 192) | func (c *crawler) FreeOne() {
method SetID (line 197) | func (c *crawler) SetID(id int) {
method GetID (line 202) | func (c *crawler) GetID() int {
function New (line 40) | func New(id int, dl downloader.Downloader, outType string, batchCap int)...
FILE: app/crawler/crawler_test.go
function TestNew (line 14) | func TestNew(t *testing.T) {
function TestCrawler_GetID (line 24) | func TestCrawler_GetID(t *testing.T) {
function TestCrawler_Init (line 31) | func TestCrawler_Init(t *testing.T) {
function TestCrawler_Init_zeroPause (line 45) | func TestCrawler_Init_zeroPause(t *testing.T) {
function TestCrawler_GetOne_UseOne_FreeOne (line 57) | func TestCrawler_GetOne_UseOne_FreeOne(t *testing.T) {
function TestCrawler_CanStop (line 75) | func TestCrawler_CanStop(t *testing.T) {
function TestCrawler_Stop (line 89) | func TestCrawler_Stop(t *testing.T) {
function TestCrawler_SetID (line 101) | func TestCrawler_SetID(t *testing.T) {
type errorDownloader (line 109) | type errorDownloader struct
method Download (line 111) | func (d *errorDownloader) Download(sp *spider.Spider, req *request.Req...
function TestCrawler_Process_downloadError (line 117) | func TestCrawler_Process_downloadError(t *testing.T) {
function TestCrawler_Run (line 131) | func TestCrawler_Run(t *testing.T) {
type successDownloader (line 153) | type successDownloader struct
method Download (line 155) | func (d *successDownloader) Download(sp *spider.Spider, req *request.R...
function TestCrawler_Process_success (line 161) | func TestCrawler_Process_success(t *testing.T) {
FILE: app/crawler/crawlerpool.go
type CrawlerPool (line 15) | type CrawlerPool interface
type cq (line 23) | type cq struct
method SetPipelineConfig (line 46) | func (cq *cq) SetPipelineConfig(outType string, batchCap int) {
method Reset (line 55) | func (cq *cq) Reset(spiderNum int) int {
method Use (line 81) | func (cq *cq) Use() Crawler {
method UseOpt (line 86) | func (cq *cq) UseOpt() option.Option[Crawler] {
method Free (line 113) | func (cq *cq) Free(crawler Crawler) {
method Stop (line 123) | func (cq *cq) Stop() {
function NewCrawlerPool (line 37) | func NewCrawlerPool(dl downloader.Downloader) CrawlerPool {
FILE: app/crawler/crawlerpool_test.go
type mockDownloader (line 13) | type mockDownloader struct
method Download (line 15) | func (d *mockDownloader) Download(_ *spider.Spider, _ *request.Request...
function TestNewCrawlerPool (line 19) | func TestNewCrawlerPool(t *testing.T) {
function TestCrawlerPool_SetPipelineConfig (line 27) | func TestCrawlerPool_SetPipelineConfig(t *testing.T) {
function TestCrawlerPool_Reset (line 32) | func TestCrawlerPool_Reset(t *testing.T) {
function TestCrawlerPool_Use_UseOpt_Free (line 57) | func TestCrawlerPool_Use_UseOpt_Free(t *testing.T) {
function TestCrawlerPool_UseOpt_returnsSome (line 86) | func TestCrawlerPool_UseOpt_returnsSome(t *testing.T) {
function TestCrawlerPool_Stop (line 102) | func TestCrawlerPool_Stop(t *testing.T) {
function TestCrawlerPool_Reset_reuse (line 114) | func TestCrawlerPool_Reset_reuse(t *testing.T) {
function TestCrawlerPool_Stop_idempotent (line 137) | func TestCrawlerPool_Stop_idempotent(t *testing.T) {
function TestCrawlerPool_Free_whenStopped (line 144) | func TestCrawlerPool_Free_whenStopped(t *testing.T) {
FILE: app/crawler/spiderqueue.go
type SpiderQueue (line 12) | type SpiderQueue interface
type sq (line 24) | type sq struct
method Reset (line 37) | func (sq *sq) Reset() {
method Add (line 42) | func (sq *sq) Add(sp *spider.Spider) {
method AddAll (line 48) | func (sq *sq) AddAll(list []*spider.Spider) {
method AddKeyins (line 56) | func (sq *sq) AddKeyins(keyins string) {
method GetByIndex (line 93) | func (sq *sq) GetByIndex(idx int) *spider.Spider {
method GetByIndexOpt (line 98) | func (sq *sq) GetByIndexOpt(idx int) option.Option[*spider.Spider] {
method GetByName (line 106) | func (sq *sq) GetByName(n string) *spider.Spider {
method GetByNameOpt (line 111) | func (sq *sq) GetByNameOpt(n string) option.Option[*spider.Spider] {
method GetAll (line 121) | func (sq *sq) GetAll() []*spider.Spider {
method Len (line 126) | func (sq *sq) Len() int {
function NewSpiderQueue (line 30) | func NewSpiderQueue() SpiderQueue {
FILE: app/crawler/spiderqueue_test.go
function makeSpider (line 9) | func makeSpider(name string, keyin string) *spider.Spider {
function TestNewSpiderQueue (line 19) | func TestNewSpiderQueue(t *testing.T) {
function TestSpiderQueue_Add_Len_Reset (line 29) | func TestSpiderQueue_Add_Len_Reset(t *testing.T) {
function TestSpiderQueue_AddAll (line 56) | func TestSpiderQueue_AddAll(t *testing.T) {
function TestSpiderQueue_GetByIndex_GetByIndexOpt (line 75) | func TestSpiderQueue_GetByIndex_GetByIndexOpt(t *testing.T) {
function TestSpiderQueue_GetByName_GetByNameOpt (line 108) | func TestSpiderQueue_GetByName_GetByNameOpt(t *testing.T) {
function TestSpiderQueue_AddKeyins (line 140) | func TestSpiderQueue_AddKeyins(t *testing.T) {
function TestSpiderQueue_Add_setsID (line 165) | func TestSpiderQueue_Add_setsID(t *testing.T) {
function TestSpiderQueue_GetByIndexOpt (line 179) | func TestSpiderQueue_GetByIndexOpt(t *testing.T) {
function TestSpiderQueue_AddKeyins_emptyUnit2 (line 187) | func TestSpiderQueue_AddKeyins_emptyUnit2(t *testing.T) {
FILE: app/distribute/integration_test.go
function freePort (line 13) | func freePort(t *testing.T) string {
function TestTP_ServerClient_Request (line 22) | func TestTP_ServerClient_Request(t *testing.T) {
FILE: app/distribute/interface.go
type Distributor (line 4) | type Distributor interface
FILE: app/distribute/master_api.go
function MasterAPI (line 12) | func MasterAPI(n Distributor) teleport.API {
type masterTaskHandle (line 20) | type masterTaskHandle struct
method Process (line 24) | func (mth *masterTaskHandle) Process(receive *teleport.NetData) *telep...
type masterLogHandle (line 33) | type masterLogHandle struct
method Process (line 35) | func (*masterLogHandle) Process(receive *teleport.NetData) *teleport.N...
FILE: app/distribute/master_api_test.go
type mockDistributor (line 10) | type mockDistributor struct
method Send (line 17) | func (m *mockDistributor) Send(clientNum int) Task {
method Receive (line 22) | func (m *mockDistributor) Receive(task *Task) {
method CountNodes (line 26) | func (m *mockDistributor) CountNodes() int {
function TestMasterAPI (line 30) | func TestMasterAPI(t *testing.T) {
function TestMasterTaskHandle_Process (line 44) | func TestMasterTaskHandle_Process(t *testing.T) {
function TestMasterLogHandle_Process (line 70) | func TestMasterLogHandle_Process(t *testing.T) {
FILE: app/distribute/slave_api.go
function SlaveAPI (line 12) | func SlaveAPI(n Distributor) teleport.API {
type slaveTaskHandle (line 19) | type slaveTaskHandle struct
method Process (line 23) | func (sth *slaveTaskHandle) Process(receive *teleport.NetData) *telepo...
FILE: app/distribute/slave_api_test.go
function TestSlaveAPI (line 10) | func TestSlaveAPI(t *testing.T) {
function TestSlaveTaskHandle_Process (line 21) | func TestSlaveTaskHandle_Process(t *testing.T) {
function TestSlaveTaskHandle_Process_InvalidJSON (line 38) | func TestSlaveTaskHandle_Process_InvalidJSON(t *testing.T) {
FILE: app/distribute/task.go
type Task (line 5) | type Task struct
FILE: app/distribute/task_test.go
function TestTask_Fields (line 7) | func TestTask_Fields(t *testing.T) {
FILE: app/distribute/taskjar.go
type TaskJar (line 4) | type TaskJar struct
method Push (line 16) | func (tj *TaskJar) Push(task *Task) {
method Pull (line 23) | func (tj *TaskJar) Pull() *Task {
method Len (line 28) | func (tj *TaskJar) Len() int {
method Send (line 33) | func (tj *TaskJar) Send(clientNum int) Task {
method Receive (line 38) | func (tj *TaskJar) Receive(task *Task) {
method CountNodes (line 43) | func (tj *TaskJar) CountNodes() int {
function NewTaskJar (line 9) | func NewTaskJar() *TaskJar {
FILE: app/distribute/taskjar_test.go
function TestNewTaskJar (line 8) | func TestNewTaskJar(t *testing.T) {
function TestTaskJar_PushPull (line 21) | func TestTaskJar_PushPull(t *testing.T) {
function TestTaskJar_SendReceive (line 37) | func TestTaskJar_SendReceive(t *testing.T) {
function TestTaskJar_PushAssignsID (line 61) | func TestTaskJar_PushAssignsID(t *testing.T) {
function TestTaskJar_CountNodes (line 77) | func TestTaskJar_CountNodes(t *testing.T) {
function TestTaskJar_Concurrent (line 84) | func TestTaskJar_Concurrent(t *testing.T) {
FILE: app/distribute/teleport/client.go
type tpClient (line 12) | type tpClient struct
method Client (line 19) | func (tp *TP) Client(serverAddr string, port string, isShort ...bool) {
method client (line 47) | func (tp *TP) client() {
method cGoConn (line 77) | func (tp *TP) cGoConn(conn net.Conn) {
method cReader (line 99) | func (tp *TP) cReader(nodeuid string) {
method cWriter (line 114) | func (tp *TP) cWriter(nodeuid string) {
FILE: app/distribute/teleport/conn.go
type Connect (line 8) | type Connect struct
method Addr (line 31) | func (conn *Connect) Addr() string {
function NewConnect (line 18) | func NewConnect(conn net.Conn, bufferLen int, wChanCap int) (k string, v...
FILE: app/distribute/teleport/conn_test.go
function TestNewConnect (line 8) | func TestNewConnect(t *testing.T) {
function TestConnect_Addr (line 31) | func TestConnect_Addr(t *testing.T) {
FILE: app/distribute/teleport/debug.go
function debugPrintf (line 9) | func debugPrintf(format string, v ...interface{}) {
function debugPrintln (line 16) | func debugPrintln(v ...interface{}) {
function debugFatal (line 23) | func debugFatal(v ...interface{}) {
FILE: app/distribute/teleport/netdata.go
constant SUCCESS (line 4) | SUCCESS = 0
constant FAILURE (line 5) | FAILURE = -1
constant LLLEGAL (line 6) | LLLEGAL = -2
type NetData (line 10) | type NetData struct
function NewNetData (line 20) | func NewNetData(from, to, operation string, flag string, body interface{...
FILE: app/distribute/teleport/netdata_test.go
function TestNewNetData (line 7) | func TestNewNetData(t *testing.T) {
FILE: app/distribute/teleport/protocol.go
constant DataLengthOfLenth (line 9) | DataLengthOfLenth = 4
type Protocol (line 13) | type Protocol struct
method ReSet (line 26) | func (p *Protocol) ReSet(header string) {
method Packet (line 32) | func (p *Protocol) Packet(message []byte) []byte {
method Unpack (line 37) | func (p *Protocol) Unpack(buffer []byte) (readerSlice [][]byte, buffer...
function NewProtocol (line 19) | func NewProtocol(packetHeader string) *Protocol {
function IntToBytes (line 67) | func IntToBytes(n int) []byte {
function BytesToInt (line 76) | func BytesToInt(b []byte) int {
FILE: app/distribute/teleport/protocol_test.go
function TestNewProtocol (line 8) | func TestNewProtocol(t *testing.T) {
function TestProtocol_ReSet (line 33) | func TestProtocol_ReSet(t *testing.T) {
function TestProtocol_Packet (line 44) | func TestProtocol_Packet(t *testing.T) {
function TestProtocol_Unpack (line 54) | func TestProtocol_Unpack(t *testing.T) {
function TestProtocol_Unpack_Multiple (line 70) | func TestProtocol_Unpack_Multiple(t *testing.T) {
function TestProtocol_Unpack_Partial (line 90) | func TestProtocol_Unpack_Partial(t *testing.T) {
function TestProtocol_Unpack_GarbageBeforeHeader (line 104) | func TestProtocol_Unpack_GarbageBeforeHeader(t *testing.T) {
function TestProtocol_Unpack_EmptyBuffer (line 121) | func TestProtocol_Unpack_EmptyBuffer(t *testing.T) {
function TestProtocol_Unpack_TooShort (line 132) | func TestProtocol_Unpack_TooShort(t *testing.T) {
function TestIntToBytes_BytesToInt (line 143) | func TestIntToBytes_BytesToInt(t *testing.T) {
FILE: app/distribute/teleport/return_func.go
function ReturnData (line 4) | func ReturnData(body interface{}, OpAndToAndFrom ...string) *NetData {
function ReturnError (line 22) | func ReturnError(receive *NetData, status int, msg string, nodeuid ...st...
FILE: app/distribute/teleport/return_func_test.go
function TestReturnData (line 7) | func TestReturnData(t *testing.T) {
function TestReturnError (line 48) | func TestReturnError(t *testing.T) {
function TestReturnError_NoNodeUID (line 68) | func TestReturnError_NoNodeUID(t *testing.T) {
FILE: app/distribute/teleport/server.go
type tpServer (line 13) | type tpServer struct
method Server (line 18) | func (tp *TP) Server(port ...string) {
method server (line 38) | func (tp *TP) server() {
method sGoConn (line 62) | func (tp *TP) sGoConn(conn net.Conn) {
method sInitConn (line 75) | func (tp *TP) sInitConn(conn *Connect, remoteAddr string) (nodeuid strin...
method sReader (line 120) | func (tp *TP) sReader(nodeuid string) {
method sWriter (line 138) | func (tp *TP) sWriter(nodeuid string) {
FILE: app/distribute/teleport/teleport.go
constant SERVER (line 16) | SERVER = iota + 1
constant CLIENT (line 17) | CLIENT
constant IDENTITY (line 22) | IDENTITY = "+identity+"
constant HEARTBEAT (line 23) | HEARTBEAT = "+heartbeat+"
constant DEFAULT_PACK_HEADER (line 24) | DEFAULT_PACK_HEADER = "andeya"
constant DEFAULT_SERVER_UID (line 25) | DEFAULT_SERVER_UID = "server"
constant DEFAULT_PORT (line 26) | DEFAULT_PORT = ":8080"
constant DEFAULT_TIMEOUT_S (line 27) | DEFAULT_TIMEOUT_S = 20e9
constant DEFAULT_TIMEOUT_C (line 28) | DEFAULT_TIMEOUT_C = 15e9
constant LOOP_TIMEOUT (line 29) | LOOP_TIMEOUT = 1e9
type Teleport (line 32) | type Teleport interface
type TP (line 50) | type TP struct
method SetUID (line 89) | func (tp *TP) SetUID(mine string, server ...string) Teleport {
method SetAPI (line 98) | func (tp *TP) SetAPI(api API) Teleport {
method Request (line 104) | func (tp *TP) Request(body interface{}, operation string, flag string,...
method Close (line 130) | func (tp *TP) Close(nodeuid ...string) {
method SetPackHeader (line 166) | func (tp *TP) SetPackHeader(header string) Teleport {
method SetApiRChan (line 172) | func (tp *TP) SetApiRChan(length int) Teleport {
method SetConnWChan (line 178) | func (tp *TP) SetConnWChan(length int) Teleport {
method SetConnBuffer (line 184) | func (tp *TP) SetConnBuffer(length int) Teleport {
method SetTimeout (line 190) | func (tp *TP) SetTimeout(long time.Duration) Teleport {
method GetMode (line 196) | func (tp *TP) GetMode() int {
method CountNodes (line 201) | func (tp *TP) CountNodes() int {
method read (line 211) | func (tp *TP) read(conn *Connect) bool {
method getConn (line 222) | func (tp *TP) getConn(nodeuid string) *Connect {
method getConnAddr (line 227) | func (tp *TP) getConnAddr(nodeuid string) string {
method closeConn (line 236) | func (tp *TP) closeConn(nodeuid string, reconnect bool) {
method closeMsg (line 256) | func (tp *TP) closeMsg(uid, addr string, short bool) {
method send (line 269) | func (tp *TP) send(data *NetData) {
method save (line 292) | func (tp *TP) save(conn *Connect) {
method apiHandle (line 314) | func (tp *TP) apiHandle() {
method autoErrorHandle (line 375) | func (tp *TP) autoErrorHandle(data *NetData, status int, msg string, r...
method checkRights (line 388) | func (tp *TP) checkRights(data *NetData, addr string) bool {
method reserveAPI (line 397) | func (tp *TP) reserveAPI() {
type API (line 66) | type API
type Handle (line 69) | type Handle interface
function New (line 74) | func New() Teleport {
type identity (line 404) | type identity struct
method Process (line 406) | func (*identity) Process(receive *NetData) *NetData {
type heartbeat (line 410) | type heartbeat struct
method Process (line 412) | func (*heartbeat) Process(receive *NetData) *NetData {
FILE: app/distribute/teleport/teleport_test.go
function freePort (line 12) | func freePort(t *testing.T) string {
function TestNew (line 21) | func TestNew(t *testing.T) {
function TestTP_SetUID (line 34) | func TestTP_SetUID(t *testing.T) {
function TestTP_SetPackHeader (line 46) | func TestTP_SetPackHeader(t *testing.T) {
function TestTP_SetApiRChan (line 54) | func TestTP_SetApiRChan(t *testing.T) {
function TestTP_SetConnWChan (line 62) | func TestTP_SetConnWChan(t *testing.T) {
function TestTP_SetConnBuffer (line 70) | func TestTP_SetConnBuffer(t *testing.T) {
function TestTP_SetTimeout (line 78) | func TestTP_SetTimeout(t *testing.T) {
function TestTP_SetAPI (line 87) | func TestTP_SetAPI(t *testing.T) {
function TestTP_ServerClient_Pipe (line 96) | func TestTP_ServerClient_Pipe(t *testing.T) {
type echoHandle (line 124) | type echoHandle struct
method Process (line 126) | func (*echoHandle) Process(receive *NetData) *NetData {
function TestTP_CloseSpecificNode (line 130) | func TestTP_CloseSpecificNode(t *testing.T) {
function TestConnect_Close (line 150) | func TestConnect_Close(t *testing.T) {
function TestTP_CheckRightsReject (line 159) | func TestTP_CheckRightsReject(t *testing.T) {
function TestDebugPrint (line 185) | func TestDebugPrint(t *testing.T) {
function TestTP_GetConnAddr (line 192) | func TestTP_GetConnAddr(t *testing.T) {
FILE: app/distribute/teleport/util.go
function MakeHash (line 14) | func MakeHash(s string) string {
function HashString (line 22) | func HashString(encode string) uint64 {
function MakeUnique (line 29) | func MakeUnique(obj interface{}) string {
function MakeMd5 (line 35) | func MakeMd5(obj interface{}, length int) string {
FILE: app/distribute/teleport/util_test.go
function TestMakeHash (line 7) | func TestMakeHash(t *testing.T) {
function TestHashString (line 24) | func TestHashString(t *testing.T) {
function TestMakeUnique (line 40) | func TestMakeUnique(t *testing.T) {
function TestMakeMd5 (line 56) | func TestMakeMd5(t *testing.T) {
FILE: app/downloader/downloader.go
type Downloader (line 12) | type Downloader interface
FILE: app/downloader/downloader_surfer.go
type Surfer (line 16) | type Surfer struct
method Download (line 35) | func (s *Surfer) Download(sp *spider.Spider, cReq *request.Request) *s...
FILE: app/downloader/downloader_test.go
function TestSurferDownloader_implementsInterface (line 12) | func TestSurferDownloader_implementsInterface(t *testing.T) {
function makeSpiderNotStopping (line 16) | func makeSpiderNotStopping(name string) *spider.Spider {
function TestSurferDownloader_Download_SurfID (line 25) | func TestSurferDownloader_Download_SurfID(t *testing.T) {
function TestSurferDownloader_Download_SurfID_error (line 51) | func TestSurferDownloader_Download_SurfID_error(t *testing.T) {
function TestSurferDownloader_Download_SurfID_4xx (line 70) | func TestSurferDownloader_Download_SurfID_4xx(t *testing.T) {
function TestSurferDownloader_Download_SurfID_badURL (line 89) | func TestSurferDownloader_Download_SurfID_badURL(t *testing.T) {
FILE: app/downloader/request/request.go
type Request (line 20) | type Request struct
method Prepare (line 64) | func (r *Request) Prepare() result.VoidResult {
method Serialize (line 126) | func (r *Request) Serialize() result.Result[string] {
method Unique (line 139) | func (r *Request) Unique() string {
method Copy (line 148) | func (r *Request) Copy() result.Result[*Request] {
method GetURL (line 158) | func (r *Request) GetURL() string {
method GetMethod (line 163) | func (r *Request) GetMethod() string {
method SetMethod (line 168) | func (r *Request) SetMethod(method string) *Request {
method SetURL (line 173) | func (r *Request) SetURL(url string) *Request {
method GetReferer (line 178) | func (r *Request) GetReferer() string {
method SetReferer (line 182) | func (r *Request) SetReferer(referer string) *Request {
method GetPostData (line 187) | func (r *Request) GetPostData() string {
method GetHeader (line 191) | func (r *Request) GetHeader() http.Header {
method SetHeader (line 195) | func (r *Request) SetHeader(key, value string) *Request {
method AddHeader (line 200) | func (r *Request) AddHeader(key, value string) *Request {
method GetEnableCookie (line 205) | func (r *Request) GetEnableCookie() bool {
method SetEnableCookie (line 209) | func (r *Request) SetEnableCookie(enableCookie bool) *Request {
method GetCookies (line 214) | func (r *Request) GetCookies() string {
method SetCookies (line 218) | func (r *Request) SetCookies(cookie string) *Request {
method GetDialTimeout (line 223) | func (r *Request) GetDialTimeout() time.Duration {
method GetConnTimeout (line 227) | func (r *Request) GetConnTimeout() time.Duration {
method GetTryTimes (line 231) | func (r *Request) GetTryTimes() int {
method GetRetryPause (line 235) | func (r *Request) GetRetryPause() time.Duration {
method GetProxy (line 239) | func (r *Request) GetProxy() string {
method SetProxy (line 243) | func (r *Request) SetProxy(proxy string) *Request {
method GetRedirectTimes (line 248) | func (r *Request) GetRedirectTimes() int {
method GetRuleName (line 252) | func (r *Request) GetRuleName() string {
method SetRuleName (line 256) | func (r *Request) SetRuleName(ruleName string) *Request {
method GetSpiderName (line 261) | func (r *Request) GetSpiderName() string {
method SetSpiderName (line 265) | func (r *Request) SetSpiderName(spiderName string) *Request {
method IsReloadable (line 270) | func (r *Request) IsReloadable() bool {
method SetReloadable (line 274) | func (r *Request) SetReloadable(can bool) *Request {
method GetTemp (line 280) | func (r *Request) GetTemp(key string, defaultValue interface{}) interf...
method GetTempOpt (line 299) | func (r *Request) GetTempOpt(key string) option.Option[interface{}] {
method GetTemps (line 314) | func (r *Request) GetTemps() Temp {
method SetTemp (line 318) | func (r *Request) SetTemp(key string, value interface{}) *Request {
method SetTemps (line 326) | func (r *Request) SetTemps(temp map[string]interface{}) *Request {
method GetPriority (line 334) | func (r *Request) GetPriority() int {
method SetPriority (line 338) | func (r *Request) SetPriority(priority int) *Request {
method GetDownloaderID (line 343) | func (r *Request) GetDownloaderID() int {
method SetDownloaderID (line 347) | func (r *Request) SetDownloaderID(id int) *Request {
method MarshalJSON (line 352) | func (r *Request) MarshalJSON() ([]byte, error) {
constant DefaultDialTimeout (line 46) | DefaultDialTimeout = 2 * time.Minute
constant DefaultConnTimeout (line 47) | DefaultConnTimeout = 2 * time.Minute
constant DefaultTryTimes (line 48) | DefaultTryTimes = 3
constant DefaultRetryPause (line 49) | DefaultRetryPause = 2 * time.Second
constant SurfID (line 53) | SurfID = 0
constant PhantomID (line 54) | PhantomID = 1
constant ChromeID (line 55) | ChromeID = 2
function UnSerialize (line 120) | func UnSerialize(s string) result.Result[*Request] {
FILE: app/downloader/request/request_test.go
function TestReqTemp (line 10) | func TestReqTemp(t *testing.T) {
type x (line 59) | type x struct
function TestPrepare (line 63) | func TestPrepare(t *testing.T) {
function TestSerializeUnSerialize (line 137) | func TestSerializeUnSerialize(t *testing.T) {
function TestUnSerializeInvalid (line 166) | func TestUnSerializeInvalid(t *testing.T) {
function TestUnique (line 173) | func TestUnique(t *testing.T) {
function TestCopy (line 183) | func TestCopy(t *testing.T) {
function TestGettersSetters (line 200) | func TestGettersSetters(t *testing.T) {
function TestGetTempOpt (line 330) | func TestGetTempOpt(t *testing.T) {
function TestGetTemps (line 356) | func TestGetTemps(t *testing.T) {
function TestSetTemps (line 365) | func TestSetTemps(t *testing.T) {
function TestGetTempPanic (line 374) | func TestGetTempPanic(t *testing.T) {
FILE: app/downloader/request/temp.go
type Temp (line 11) | type Temp
method get (line 14) | func (t Temp) get(key string, defaultValue interface{}) interface{} {
method set (line 37) | func (t Temp) set(key string, value interface{}) Temp {
FILE: app/downloader/surfer/agent/agent.go
type TemplateData (line 19) | type TemplateData struct
type OSAttributes (line 28) | type OSAttributes struct
constant Windows (line 39) | Windows int = iota
constant Linux (line 41) | Linux
constant Macintosh (line 43) | Macintosh
type Formats (line 57) | type Formats
type UAData (line 60) | type UAData struct
type UATable (line 68) | type UATable
function init (line 153) | func init() {
function CreateReal (line 182) | func CreateReal() string {
function CreateDefault (line 187) | func CreateDefault(browser string) string {
function CreateVersion (line 202) | func CreateVersion(browser, version string) string {
function TopVersion (line 217) | func TopVersion(bname string) string {
function Format (line 231) | func Format(bname, bver string) string {
function createFromDetails (line 250) | func createFromDetails(bname, bver, osname, osver string, c []string) st...
FILE: app/downloader/surfer/agent/agent_bsd.go
function osName (line 12) | func osName() string {
function osVersion (line 21) | func osVersion() string {
FILE: app/downloader/surfer/agent/agent_linux.go
function osName (line 11) | func osName() string {
function osVersion (line 21) | func osVersion() string {
function charsToString (line 31) | func charsToString(ca [65]int8) string {
FILE: app/downloader/surfer/agent/agent_linux_arm.go
function osName (line 11) | func osName() string {
function osVersion (line 21) | func osVersion() string {
function charsToString (line 31) | func charsToString(ca [65]uint8) string {
FILE: app/downloader/surfer/agent/agent_test.go
function TestUserAgentsPopulated (line 8) | func TestUserAgentsPopulated(t *testing.T) {
function TestCreateReal (line 20) | func TestCreateReal(t *testing.T) {
function TestCreateDefault (line 30) | func TestCreateDefault(t *testing.T) {
function TestCreateVersion (line 42) | func TestCreateVersion(t *testing.T) {
function TestTopVersion (line 52) | func TestTopVersion(t *testing.T) {
function TestFormat (line 71) | func TestFormat(t *testing.T) {
function TestFormatFallbackToTopVersion (line 86) | func TestFormatFallbackToTopVersion(t *testing.T) {
function TestFormatUnknownVersion (line 93) | func TestFormatUnknownVersion(t *testing.T) {
function TestCreateVersionVariousBrowsers (line 112) | func TestCreateVersionVariousBrowsers(t *testing.T) {
function TestDatabaseContainsExpectedBrowsers (line 136) | func TestDatabaseContainsExpectedBrowsers(t *testing.T) {
function TestDefaultOSAttributes (line 145) | func TestDefaultOSAttributes(t *testing.T) {
FILE: app/downloader/surfer/agent/agent_windows.go
function osName (line 13) | func osName() string {
function osVersion (line 18) | func osVersion() string {
FILE: app/downloader/surfer/chrome.go
type Chrome (line 26) | type Chrome struct
method ensureBrowser (line 48) | func (c *Chrome) ensureBrowser(ua string) {
method Download (line 86) | func (c *Chrome) Download(req Request) (r result.Result[*http.Response...
function NewChrome (line 36) | func NewChrome(jar ...*cookiejar.Jar) Surfer {
function chromeAllocatorOpts (line 60) | func chromeAllocatorOpts(ua string) []chromedp.ExecAllocatorOption {
function hideWebdriver (line 81) | func hideWebdriver() chromedp.Action {
function tryDownload (line 161) | func tryDownload(ctx context.Context, targetURL string) (string, error) {
function waitUntilNotVerification (line 205) | func waitUntilNotVerification(ctx context.Context, maxWait time.Duration) {
function isVerificationPage (line 217) | func isVerificationPage(ctx context.Context) bool {
FILE: app/downloader/surfer/chrome_stub.go
type ChromeStub (line 13) | type ChromeStub struct
method Download (line 27) | func (c *ChromeStub) Download(req Request) result.Result[*http.Respons...
function NewChrome (line 17) | func NewChrome(jar ...*cookiejar.Jar) Surfer {
FILE: app/downloader/surfer/chrome_test.go
function TestChromeDownloaderBaiduSearch (line 9) | func TestChromeDownloaderBaiduSearch(t *testing.T) {
function TestChromeDownloaderReuseSession (line 45) | func TestChromeDownloaderReuseSession(t *testing.T) {
function TestExtractHomepage (line 75) | func TestExtractHomepage(t *testing.T) {
FILE: app/downloader/surfer/example/example.go
function main (line 11) | func main() {
FILE: app/downloader/surfer/param.go
type Param (line 32) | type Param struct
method writeback (line 111) | func (p *Param) writeback(resp *http.Response) *http.Response {
method checkRedirect (line 133) | func (p *Param) checkRedirect(req *http.Request, via []*http.Request) ...
function NewParam (line 47) | func NewParam(req Request) (r result.Result[*Param]) {
FILE: app/downloader/surfer/param_test.go
function TestNewParam (line 10) | func TestNewParam(t *testing.T) {
function TestNewParamWithProxy (line 76) | func TestNewParamWithProxy(t *testing.T) {
function TestNewParamWithUserAgent (line 91) | func TestNewParamWithUserAgent(t *testing.T) {
function TestRedirectUnlimited (line 106) | func TestRedirectUnlimited(t *testing.T) {
function TestRedirectLimited (line 140) | func TestRedirectLimited(t *testing.T) {
function TestRedirectNotAllowed (line 163) | func TestRedirectNotAllowed(t *testing.T) {
FILE: app/downloader/surfer/phantom.go
type Phantom (line 40) | type Phantom struct
method Download (line 101) | func (p *Phantom) Download(req Request) (r result.Result[*http.Respons...
method DestroyJsFiles (line 208) | func (p *Phantom) DestroyJsFiles() {
method createJsFile (line 221) | func (p *Phantom) createJsFile(fileName, jsCode string) {
type Response (line 47) | type Response struct
type Cookie (line 58) | type Cookie struct
function NewPhantom (line 66) | func NewPhantom(phantomjsFile, tempJsDir string, jar ...*cookiejar.Jar) ...
constant js (line 239) | js string = `
FILE: app/downloader/surfer/phantom_stub.go
type PhantomStub (line 13) | type PhantomStub struct
method Download (line 27) | func (p *PhantomStub) Download(req Request) result.Result[*http.Respon...
method DestroyJsFiles (line 31) | func (p *PhantomStub) DestroyJsFiles() {}
function NewPhantom (line 17) | func NewPhantom(phantomjsFile, tempJsDir string, jar ...*cookiejar.Jar) ...
FILE: app/downloader/surfer/request.go
type Request (line 25) | type Request interface
type DefaultRequest (line 53) | type DefaultRequest struct
method prepare (line 95) | func (dr *DefaultRequest) prepare() {
method GetURL (line 131) | func (dr *DefaultRequest) GetURL() string {
method GetMethod (line 137) | func (dr *DefaultRequest) GetMethod() string {
method GetPostData (line 143) | func (dr *DefaultRequest) GetPostData() string {
method GetHeader (line 149) | func (dr *DefaultRequest) GetHeader() http.Header {
method GetEnableCookie (line 155) | func (dr *DefaultRequest) GetEnableCookie() bool {
method GetDialTimeout (line 161) | func (dr *DefaultRequest) GetDialTimeout() time.Duration {
method GetConnTimeout (line 167) | func (dr *DefaultRequest) GetConnTimeout() time.Duration {
method GetTryTimes (line 173) | func (dr *DefaultRequest) GetTryTimes() int {
method GetRetryPause (line 179) | func (dr *DefaultRequest) GetRetryPause() time.Duration {
method GetProxy (line 185) | func (dr *DefaultRequest) GetProxy() string {
method GetRedirectTimes (line 191) | func (dr *DefaultRequest) GetRedirectTimes() int {
method GetDownloaderID (line 197) | func (dr *DefaultRequest) GetDownloaderID() int {
constant SurfID (line 83) | SurfID = 0
constant PhantomJsID (line 84) | PhantomJsID = 1
constant ChromeID (line 85) | ChromeID = 2
constant PhomtomJsID (line 87) | PhomtomJsID = PhantomJsID
constant DefaultMethod (line 88) | DefaultMethod = "GET"
constant DefaultDialTimeout (line 89) | DefaultDialTimeout = 2 * time.Minute
constant DefaultConnTimeout (line 90) | DefaultConnTimeout = 2 * time.Minute
constant DefaultTryTimes (line 91) | DefaultTryTimes = 3
constant DefaultRetryPause (line 92) | DefaultRetryPause = 2 * time.Second
FILE: app/downloader/surfer/request_test.go
function TestDefaultRequestPrepare (line 9) | func TestDefaultRequestPrepare(t *testing.T) {
function TestDefaultRequestGetters (line 133) | func TestDefaultRequestGetters(t *testing.T) {
FILE: app/downloader/surfer/surf.go
type Surf (line 36) | type Surf struct
method Download (line 52) | func (s *Surf) Download(req Request) (r result.Result[*http.Response]) {
method buildClient (line 103) | func (s *Surf) buildClient(param *Param) *http.Client {
method httpRequest (line 158) | func (s *Surf) httpRequest(param *Param) (resp *http.Response, err err...
function New (line 41) | func New(jar ...*cookiejar.Jar) Surfer {
type DnsCache (line 83) | type DnsCache struct
method Reg (line 88) | func (d *DnsCache) Reg(addr, ipPort string) {
method Del (line 93) | func (d *DnsCache) Del(addr string) {
method Query (line 98) | func (d *DnsCache) Query(addr string) option.Option[string] {
FILE: app/downloader/surfer/surf_stub_test.go
function TestDownloadPhantomJsIDStub (line 7) | func TestDownloadPhantomJsIDStub(t *testing.T) {
function TestDownloadChromeIDStub (line 15) | func TestDownloadChromeIDStub(t *testing.T) {
function TestDestroyJsFilesStub (line 23) | func TestDestroyJsFilesStub(t *testing.T) {
FILE: app/downloader/surfer/surf_test.go
function TestNew (line 17) | func TestNew(t *testing.T) {
function TestSurfDownload (line 33) | func TestSurfDownload(t *testing.T) {
function TestSurfDownloadGzip (line 78) | func TestSurfDownloadGzip(t *testing.T) {
function TestSurfDownloadPOST (line 109) | func TestSurfDownloadPOST(t *testing.T) {
function TestSurfDownloadPOSTM (line 142) | func TestSurfDownloadPOSTM(t *testing.T) {
function TestSurfDownloadRetry (line 176) | func TestSurfDownloadRetry(t *testing.T) {
function TestSurfDownloadWithCookie (line 220) | func TestSurfDownloadWithCookie(t *testing.T) {
function TestSurfDownloadHTTPS (line 245) | func TestSurfDownloadHTTPS(t *testing.T) {
function TestSurfDownloadDeflate (line 273) | func TestSurfDownloadDeflate(t *testing.T) {
function TestSurfDownloadZlib (line 307) | func TestSurfDownloadZlib(t *testing.T) {
function TestDownloadSurfID (line 341) | func TestDownloadSurfID(t *testing.T) {
function TestDestroyJsFiles (line 369) | func TestDestroyJsFiles(t *testing.T) {
function TestDownloadUnknownID (line 373) | func TestDownloadUnknownID(t *testing.T) {
type mockRequest (line 381) | type mockRequest struct
method GetURL (line 385) | func (m *mockRequest) GetURL() string { return "http://...
method GetMethod (line 386) | func (m *mockRequest) GetMethod() string { return "GET" }
method GetPostData (line 387) | func (m *mockRequest) GetPostData() string { return "" }
method GetHeader (line 388) | func (m *mockRequest) GetHeader() http.Header { return nil }
method GetEnableCookie (line 389) | func (m *mockRequest) GetEnableCookie() bool { return false }
method GetDialTimeout (line 390) | func (m *mockRequest) GetDialTimeout() time.Duration { return time.Sec...
method GetConnTimeout (line 391) | func (m *mockRequest) GetConnTimeout() time.Duration { return time.Sec...
method GetTryTimes (line 392) | func (m *mockRequest) GetTryTimes() int { return 1 }
method GetRetryPause (line 393) | func (m *mockRequest) GetRetryPause() time.Duration { return time.Mil...
method GetProxy (line 394) | func (m *mockRequest) GetProxy() string { return "" }
method GetRedirectTimes (line 395) | func (m *mockRequest) GetRedirectTimes() int { return 0 }
method GetDownloaderID (line 396) | func (m *mockRequest) GetDownloaderID() int { return m.downlo...
function TestDnsCache (line 398) | func TestDnsCache(t *testing.T) {
FILE: app/downloader/surfer/surfer.go
function Download (line 42) | func Download(req Request) result.Result[*http.Response] {
function DestroyJsFiles (line 58) | func DestroyJsFiles() {
type Surfer (line 65) | type Surfer interface
FILE: app/downloader/surfer/util.go
function AutoToUTF8 (line 32) | func AutoToUTF8(resp *http.Response) error {
function BodyBytes (line 44) | func BodyBytes(resp *http.Response) ([]byte, error) {
function URLEncode (line 51) | func URLEncode(urlStr string) (*url.URL, error) {
function GetWDPath (line 58) | func GetWDPath() string {
function IsDirExists (line 67) | func IsDirExists(path string) bool {
function IsFileExists (line 77) | func IsFileExists(path string) bool {
function WalkDir (line 87) | func WalkDir(targpath string, suffixes ...string) (dirlist []string) {
function ExtractHomepage (line 120) | func ExtractHomepage(rawURL string) string {
type Body (line 134) | type Body struct
method Read (line 139) | func (b *Body) Read(p []byte) (int, error) {
FILE: app/downloader/surfer/util_test.go
function TestURLEncode (line 14) | func TestURLEncode(t *testing.T) {
function TestBodyBytes (line 36) | func TestBodyBytes(t *testing.T) {
function TestAutoToUTF8 (line 50) | func TestAutoToUTF8(t *testing.T) {
function TestBodyRead (line 70) | func TestBodyRead(t *testing.T) {
function TestIsDirExists (line 86) | func TestIsDirExists(t *testing.T) {
function TestIsFileExists (line 103) | func TestIsFileExists(t *testing.T) {
function TestWalkDir (line 120) | func TestWalkDir(t *testing.T) {
FILE: app/pipeline/collector/collector.go
type Collector (line 18) | type Collector struct
method CollectData (line 49) | func (c *Collector) CollectData(dataCell data.DataCell) (r result.Void...
method CollectFile (line 61) | func (c *Collector) CollectFile(fileCell data.FileCell) (r result.Void...
method Stop (line 73) | func (c *Collector) Stop() {
method Start (line 93) | func (c *Collector) Start() {
method resetDataBuf (line 142) | func (c *Collector) resetDataBuf() {
method dataSum (line 150) | func (c *Collector) dataSum() uint64 {
method addDataSum (line 157) | func (c *Collector) addDataSum(add uint64) {
method fileSum (line 165) | func (c *Collector) fileSum() uint64 {
method addFileSum (line 172) | func (c *Collector) addFileSum(add uint64) {
method Report (line 180) | func (c *Collector) Report() {
function NewCollector (line 34) | func NewCollector(sp *spider.Spider, outType string, batchCap int) *Coll...
FILE: app/pipeline/collector/collector_test.go
function TestNewCollector (line 16) | func TestNewCollector(t *testing.T) {
function TestCollector_CollectData (line 46) | func TestCollector_CollectData(t *testing.T) {
function TestCollector_CollectFile (line 62) | func TestCollector_CollectFile(t *testing.T) {
function TestCollector_Stop (line 78) | func TestCollector_Stop(t *testing.T) {
function TestCollector_OutputData_EmptyBuf (line 88) | func TestCollector_OutputData_EmptyBuf(t *testing.T) {
function TestCollector_OutputData_PanicRecovery (line 97) | func TestCollector_OutputData_PanicRecovery(t *testing.T) {
function TestCollector_OutputData_ErrorResult (line 115) | func TestCollector_OutputData_ErrorResult(t *testing.T) {
function TestCollector_OutputCSV (line 133) | func TestCollector_OutputCSV(t *testing.T) {
function TestCollector_OutputCSV_NonStringData (line 194) | func TestCollector_OutputCSV_NonStringData(t *testing.T) {
function TestCollector_OutputCSV_NotDefaultField (line 232) | func TestCollector_OutputCSV_NotDefaultField(t *testing.T) {
function TestCollector_OutputExcel (line 271) | func TestCollector_OutputExcel(t *testing.T) {
function TestCollector_OutputFile (line 316) | func TestCollector_OutputFile(t *testing.T) {
function readFile (line 354) | func readFile(path string) (string, error) {
function TestCollector_OutputFile_MkdirFail (line 362) | func TestCollector_OutputFile_MkdirFail(t *testing.T) {
FILE: app/pipeline/collector/data/data.go
constant FieldRuleName (line 9) | FieldRuleName = "RuleName"
constant FieldURL (line 10) | FieldURL = "Url"
constant FieldParentURL (line 11) | FieldParentURL = "ParentUrl"
constant FieldDownloadTime (line 12) | FieldDownloadTime = "DownloadTime"
type DataCell (line 17) | type DataCell
type FileCell (line 20) | type FileCell
function GetDataCell (line 37) | func GetDataCell(ruleName string, data map[string]interface{}, url strin...
function GetFileCell (line 48) | func GetFileCell(ruleName, name string, bytes []byte) FileCell {
function PutDataCell (line 57) | func PutDataCell(cell DataCell) {
function PutFileCell (line 67) | func PutFileCell(cell FileCell) {
FILE: app/pipeline/collector/data/data_test.go
function TestGetDataCell (line 7) | func TestGetDataCell(t *testing.T) {
function TestGetFileCell (line 29) | func TestGetFileCell(t *testing.T) {
function TestPutDataCell (line 44) | func TestPutDataCell(t *testing.T) {
function TestPutFileCell (line 55) | func TestPutFileCell(t *testing.T) {
function TestPoolReuseDataCell (line 69) | func TestPoolReuseDataCell(t *testing.T) {
FILE: app/pipeline/collector/output_beanstalkd.go
function init (line 18) | func init() {
FILE: app/pipeline/collector/output_beanstalkd_stub.go
function init (line 9) | func init() {
FILE: app/pipeline/collector/output_csv.go
function init (line 16) | func init() {
FILE: app/pipeline/collector/output_data.go
type Refresher (line 10) | type Refresher interface
method outputData (line 26) | func (c *Collector) outputData() {
function Register (line 60) | func Register(outType string, outFunc func(col *Collector) result.VoidRe...
function RegisterRefresher (line 65) | func RegisterRefresher(outType string, r Refresher) {
function RefreshBackend (line 70) | func RefreshBackend(outType string) {
FILE: app/pipeline/collector/output_data_test.go
function TestRegister (line 9) | func TestRegister(t *testing.T) {
function TestRegisterRefresher (line 16) | func TestRegisterRefresher(t *testing.T) {
function TestRefreshBackend_Unregistered (line 25) | func TestRefreshBackend_Unregistered(t *testing.T) {
type testRefresher (line 29) | type testRefresher struct
method Refresh (line 33) | func (t *testRefresher) Refresh() {
FILE: app/pipeline/collector/output_excel.go
function init (line 17) | func init() {
FILE: app/pipeline/collector/output_file.go
method outputFile (line 20) | func (c *Collector) outputFile(file data.FileCell) {
FILE: app/pipeline/collector/output_kafka.go
function init (line 17) | func init() {
FILE: app/pipeline/collector/output_kafka_stub.go
function init (line 9) | func init() {
FILE: app/pipeline/collector/output_mgo.go
function init (line 18) | func init() {
FILE: app/pipeline/collector/output_mgo_stub.go
function init (line 9) | func init() {
FILE: app/pipeline/collector/output_mysql.go
function init (line 15) | func init() {
FILE: app/pipeline/collector/output_mysql_stub.go
function init (line 9) | func init() {
FILE: app/pipeline/collector/output_util.go
method namespace (line 8) | func (c *Collector) namespace() string {
method subNamespace (line 19) | func (c *Collector) subNamespace(dataCell map[string]interface{}) string {
function joinNamespaces (line 32) | func joinNamespaces(namespace, subNamespace string) string {
FILE: app/pipeline/collector/output_util_test.go
function TestJoinNamespaces (line 10) | func TestJoinNamespaces(t *testing.T) {
function TestCollector_Namespace (line 30) | func TestCollector_Namespace(t *testing.T) {
function TestCollector_SubNamespace (line 71) | func TestCollector_SubNamespace(t *testing.T) {
FILE: app/pipeline/output.go
function init (line 15) | func init() {
type refresherFunc (line 27) | type refresherFunc
method Refresh (line 29) | func (f refresherFunc) Refresh() { f() }
function RegisterOutput (line 32) | func RegisterOutput(name string, fn func(*collector.Collector) result.Vo...
function GetOutputLib (line 39) | func GetOutputLib() []string {
function RefreshOutput (line 44) | func RefreshOutput() {
FILE: app/pipeline/pipeline.go
type Pipeline (line 12) | type Pipeline interface
function New (line 20) | func New(sp *spider.Spider, outType string, batchCap int) Pipeline {
FILE: app/pipeline/pipeline_test.go
function TestNew (line 13) | func TestNew(t *testing.T) {
function TestGetOutputLib (line 50) | func TestGetOutputLib(t *testing.T) {
function TestRegisterOutput (line 62) | func TestRegisterOutput(t *testing.T) {
function TestRefreshOutput (line 86) | func TestRefreshOutput(t *testing.T) {
function TestRefresherFunc (line 96) | func TestRefresherFunc(t *testing.T) {
function TestPipeline_StartStopCollect (line 105) | func TestPipeline_StartStopCollect(t *testing.T) {
FILE: app/scheduler/matrix.go
type Matrix (line 18) | type Matrix struct
method Push (line 51) | func (m *Matrix) Push(req *request.Request) {
method Pull (line 101) | func (m *Matrix) Pull() (req *request.Request) {
method Use (line 127) | func (m *Matrix) Use() {
method Free (line 138) | func (m *Matrix) Free() {
method DoHistory (line 144) | func (m *Matrix) DoHistory(req *request.Request, ok bool) bool {
method CanStop (line 172) | func (m *Matrix) CanStop() bool {
method TryFlushSuccess (line 207) | func (m *Matrix) TryFlushSuccess() {
method TryFlushFailure (line 214) | func (m *Matrix) TryFlushFailure() {
method Wait (line 221) | func (m *Matrix) Wait() {
method Len (line 231) | func (m *Matrix) Len() int {
method hasHistory (line 241) | func (m *Matrix) hasHistory(reqUnique string) bool {
method insertTempHistory (line 251) | func (m *Matrix) insertTempHistory(reqUnique string) {
method setFailures (line 257) | func (m *Matrix) setFailures(reqs map[string]*request.Request) {
function newMatrix (line 32) | func newMatrix(spiderName, spiderSubName string, maxPage int64) *Matrix {
FILE: app/scheduler/scheduler.go
type scheduler (line 14) | type scheduler struct
method avgRes (line 93) | func (sched *scheduler) avgRes() int32 {
method checkStatus (line 101) | func (sched *scheduler) checkStatus(s int) bool {
function Init (line 31) | func Init(threadNum int, proxyMinute int64) {
function ReloadProxyLib (line 53) | func ReloadProxyLib() {
function AddMatrix (line 58) | func AddMatrix(spiderName, spiderSubName string, maxPage int64) *Matrix {
function PauseRecover (line 67) | func PauseRecover() {
function Stop (line 79) | func Stop() {
FILE: app/scheduler/scheduler_test.go
function makeReq (line 11) | func makeReq(url, rule string) *request.Request {
function TestInit (line 17) | func TestInit(t *testing.T) {
function TestAddMatrix (line 33) | func TestAddMatrix(t *testing.T) {
function TestPauseRecover (line 56) | func TestPauseRecover(t *testing.T) {
function TestReloadProxyLib (line 62) | func TestReloadProxyLib(t *testing.T) {
function TestMatrix_PushPull_Len (line 66) | func TestMatrix_PushPull_Len(t *testing.T) {
function TestMatrix_Push_ignored_when_maxPage_non_negative (line 103) | func TestMatrix_Push_ignored_when_maxPage_non_negative(t *testing.T) {
function TestMatrix_Pull_empty_returns_nil (line 113) | func TestMatrix_Pull_empty_returns_nil(t *testing.T) {
function TestMatrix_Pull_paused_returns_nil (line 121) | func TestMatrix_Pull_paused_returns_nil(t *testing.T) {
function TestMatrix_Use_Free (line 133) | func TestMatrix_Use_Free(t *testing.T) {
function TestMatrix_DoHistory (line 140) | func TestMatrix_DoHistory(t *testing.T) {
function TestMatrix_DoHistory_reloadable (line 168) | func TestMatrix_DoHistory_reloadable(t *testing.T) {
function TestMatrix_CanStop (line 187) | func TestMatrix_CanStop(t *testing.T) {
function TestMatrix_CanStop_after_Stop (line 218) | func TestMatrix_CanStop_after_Stop(t *testing.T) {
function TestMatrix_TryFlushSuccess_Failure (line 230) | func TestMatrix_TryFlushSuccess_Failure(t *testing.T) {
function TestMatrix_Wait (line 245) | func TestMatrix_Wait(t *testing.T) {
function TestMatrix_Push_duplicate_skipped (line 251) | func TestMatrix_Push_duplicate_skipped(t *testing.T) {
function TestMatrix_Pull_priority (line 262) | func TestMatrix_Pull_priority(t *testing.T) {
function TestMatrix_Pull_request_with_proxy_passthrough (line 280) | func TestMatrix_Pull_request_with_proxy_passthrough(t *testing.T) {
FILE: app/spider/common/common.go
function CleanHtml (line 19) | func CleanHtml(str string, depth int) string {
function ExtractArticle (line 45) | func ExtractArticle(html string) string {
function Deprive (line 80) | func Deprive(s string) string {
function Deprive2 (line 89) | func Deprive2(s string) string {
function Floor (line 101) | func Floor(f float64, n int) float64 {
function SplitCookies (line 107) | func SplitCookies(cookieStr string) (cookies []*http.Cookie) {
function DecodeString (line 121) | func DecodeString(src, charset string) string {
function EncodeString (line 125) | func EncodeString(src, charset string) string {
function ConvertToString (line 129) | func ConvertToString(src string, srcCode string, tagCode string) string {
function GBKToUTF8 (line 138) | func GBKToUTF8(src string) string {
function UnicodeToUTF8 (line 143) | func UnicodeToUTF8(str string) string {
function Unicode16ToUTF8 (line 157) | func Unicode16ToUTF8(str string) string {
function MakeUrl (line 178) | func MakeUrl(path string, schemeAndHost ...string) (string, bool) {
function Pinger (line 198) | func Pinger(address string, timeoutSecond int) result.VoidResult {
function Ping (line 202) | func Ping(address string, timeoutSecond int) result.Result[ping.PingResu...
function ProcessHtml (line 210) | func ProcessHtml(html string) string {
function DepriveBreak (line 216) | func DepriveBreak(s string) string {
function DepriveMutiBreak (line 227) | func DepriveMutiBreak(s string) string {
function HrefSub (line 234) | func HrefSub(src string, sub string) string {
function GetHref (line 248) | func GetHref(baseURL string, url string, href string, mustBase bool) str...
FILE: app/spider/common/common_test.go
function TestCleanHtml (line 7) | func TestCleanHtml(t *testing.T) {
function TestDeprive (line 30) | func TestDeprive(t *testing.T) {
function TestDeprive2 (line 45) | func TestDeprive2(t *testing.T) {
function TestFloor (line 62) | func TestFloor(t *testing.T) {
function TestSplitCookies (line 79) | func TestSplitCookies(t *testing.T) {
function TestDecodeString (line 106) | func TestDecodeString(t *testing.T) {
function TestEncodeString (line 122) | func TestEncodeString(t *testing.T) {
function TestConvertToString (line 138) | func TestConvertToString(t *testing.T) {
function TestGBKToUTF8 (line 155) | func TestGBKToUTF8(t *testing.T) {
function TestUnicodeToUTF8 (line 170) | func TestUnicodeToUTF8(t *testing.T) {
function TestUnicode16ToUTF8 (line 186) | func TestUnicode16ToUTF8(t *testing.T) {
function TestMakeUrl (line 203) | func TestMakeUrl(t *testing.T) {
function TestProcessHtml (line 224) | func TestProcessHtml(t *testing.T) {
function TestDepriveBreak (line 240) | func TestDepriveBreak(t *testing.T) {
function TestDepriveMutiBreak (line 255) | func TestDepriveMutiBreak(t *testing.T) {
function TestHrefSub (line 270) | func TestHrefSub(t *testing.T) {
function TestGetHref (line 287) | func TestGetHref(t *testing.T) {
function TestExtractArticle (line 317) | func TestExtractArticle(t *testing.T) {
function TestPinger (line 330) | func TestPinger(t *testing.T) {
function TestPing (line 337) | func TestPing(t *testing.T) {
FILE: app/spider/common/form.go
type Form (line 14) | type Form struct
method Method (line 50) | func (f *Form) Method() string {
method Action (line 56) | func (f *Form) Action() string {
method Input (line 61) | func (f *Form) Input(name, value string) *Form {
method Inputs (line 69) | func (f *Form) Inputs(kv map[string]string) *Form {
method Submit (line 81) | func (f *Form) Submit() bool {
method Click (line 91) | func (f *Form) Click(button string) bool {
method Dom (line 99) | func (f *Form) Dom() *goquery.Selection {
method send (line 104) | func (f *Form) send(buttonName, buttonValue string) bool {
function NewForm (line 25) | func NewForm(ctx *spider.Context, rule string, u string, form *goquery.S...
function serializeForm (line 145) | func serializeForm(sel *goquery.Selection) (url.Values, url.Values) {
function formAttributes (line 171) | func formAttributes(u string, form *goquery.Selection, schemeAndHost ......
FILE: app/spider/common/form_test.go
function TestNewForm (line 11) | func TestNewForm(t *testing.T) {
function TestNewForm_Post (line 51) | func TestNewForm_Post(t *testing.T) {
function TestNewForm_Multipart (line 74) | func TestNewForm_Multipart(t *testing.T) {
function TestNewForm_NoSchemeAndHost (line 94) | func TestNewForm_NoSchemeAndHost(t *testing.T) {
FILE: app/spider/context.go
type Context (line 25) | type Context struct
method SetResponse (line 77) | func (ctx *Context) SetResponse(resp *http.Response) *Context {
method SetError (line 83) | func (ctx *Context) SetError(err error) {
method AddQueue (line 104) | func (ctx *Context) AddQueue(req *request.Request) *Context {
method JsAddQueue (line 141) | func (ctx *Context) JsAddQueue(jreq map[string]interface{}) *Context {
method Output (line 214) | func (ctx *Context) Output(item interface{}, ruleName ...string) {
method FileOutput (line 247) | func (ctx *Context) FileOutput(nameOrExt ...string) {
method CreateItem (line 288) | func (ctx *Context) CreateItem(item map[int]interface{}, ruleName ...s...
method SetTemp (line 304) | func (ctx *Context) SetTemp(key string, value interface{}) *Context {
method SetURL (line 309) | func (ctx *Context) SetURL(url string) *Context {
method SetReferer (line 314) | func (ctx *Context) SetReferer(referer string) *Context {
method UpsertItemField (line 322) | func (ctx *Context) UpsertItemField(field string, ruleName ...string) ...
method Aid (line 333) | func (ctx *Context) Aid(aid map[string]interface{}, ruleName ...string...
method Parse (line 356) | func (ctx *Context) Parse(ruleName ...string) *Context {
method SetKeyin (line 378) | func (ctx *Context) SetKeyin(keyin string) *Context {
method SetLimit (line 384) | func (ctx *Context) SetLimit(max int) *Context {
method SetPausetime (line 391) | func (ctx *Context) SetPausetime(pause int64, runtime ...bool) *Context {
method SetTimer (line 399) | func (ctx *Context) SetTimer(id string, tol time.Duration, bell *Bell)...
method RunTimer (line 404) | func (ctx *Context) RunTimer(id string) bool {
method ResetText (line 409) | func (ctx *Context) ResetText(body string) *Context {
method GetError (line 420) | func (ctx *Context) GetError() error {
method Log (line 428) | func (*Context) Log() logs.Logs {
method GetSpider (line 433) | func (ctx *Context) GetSpider() *Spider {
method GetResponse (line 438) | func (ctx *Context) GetResponse() *http.Response {
method GetStatusCode (line 443) | func (ctx *Context) GetStatusCode() int {
method GetRequest (line 451) | func (ctx *Context) GetRequest() *request.Request {
method CopyRequest (line 456) | func (ctx *Context) CopyRequest() *request.Request {
method GetItemFields (line 461) | func (ctx *Context) GetItemFields(ruleName ...string) []string {
method GetItemField (line 472) | func (ctx *Context) GetItemField(index int, ruleName ...string) (field...
method GetItemFieldIndex (line 483) | func (ctx *Context) GetItemFieldIndex(field string, ruleName ...string...
method PullItems (line 493) | func (ctx *Context) PullItems() (ds []data.DataCell) {
method PullFiles (line 502) | func (ctx *Context) PullFiles() (fs []data.FileCell) {
method GetKeyin (line 511) | func (ctx *Context) GetKeyin() string {
method GetLimit (line 516) | func (ctx *Context) GetLimit() int {
method GetName (line 521) | func (ctx *Context) GetName() string {
method GetRules (line 526) | func (ctx *Context) GetRules() map[string]*Rule {
method GetRule (line 531) | func (ctx *Context) GetRule(ruleName string) *Rule {
method GetRuleName (line 536) | func (ctx *Context) GetRuleName() string {
method GetTemp (line 542) | func (ctx *Context) GetTemp(key string, defaultValue interface{}) inte...
method GetTemps (line 547) | func (ctx *Context) GetTemps() request.Temp {
method CopyTemps (line 552) | func (ctx *Context) CopyTemps() request.Temp {
method GetURL (line 561) | func (ctx *Context) GetURL() string {
method GetMethod (line 566) | func (ctx *Context) GetMethod() string {
method GetHost (line 571) | func (ctx *Context) GetHost() string {
method GetHeader (line 579) | func (ctx *Context) GetHeader() http.Header {
method GetRequestHeader (line 587) | func (ctx *Context) GetRequestHeader() http.Header {
method GetReferer (line 595) | func (ctx *Context) GetReferer() string {
method GetCookie (line 603) | func (ctx *Context) GetCookie() string {
method GetDom (line 612) | func (ctx *Context) GetDom() *goquery.Document {
method GetText (line 631) | func (ctx *Context) GetText() string {
method getRule (line 649) | func (ctx *Context) getRule(ruleName ...string) (name string, rule *Ru...
method initDom (line 663) | func (ctx *Context) initDom() (*goquery.Document, error) {
method initText (line 678) | func (ctx *Context) initText() error {
function GetContext (line 51) | func GetContext(sp *Spider, req *request.Request) *Context {
function PutContext (line 59) | func PutContext(ctx *Context) {
function jsToInt64 (line 128) | func jsToInt64(v interface{}) (int64, bool) {
function detectCharset (line 703) | func detectCharset(responseContentType, requestContentType string) string {
function isUTF8 (line 714) | func isUTF8(charset string) bool {
function convertEncoding (line 723) | func convertEncoding(body []byte, charsetLabel string) ([]byte, error) {
FILE: app/spider/parsejs.go
function evalScript (line 27) | func evalScript(vm *otto.Otto, script string) (r result.Result[otto.Valu...
type SpiderModle (line 36) | type SpiderModle struct
type RuleModle (line 50) | type RuleModle struct
function RegisterDynamicSpiders (line 60) | func RegisterDynamicSpiders() {
function doRegisterDynamicSpiders (line 64) | func doRegisterDynamicSpiders() {
function wrapScriptCDATA (line 154) | func wrapScriptCDATA(data []byte) []byte {
function getSpiderModles (line 173) | func getSpiderModles() (ms []*SpiderModle) {
FILE: app/spider/species.go
type SpiderSpecies (line 12) | type SpiderSpecies struct
method Add (line 25) | func (ss *SpiderSpecies) Add(sp *Spider) *Spider {
method Get (line 42) | func (ss *SpiderSpecies) Get() []*Spider {
method GetByNameOpt (line 62) | func (ss *SpiderSpecies) GetByNameOpt(name string) option.Option[*Spid...
FILE: app/spider/species_test.go
function TestSpiderSpecies_Add (line 7) | func TestSpiderSpecies_Add(t *testing.T) {
function TestSpiderSpecies_Get (line 34) | func TestSpiderSpecies_Get(t *testing.T) {
function TestSpiderSpecies_GetByNameOpt (line 53) | func TestSpiderSpecies_GetByNameOpt(t *testing.T) {
function TestSpider_GetName (line 75) | func TestSpider_GetName(t *testing.T) {
function TestSpider_GetSubName (line 91) | func TestSpider_GetSubName(t *testing.T) {
function TestSpider_GetRule (line 102) | func TestSpider_GetRule(t *testing.T) {
function TestSpider_MustGetRule (line 115) | func TestSpider_MustGetRule(t *testing.T) {
function TestSpider_GetRules (line 130) | func TestSpider_GetRules(t *testing.T) {
function TestSpider_GetItemFields (line 139) | func TestSpider_GetItemFields(t *testing.T) {
function TestSpider_GetItemField (line 148) | func TestSpider_GetItemField(t *testing.T) {
function TestSpider_GetItemFieldIndex (line 167) | func TestSpider_GetItemFieldIndex(t *testing.T) {
function TestSpider_UpsertItemField (line 186) | func TestSpider_UpsertItemField(t *testing.T) {
function TestSpider_GetID_SetID (line 200) | func TestSpider_GetID_SetID(t *testing.T) {
function TestSpider_GetKeyin_SetKeyin (line 208) | func TestSpider_GetKeyin_SetKeyin(t *testing.T) {
function TestSpider_GetLimit_SetLimit (line 216) | func TestSpider_GetLimit_SetLimit(t *testing.T) {
function TestSpider_GetEnableCookie (line 224) | func TestSpider_GetEnableCookie(t *testing.T) {
function TestSpider_GetDescription (line 231) | func TestSpider_GetDescription(t *testing.T) {
function TestSpider_SetPausetime (line 238) | func TestSpider_SetPausetime(t *testing.T) {
function TestSpider_OutDefaultField (line 254) | func TestSpider_OutDefaultField(t *testing.T) {
function TestSpider_Copy (line 265) | func TestSpider_Copy(t *testing.T) {
FILE: app/spider/spider.go
constant KEYIN (line 19) | KEYIN = util.USE_KEYIN
constant LIMIT (line 20) | LIMIT = math.MaxInt64
constant FORCED_STOP (line 21) | FORCED_STOP = "-- Forced stop of Spider --"
type Spider (line 26) | type Spider struct
method Register (line 62) | func (sp *Spider) Register() *Spider {
method GetItemFields (line 68) | func (sp *Spider) GetItemFields(rule *Rule) []string {
method GetItemField (line 73) | func (sp *Spider) GetItemField(rule *Rule, index int) (field string) {
method GetItemFieldIndex (line 81) | func (sp *Spider) GetItemFieldIndex(rule *Rule, field string) (index i...
method UpsertItemField (line 92) | func (sp *Spider) UpsertItemField(rule *Rule, field string) (index int) {
method GetName (line 103) | func (sp *Spider) GetName() string {
method GetSubName (line 108) | func (sp *Spider) GetSubName() string {
method GetRule (line 117) | func (sp *Spider) GetRule(ruleName string) *Rule {
method MustGetRule (line 126) | func (sp *Spider) MustGetRule(ruleName string) *Rule {
method GetRules (line 135) | func (sp *Spider) GetRules() map[string]*Rule {
method GetDescription (line 140) | func (sp *Spider) GetDescription() string {
method GetID (line 145) | func (sp *Spider) GetID() int {
method SetID (line 150) | func (sp *Spider) SetID(id int) {
method GetKeyin (line 155) | func (sp *Spider) GetKeyin() string {
method SetKeyin (line 160) | func (sp *Spider) SetKeyin(keyword string) {
method GetLimit (line 166) | func (sp *Spider) GetLimit() int64 {
method SetLimit (line 171) | func (sp *Spider) SetLimit(max int64) {
method GetEnableCookie (line 176) | func (sp *Spider) GetEnableCookie() bool {
method SetPausetime (line 181) | func (sp *Spider) SetPausetime(pause int64, runtime ...bool) {
method SetTimer (line 189) | func (sp *Spider) SetTimer(id string, tol time.Duration, bell *Bell) b...
method RunTimer (line 197) | func (sp *Spider) RunTimer(id string) bool {
method Copy (line 205) | func (sp *Spider) Copy() *Spider {
method ReqmatrixInit (line 241) | func (sp *Spider) ReqmatrixInit() *Spider {
method DoHistory (line 252) | func (sp *Spider) DoHistory(req *request.Request, ok bool) bool {
method RequestPush (line 257) | func (sp *Spider) RequestPush(req *request.Request) {
method RequestPull (line 262) | func (sp *Spider) RequestPull() *request.Request {
method RequestUse (line 266) | func (sp *Spider) RequestUse() {
method RequestFree (line 270) | func (sp *Spider) RequestFree() {
method RequestLen (line 274) | func (sp *Spider) RequestLen() int {
method TryFlushSuccess (line 278) | func (sp *Spider) TryFlushSuccess() {
method TryFlushFailure (line 282) | func (sp *Spider) TryFlushFailure() {
method Start (line 287) | func (sp *Spider) Start() {
method Stop (line 300) | func (sp *Spider) Stop() {
method CanStop (line 314) | func (sp *Spider) CanStop() bool {
method IsStopping (line 321) | func (sp *Spider) IsStopping() bool {
method tryStop (line 328) | func (sp *Spider) tryStop() error {
method Defer (line 336) | func (sp *Spider) Defer() {
method OutDefaultField (line 346) | func (sp *Spider) OutDefaultField() bool {
type RuleTree (line 49) | type RuleTree struct
type Rule (line 54) | type Rule struct
FILE: app/spider/timer.go
type Timer (line 11) | type Timer struct
method sleep (line 24) | func (t *Timer) sleep(id string) bool {
method set (line 51) | func (t *Timer) set(id string, tol time.Duration, bell *Bell) bool {
method drop (line 69) | func (t *Timer) drop() {
function newTimer (line 17) | func newTimer() *Timer {
type Clock (line 81) | type Clock struct
method sleep (line 127) | func (c *Clock) sleep() {
method wake (line 137) | func (c *Clock) wake() {
method duration (line 141) | func (c *Clock) duration() time.Duration {
type Bell (line 89) | type Bell struct
constant A (line 97) | A = iota
constant T (line 98) | T
function newClock (line 103) | func newClock(id string, tol time.Duration, bell *Bell) (*Clock, bool) {
function newT (line 159) | func newT() *time.Timer {
FILE: app/spider/timer_test.go
function TestTimer1 (line 8) | func TestTimer1(t *testing.T) {
function TestTimer2 (line 17) | func TestTimer2(t *testing.T) {
FILE: cmd/cmd_test.go
function TestFlag (line 8) | func TestFlag(t *testing.T) {
FILE: cmd/pholcus-cmd.go
function Flag (line 25) | func Flag() {
function Run (line 47) | func Run() {
function run (line 68) | func run() {
function parseInput (line 89) | func parseInput() {
FILE: common/beanstalkd/beanstalkd.go
type BeanstalkdClient (line 15) | type BeanstalkdClient struct
method Close (line 44) | func (srv *BeanstalkdClient) Close() {
method Send (line 51) | func (srv *BeanstalkdClient) Send(content url.Values) result.VoidResult {
function New (line 21) | func New() result.Result[*BeanstalkdClient] {
function NewFromConfig (line 26) | func NewFromConfig(cfg config.BeanstalkdConfig) result.Result[*Beanstalk...
FILE: common/beanstalkd/beanstalkd_test.go
function TestNewFromConfig_EmptyHost (line 12) | func TestNewFromConfig_EmptyHost(t *testing.T) {
function TestNewFromConfig_EmptyTube (line 22) | func TestNewFromConfig_EmptyTube(t *testing.T) {
function TestNewFromConfig_ConnectionError (line 32) | func TestNewFromConfig_ConnectionError(t *testing.T) {
function TestClose_NilConn (line 42) | func TestClose_NilConn(t *testing.T) {
function TestSend_NilConn (line 47) | func TestSend_NilConn(t *testing.T) {
function TestSend_EmptyValues (line 55) | func TestSend_EmptyValues(t *testing.T) {
function TestClose_WithConn (line 63) | func TestClose_WithConn(t *testing.T) {
function TestSend_PutError (line 71) | func TestSend_PutError(t *testing.T) {
function TestNew (line 83) | func TestNew(t *testing.T) {
FILE: common/bytes/bytes.go
type Bytes (line 11) | type Bytes struct
method Format (line 37) | func (*Bytes) Format(b uint64) string {
method Parse (line 66) | func (*Bytes) Parse(value string) (i uint64, err error) {
constant B (line 16) | B = 1 << (10 * iota)
constant KB (line 17) | KB
constant MB (line 18) | MB
constant GB (line 19) | GB
constant TB (line 20) | TB
constant PB (line 21) | PB
constant EB (line 22) | EB
function New (line 31) | func New() *Bytes {
function Format (line 97) | func Format(b uint64) string {
function Parse (line 102) | func Parse(val string) (uint64, error) {
FILE: common/bytes/bytes_test.go
function TestFormat (line 7) | func TestFormat(t *testing.T) {
function TestParse (line 36) | func TestParse(t *testing.T) {
function TestConstants (line 72) | func TestConstants(t *testing.T) {
FILE: common/closer/closer.go
type LogFunc (line 11) | type LogFunc
function LogClose (line 15) | func LogClose(c io.Closer, logf LogFunc) {
FILE: common/closer/closer_test.go
type mockCloser (line 8) | type mockCloser struct
method Close (line 12) | func (m *mockCloser) Close() error { return m.err }
function TestLogClose_NoError (line 14) | func TestLogClose_NoError(t *testing.T) {
function TestLogClose_WithError (line 22) | func TestLogClose_WithError(t *testing.T) {
FILE: common/gc/gc.go
constant GC_SIZE (line 12) | GC_SIZE = 50 << 20
function ManualGC (line 22) | func ManualGC() {
FILE: common/gc/gc_test.go
function TestGCSizeConstant (line 7) | func TestGCSizeConstant(t *testing.T) {
function TestManualGCDoesNotPanic (line 14) | func TestManualGCDoesNotPanic(t *testing.T) {
FILE: common/goquery/array.go
method First (line 10) | func (s *Selection) First() *Selection {
method Last (line 17) | func (s *Selection) Last() *Selection {
method Eq (line 25) | func (s *Selection) Eq(index int) *Selection {
method Slice (line 39) | func (s *Selection) Slice(start, end int) *Selection {
method Get (line 52) | func (s *Selection) Get(index int) *html.Node {
method Index (line 61) | func (s *Selection) Index() int {
method IndexSelector (line 71) | func (s *Selection) IndexSelector(selector string) int {
method IndexMatcher (line 82) | func (s *Selection) IndexMatcher(m Matcher) int {
method IndexOfNode (line 92) | func (s *Selection) IndexOfNode(node *html.Node) int {
method IndexOfSelection (line 98) | func (s *Selection) IndexOfSelection(sel *Selection) int {
FILE: common/goquery/array_test.go
function TestFirst (line 7) | func TestFirst(t *testing.T) {
function TestFirstEmpty (line 12) | func TestFirstEmpty(t *testing.T) {
function TestFirstInvalid (line 17) | func TestFirstInvalid(t *testing.T) {
function TestFirstRollback (line 22) | func TestFirstRollback(t *testing.T) {
function TestLast (line 28) | func TestLast(t *testing.T) {
function TestLastEmpty (line 39) | func TestLastEmpty(t *testing.T) {
function TestLastInvalid (line 44) | func TestLastInvalid(t *testing.T) {
function TestLastRollback (line 49) | func TestLastRollback(t *testing.T) {
function TestEq (line 55) | func TestEq(t *testing.T) {
function TestEqNegative (line 60) | func TestEqNegative(t *testing.T) {
function TestEqEmpty (line 71) | func TestEqEmpty(t *testing.T) {
function TestEqInvalid (line 76) | func TestEqInvalid(t *testing.T) {
function TestEqInvalidPositive (line 81) | func TestEqInvalidPositive(t *testing.T) {
function TestEqInvalidNegative (line 86) | func TestEqInvalidNegative(t *testing.T) {
function TestEqRollback (line 91) | func TestEqRollback(t *testing.T) {
function TestSlice (line 97) | func TestSlice(t *testing.T) {
function TestSliceEmpty (line 103) | func TestSliceEmpty(t *testing.T) {
function TestSliceInvalid (line 108) | func TestSliceInvalid(t *testing.T) {
function TestSliceOutOfBounds (line 113) | func TestSliceOutOfBounds(t *testing.T) {
function TestNegativeSliceStart (line 118) | func TestNegativeSliceStart(t *testing.T) {
function TestNegativeSliceEnd (line 124) | func TestNegativeSliceEnd(t *testing.T) {
function TestNegativeSliceBoth (line 131) | func TestNegativeSliceBoth(t *testing.T) {
function TestNegativeSliceOutOfBounds (line 138) | func TestNegativeSliceOutOfBounds(t *testing.T) {
function TestSliceRollback (line 143) | func TestSliceRollback(t *testing.T) {
function TestGet (line 149) | func TestGet(t *testing.T) {
function TestGetNegative (line 157) | func TestGetNegative(t *testing.T) {
function TestGetInvalid (line 165) | func TestGetInvalid(t *testing.T) {
function TestIndex (line 171) | func TestIndex(t *testing.T) {
function TestIndexSelector (line 178) | func TestIndexSelector(t *testing.T) {
function TestIndexSelectorInvalid (line 185) | func TestIndexSelectorInvalid(t *testing.T) {
function TestIndexOfNode (line 192) | func TestIndexOfNode(t *testing.T) {
function TestIndexOfNilNode (line 199) | func TestIndexOfNilNode(t *testing.T) {
function TestIndexOfSelection (line 206) | func TestIndexOfSelection(t *testing.T) {
FILE: common/goquery/bench_array_test.go
function BenchmarkFirst (line 7) | func BenchmarkFirst(b *testing.B) {
function BenchmarkLast (line 16) | func BenchmarkLast(b *testing.B) {
function BenchmarkEq (line 25) | func BenchmarkEq(b *testing.B) {
function BenchmarkSlice (line 38) | func BenchmarkSlice(b *testing.B) {
function BenchmarkGet (line 51) | func BenchmarkGet(b *testing.B) {
function BenchmarkIndex (line 64) | func BenchmarkIndex(b *testing.B) {
function BenchmarkIndexSelector (line 78) | func BenchmarkIndexSelector(b *testing.B) {
function BenchmarkIndexOfNode (line 92) | func BenchmarkIndexOfNode(b *testing.B) {
function BenchmarkIndexOfSelection (line 108) | func BenchmarkIndexOfSelection(b *testing.B) {
FILE: common/goquery/bench_example_test.go
function BenchmarkMetalReviewExample (line 10) | func BenchmarkMetalReviewExample(b *testing.B) {
FILE: common/goquery/bench_expand_test.go
function BenchmarkAdd (line 7) | func BenchmarkAdd(b *testing.B) {
function BenchmarkAddSelection (line 25) | func BenchmarkAddSelection(b *testing.B) {
function BenchmarkAddNodes (line 44) | func BenchmarkAddNodes(b *testing.B) {
function BenchmarkAddNodesBig (line 64) | func BenchmarkAddNodesBig(b *testing.B) {
function BenchmarkAndSelf (line 88) | func BenchmarkAndSelf(b *testing.B) {
FILE: common/goquery/bench_filter_test.go
function BenchmarkFilter (line 7) | func BenchmarkFilter(b *testing.B) {
function BenchmarkNot (line 25) | func BenchmarkNot(b *testing.B) {
function BenchmarkFilterFunction (line 43) | func BenchmarkFilterFunction(b *testing.B) {
function BenchmarkNotFunction (line 64) | func BenchmarkNotFunction(b *testing.B) {
function BenchmarkFilterNodes (line 85) | func BenchmarkFilterNodes(b *testing.B) {
function BenchmarkNotNodes (line 105) | func BenchmarkNotNodes(b *testing.B) {
function BenchmarkFilterSelection (line 125) | func BenchmarkFilterSelection(b *testing.B) {
function BenchmarkNotSelection (line 144) | func BenchmarkNotSelection(b *testing.B) {
function BenchmarkHas (line 163) | func BenchmarkHas(b *testing.B) {
function BenchmarkHasNodes (line 181) | func BenchmarkHasNodes(b *testing.B) {
function BenchmarkHasSelection (line 201) | func BenchmarkHasSelection(b *testing.B) {
function BenchmarkEnd (line 220) | func BenchmarkEnd(b *testing.B) {
FILE: common/goquery/bench_iteration_test.go
function BenchmarkEach (line 7) | func BenchmarkEach(b *testing.B) {
function BenchmarkMap (line 27) | func BenchmarkMap(b *testing.B) {
function BenchmarkEachWithBreak (line 48) | func BenchmarkEachWithBreak(b *testing.B) {
FILE: common/goquery/bench_property_test.go
function BenchmarkAttr (line 7) | func BenchmarkAttr(b *testing.B) {
function BenchmarkText (line 21) | func BenchmarkText(b *testing.B) {
function BenchmarkLength (line 30) | func BenchmarkLength(b *testing.B) {
function BenchmarkHtml (line 44) | func BenchmarkHtml(b *testing.B) {
FILE: common/goquery/bench_query_test.go
function BenchmarkIs (line 7) | func BenchmarkIs(b *testing.B) {
function BenchmarkIsPositional (line 21) | func BenchmarkIsPositional(b *testing.B) {
function BenchmarkIsFunction (line 35) | func BenchmarkIsFunction(b *testing.B) {
function BenchmarkIsSelection (line 52) | func BenchmarkIsSelection(b *testing.B) {
function BenchmarkIsNodes (line 67) | func BenchmarkIsNodes(b *testing.B) {
function BenchmarkHasClass (line 83) | func BenchmarkHasClass(b *testing.B) {
function BenchmarkContains (line 97) | func BenchmarkContains(b *testing.B) {
FILE: common/goquery/bench_traversal_test.go
function BenchmarkFind (line 7) | func BenchmarkFind(b *testing.B) {
function BenchmarkFindWithinSelection (line 23) | func BenchmarkFindWithinSelection(b *testing.B) {
function BenchmarkFindSelection (line 41) | func BenchmarkFindSelection(b *testing.B) {
function BenchmarkFindNodes (line 60) | func BenchmarkFindNodes(b *testing.B) {
function BenchmarkContents (line 80) | func BenchmarkContents(b *testing.B) {
function BenchmarkContentsFiltered (line 98) | func BenchmarkContentsFiltered(b *testing.B) {
function BenchmarkChildren (line 116) | func BenchmarkChildren(b *testing.B) {
function BenchmarkChildrenFiltered (line 134) | func BenchmarkChildrenFiltered(b *testing.B) {
function BenchmarkParent (line 152) | func BenchmarkParent(b *testing.B) {
function BenchmarkParentFiltered (line 170) | func BenchmarkParentFiltered(b *testing.B) {
function BenchmarkParents (line 188) | func BenchmarkParents(b *testing.B) {
function BenchmarkParentsFiltered (line 206) | func BenchmarkParentsFiltered(b *testing.B) {
function BenchmarkParentsUntil (line 224) | func BenchmarkParentsUntil(b *testing.B) {
function BenchmarkParentsUntilSelection (line 242) | func BenchmarkParentsUntilSelection(b *testing.B) {
function BenchmarkParentsUntilNodes (line 261) | func BenchmarkParentsUntilNodes(b *testing.B) {
function BenchmarkParentsFilteredUntil (line 281) | func BenchmarkParentsFilteredUntil(b *testing.B) {
function BenchmarkParentsFilteredUntilSelection (line 299) | func BenchmarkParentsFilteredUntilSelection(b *testing.B) {
function BenchmarkParentsFilteredUntilNodes (line 318) | func BenchmarkParentsFilteredUntilNodes(b *testing.B) {
function BenchmarkSiblings (line 338) | func BenchmarkSiblings(b *testing.B) {
function BenchmarkSiblingsFiltered (line 356) | func BenchmarkSiblingsFiltered(b *testing.B) {
function BenchmarkNext (line 374) | func BenchmarkNext(b *testing.B) {
function BenchmarkNextFiltered (line 392) | func BenchmarkNextFiltered(b *testing.B) {
function BenchmarkNextAll (line 410) | func BenchmarkNextAll(b *testing.B) {
function BenchmarkNextAllFiltered (line 428) | func BenchmarkNextAllFiltered(b *testing.B) {
function BenchmarkPrev (line 446) | func BenchmarkPrev(b *testing.B) {
function BenchmarkPrevFiltered (line 464) | func BenchmarkPrevFiltered(b *testing.B) {
function BenchmarkPrevAll (line 484) | func BenchmarkPrevAll(b *testing.B) {
function BenchmarkPrevAllFiltered (line 502) | func BenchmarkPrevAllFiltered(b *testing.B) {
function BenchmarkNextUntil (line 520) | func BenchmarkNextUntil(b *testing.B) {
function BenchmarkNextUntilSelection (line 538) | func BenchmarkNextUntilSelection(b *testing.B) {
function BenchmarkNextUntilNodes (line 557) | func BenchmarkNextUntilNodes(b *testing.B) {
function BenchmarkPrevUntil (line 577) | func BenchmarkPrevUntil(b *testing.B) {
function BenchmarkPrevUntilSelection (line 595) | func BenchmarkPrevUntilSelection(b *testing.B) {
function BenchmarkPrevUntilNodes (line 614) | func BenchmarkPrevUntilNodes(b *testing.B) {
function BenchmarkNextFilteredUntil (line 634) | func BenchmarkNextFilteredUntil(b *testing.B) {
function BenchmarkNextFilteredUntilSelection (line 652) | func BenchmarkNextFilteredUntilSelection(b *testing.B) {
function BenchmarkNextFilteredUntilNodes (line 671) | func BenchmarkNextFilteredUntilNodes(b *testing.B) {
function BenchmarkPrevFilteredUntil (line 691) | func BenchmarkPrevFilteredUntil(b *testing.B) {
function BenchmarkPrevFilteredUntilSelection (line 709) | func BenchmarkPrevFilteredUntilSelection(b *testing.B) {
function BenchmarkPrevFilteredUntilNodes (line 728) | func BenchmarkPrevFilteredUntilNodes(b *testing.B) {
function BenchmarkClosest (line 748) | func BenchmarkClosest(b *testing.B) {
function BenchmarkClosestSelection (line 766) | func BenchmarkClosestSelection(b *testing.B) {
function BenchmarkClosestNodes (line 785) | func BenchmarkClosestNodes(b *testing.B) {
FILE: common/goquery/example_test.go
function Example (line 11) | func Example() {
FILE: common/goquery/expand.go
method Add (line 9) | func (s *Selection) Add(selector string) *Selection {
method AddMatcher (line 17) | func (s *Selection) AddMatcher(m Matcher) *Selection {
method AddSelection (line 23) | func (s *Selection) AddSelection(sel *Selection) *Selection {
method Union (line 31) | func (s *Selection) Union(sel *Selection) *Selection {
method AddNodes (line 37) | func (s *Selection) AddNodes(nodes ...*html.Node) *Selection {
method AndSelf (line 44) | func (s *Selection) AndSelf() *Selection {
FILE: common/goquery/expand_test.go
function TestAdd (line 7) | func TestAdd(t *testing.T) {
function TestAddInvalid (line 12) | func TestAddInvalid(t *testing.T) {
function TestAddRollback (line 22) | func TestAddRollback(t *testing.T) {
function TestAddSelection (line 28) | func TestAddSelection(t *testing.T) {
function TestAddSelectionNil (line 35) | func TestAddSelectionNil(t *testing.T) {
function TestAddSelectionRollback (line 43) | func TestAddSelectionRollback(t *testing.T) {
function TestAddNodes (line 50) | func TestAddNodes(t *testing.T) {
function TestAddNodesNone (line 57) | func TestAddNodesNone(t *testing.T) {
function TestAddNodesRollback (line 62) | func TestAddNodesRollback(t *testing.T) {
function TestAddNodesBig (line 69) | func TestAddNodesBig(t *testing.T) {
function TestAndSelf (line 87) | func TestAndSelf(t *testing.T) {
function TestAndSelfRollback (line 92) | func TestAndSelfRollback(t *testing.T) {
FILE: common/goquery/filter.go
method Filter (line 7) | func (s *Selection) Filter(selector string) *Selection {
method FilterMatcher (line 14) | func (s *Selection) FilterMatcher(m Matcher) *Selection {
method Not (line 20) | func (s *Selection) Not(selector string) *Selection {
method NotMatcher (line 26) | func (s *Selection) NotMatcher(m Matcher) *Selection {
method FilterFunction (line 32) | func (s *Selection) FilterFunction(f func(int, *Selection) bool) *Select...
method NotFunction (line 38) | func (s *Selection) NotFunction(f func(int, *Selection) bool) *Selection {
method FilterNodes (line 44) | func (s *Selection) FilterNodes(nodes ...*html.Node) *Selection {
method NotNodes (line 50) | func (s *Selection) NotNodes(nodes ...*html.Node) *Selection {
method FilterSelection (line 57) | func (s *Selection) FilterSelection(sel *Selection) *Selection {
method NotSelection (line 66) | func (s *Selection) NotSelection(sel *Selection) *Selection {
method Intersection (line 74) | func (s *Selection) Intersection(sel *Selection) *Selection {
method Has (line 81) | func (s *Selection) Has(selector string) *Selection {
method HasMatcher (line 88) | func (s *Selection) HasMatcher(m Matcher) *Selection {
method HasNodes (line 95) | func (s *Selection) HasNodes(nodes ...*html.Node) *Selection {
method HasSelection (line 110) | func (s *Selection) HasSelection(sel *Selection) *Selection {
method End (line 119) | func (s *Selection) End() *Selection {
function winnow (line 128) | func winnow(sel *Selection, m Matcher, keep bool) []*html.Node {
function winnowNodes (line 141) | func winnowNodes(sel *Selection, nodes []*html.Node, keep bool) []*html....
function winnowFunction (line 159) | func winnowFunction(sel *Selection, f func(int, *Selection) bool, keep b...
FILE: common/goquery/filter_test.go
function TestFilter (line 7) | func TestFilter(t *testing.T) {
function TestFilterNone (line 12) | func TestFilterNone(t *testing.T) {
function TestFilterInvalid (line 17) | func TestFilterInvalid(t *testing.T) {
function TestFilterRollback (line 22) | func TestFilterRollback(t *testing.T) {
function TestFilterFunction (line 28) | func TestFilterFunction(t *testing.T) {
function TestFilterFunctionRollback (line 35) | func TestFilterFunctionRollback(t *testing.T) {
function TestFilterNode (line 43) | func TestFilterNode(t *testing.T) {
function TestFilterNodeRollback (line 49) | func TestFilterNodeRollback(t *testing.T) {
function TestFilterSelection (line 55) | func TestFilterSelection(t *testing.T) {
function TestFilterSelectionRollback (line 62) | func TestFilterSelectionRollback(t *testing.T) {
function TestFilterSelectionNil (line 69) | func TestFilterSelectionNil(t *testing.T) {
function TestNot (line 77) | func TestNot(t *testing.T) {
function TestNotInvalid (line 82) | func TestNotInvalid(t *testing.T) {
function TestNotRollback (line 87) | func TestNotRollback(t *testing.T) {
function TestNotNone (line 93) | func TestNotNone(t *testing.T) {
function TestNotFunction (line 98) | func TestNotFunction(t *testing.T) {
function TestNotFunctionRollback (line 105) | func TestNotFunctionRollback(t *testing.T) {
function TestNotNode (line 113) | func TestNotNode(t *testing.T) {
function TestNotNodeRollback (line 119) | func TestNotNodeRollback(t *testing.T) {
function TestNotSelection (line 125) | func TestNotSelection(t *testing.T) {
function TestNotSelectionRollback (line 132) | func TestNotSelectionRollback(t *testing.T) {
function TestIntersection (line 139) | func TestIntersection(t *testing.T) {
function TestIntersectionRollback (line 145) | func TestIntersectionRollback(t *testing.T) {
function TestHas (line 152) | func TestHas(t *testing.T) {
function TestHasInvalid (line 158) | func TestHasInvalid(t *testing.T) {
function TestHasRollback (line 163) | func TestHasRollback(t *testing.T) {
function TestHasNodes (line 169) | func TestHasNodes(t *testing.T) {
function TestHasNodesRollback (line 177) | func TestHasNodesRollback(t *testing.T) {
function TestHasSelection (line 184) | func TestHasSelection(t *testing.T) {
function TestHasSelectionRollback (line 191) | func TestHasSelectionRollback(t *testing.T) {
function TestEnd (line 198) | func TestEnd(t *testing.T) {
function TestEndToTop (line 203) | func TestEndToTop(t *testing.T) {
FILE: common/goquery/iteration.go
method Each (line 8) | func (s *Selection) Each(f func(int, *Selection)) *Selection {
method EachWithBreak (line 19) | func (s *Selection) EachWithBreak(f func(int, *Selection) bool) *Selecti...
method Map (line 33) | func (s *Selection) Map(f func(int, *Selection) string) (result []string) {
FILE: common/goquery/iteration_test.go
function TestEach (line 9) | func TestEach(t *testing.T) {
function TestEachWithBreak (line 23) | func TestEachWithBreak(t *testing.T) {
function TestEachEmptySelection (line 38) | func TestEachEmptySelection(t *testing.T) {
function TestMap (line 52) | func TestMap(t *testing.T) {
function TestForRange (line 71) | func TestForRange(t *testing.T) {
FILE: common/goquery/manipulation.go
method After (line 17) | func (s *Selection) After(selector string) *Selection {
method AfterMatcher (line 28) | func (s *Selection) AfterMatcher(m Matcher) *Selection {
method AfterSelection (line 36) | func (s *Selection) AfterSelection(sel *Selection) *Selection {
method AfterHtml (line 43) | func (s *Selection) AfterHtml(html string) *Selection {
method AfterNodes (line 50) | func (s *Selection) AfterNodes(ns ...*html.Node) *Selection {
method Append (line 68) | func (s *Selection) Append(selector string) *Selection {
method AppendMatcher (line 76) | func (s *Selection) AppendMatcher(m Matcher) *Selection {
method AppendSelection (line 84) | func (s *Selection) AppendSelection(sel *Selection) *Selection {
method AppendHtml (line 89) | func (s *Selection) AppendHtml(html string) *Selection {
method AppendNodes (line 96) | func (s *Selection) AppendNodes(ns ...*html.Node) *Selection {
method Before (line 105) | func (s *Selection) Before(selector string) *Selection {
method BeforeMatcher (line 112) | func (s *Selection) BeforeMatcher(m Matcher) *Selection {
method BeforeSelection (line 120) | func (s *Selection) BeforeSelection(sel *Selection) *Selection {
method BeforeHtml (line 127) | func (s *Selection) BeforeHtml(html string) *Selection {
method BeforeNodes (line 134) | func (s *Selection) BeforeNodes(ns ...*html.Node) *Selection {
method Clone (line 144) | func (s *Selection) Clone() *Selection {
method Empty (line 152) | func (s *Selection) Empty() *Selection {
method Prepend (line 167) | func (s *Selection) Prepend(selector string) *Selection {
method PrependMatcher (line 175) | func (s *Selection) PrependMatcher(m Matcher) *Selection {
method PrependSelection (line 183) | func (s *Selection) PrependSelection(sel *Selection) *Selection {
method PrependHtml (line 188) | func (s *Selection) PrependHtml(html string) *Selection {
method PrependNodes (line 196) | func (s *Selection) PrependNodes(ns ...*html.Node) *Selection {
method Remove (line 206) | func (s *Selection) Remove() *Selection {
method RemoveFiltered (line 218) | func (s *Selection) RemoveFiltered(selector string) *Selection {
method RemoveMatcher (line 224) | func (s *Selection) RemoveMatcher(m Matcher) *Selection {
method ReplaceWith (line 233) | func (s *Selection) ReplaceWith(selector string) *Selection {
method ReplaceWithMatcher (line 242) | func (s *Selection) ReplaceWithMatcher(m Matcher) *Selection {
method ReplaceWithSelection (line 251) | func (s *Selection) ReplaceWithSelection(sel *Selection) *Selection {
method ReplaceWithHtml (line 260) | func (s *Selection) ReplaceWithHtml(html string) *Selection {
method ReplaceWithNodes (line 269) | func (s *Selection) ReplaceWithNodes(ns ...*html.Node) *Selection {
method Unwrap (line 277) | func (s *Selection) Unwrap() *Selection {
method Wrap (line 295) | func (s *Selection) Wrap(selector string) *Selection {
method WrapMatcher (line 304) | func (s *Selection) WrapMatcher(m Matcher) *Selection {
method WrapSelection (line 313) | func (s *Selection) WrapSelection(sel *Selection) *Selection {
method WrapHtml (line 321) | func (s *Selection) WrapHtml(html string) *Selection {
method WrapNode (line 330) | func (s *Selection) WrapNode(n *html.Node) *Selection {
method wrapNodes (line 334) | func (s *Selection) wrapNodes(ns ...*html.Node) *Selection {
method WrapAll (line 347) | func (s *Selection) WrapAll(selector string) *Selection {
method WrapAllMatcher (line 356) | func (s *Selection) WrapAllMatcher(m Matcher) *Selection {
method WrapAllSelection (line 365) | func (s *Selection) WrapAllSelection(sel *Selection) *Selection {
method WrapAllHtml (line 374) | func (s *Selection) WrapAllHtml(html string) *Selection {
method wrapAllNodes (line 378) | func (s *Selection) wrapAllNodes(ns ...*html.Node) *Selection {
method WrapAllNode (line 390) | func (s *Selection) WrapAllNode(n *html.Node) *Selection {
method WrapInner (line 417) | func (s *Selection) WrapInner(selector string) *Selection {
method WrapInnerMatcher (line 426) | func (s *Selection) WrapInnerMatcher(m Matcher) *Selection {
method WrapInnerSelection (line 435) | func (s *Selection) WrapInnerSelection(sel *Selection) *Selection {
method WrapInnerHtml (line 444) | func (s *Selection) WrapInnerHtml(html string) *Selection {
method WrapInnerNode (line 453) | func (s *Selection) WrapInnerNode(n *html.Node) *Selection {
method wrapInnerNodes (line 457) | func (s *Selection) wrapInnerNodes(ns ...*html.Node) *Selection {
function parseHtml (line 475) | func parseHtml(h string) []*html.Node {
function getFirstChildEl (line 486) | func getFirstChildEl(n *html.Node) *html.Node {
function cloneNodes (line 495) | func cloneNodes(ns []*html.Node) []*html.Node {
function cloneNode (line 507) | func cloneNode(n *html.Node) *html.Node {
method manipulateNodes (line 523) | func (s *Selection) manipulateNodes(ns []*html.Node, reverse bool,
FILE: common/goquery/manipulation_test.go
constant wrapHtml (line 8) | wrapHtml = "<div id=\"ins\">test string<div><p><em><b></b></em></p></div...
function TestAfter (line 11) | func TestAfter(t *testing.T) {
function TestAfterMany (line 21) | func TestAfterMany(t *testing.T) {
function TestAfterWithRemoved (line 31) | func TestAfterWithRemoved(t *testing.T) {
function TestAfterSelection (line 41) | func TestAfterSelection(t *testing.T) {
function TestAfterHtml (line 51) | func TestAfterHtml(t *testing.T) {
function TestAppend (line 59) | func TestAppend(t *testing.T) {
function TestAppendBody (line 68) | func TestAppendBody(t *testing.T) {
function TestAppendSelection (line 78) | func TestAppendSelection(t *testing.T) {
function TestAppendSelectionExisting (line 89) | func TestAppendSelectionExisting(t *testing.T) {
function TestAppendClone (line 99) | func TestAppendClone(t *testing.T) {
function TestAppendHtml (line 108) | func TestAppendHtml(t *testing.T) {
function TestBefore (line 116) | func TestBefore(t *testing.T) {
function TestBeforeWithRemoved (line 126) | func TestBeforeWithRemoved(t *testing.T) {
function TestBeforeSelection (line 136) | func TestBeforeSelection(t *testing.T) {
function TestBeforeHtml (line 146) | func TestBeforeHtml(t *testing.T) {
function TestEmpty (line 154) | func TestEmpty(t *testing.T) {
function TestPrepend (line 163) | func TestPrepend(t *testing.T) {
function TestPrependBody (line 172) | func TestPrependBody(t *testing.T) {
function TestPrependSelection (line 182) | func TestPrependSelection(t *testing.T) {
function TestPrependSelectionExisting (line 193) | func TestPrependSelectionExisting(t *testing.T) {
function TestPrependClone (line 204) | func TestPrependClone(t *testing.T) {
function TestPrependHtml (line 213) | func TestPrependHtml(t *testing.T) {
function TestRemove (line 221) | func TestRemove(t *testing.T) {
function TestRemoveAll (line 229) | func TestRemoveAll(t *testing.T) {
function TestRemoveRoot (line 237) | func TestRemoveRoot(t *testing.T) {
function TestRemoveFiltered (line 245) | func TestRemoveFiltered(t *testing.T) {
function TestReplaceWith (line 258) | func TestReplaceWith(t *testing.T) {
function TestReplaceWithHtml (line 272) | func TestReplaceWithHtml(t *testing.T) {
function TestReplaceWithSelection (line 281) | func TestReplaceWithSelection(t *testing.T) {
function TestUnwrap (line 292) | func TestUnwrap(t *testing.T) {
function TestUnwrapBody (line 313) | func TestUnwrapBody(t *testing.T) {
function TestUnwrapHead (line 323) | func TestUnwrapHead(t *testing.T) {
function TestUnwrapHtml (line 334) | func TestUnwrapHtml(t *testing.T) {
function TestWrap (line 345) | func TestWrap(t *testing.T) {
function TestWrapEmpty (line 357) | func TestWrapEmpty(t *testing.T) {
function TestWrapHtml (line 371) | func TestWrapHtml(t *testing.T) {
function TestWrapSelection (line 379) | func TestWrapSelection(t *testing.T) {
function TestWrapAll (line 391) | func TestWrapAll(t *testing.T) {
function TestWrapAllHtml (line 403) | func TestWrapAllHtml(t *testing.T) {
function TestWrapInnerNoContent (line 411) | func TestWrapInnerNoContent(t *testing.T) {
function TestWrapInnerWithContent (line 422) | func TestWrapInnerWithContent(t *testing.T) {
function TestWrapInnerNoWrapper (line 433) | func TestWrapInnerNoWrapper(t *testing.T) {
function TestWrapInnerHtml (line 445) | func TestWrapInnerHtml(t *testing.T) {
FILE: common/goquery/property.go
method Attr (line 17) | func (s *Selection) Attr(attrName string) option.Option[string] {
method AttrOr (line 25) | func (s *Selection) AttrOr(attrName, defaultValue string) string {
method RemoveAttr (line 30) | func (s *Selection) RemoveAttr(attrName string) *Selection {
method SetAttr (line 39) | func (s *Selection) SetAttr(attrName, val string) *Selection {
method Text (line 54) | func (s *Selection) Text() string {
method Size (line 65) | func (s *Selection) Size() int {
method Length (line 70) | func (s *Selection) Length() int {
method Html (line 76) | func (s *Selection) Html() (ret string, e error) {
method AddClass (line 96) | func (s *Selection) AddClass(class ...string) *Selection {
method HasClass (line 120) | func (s *Selection) HasClass(class string) bool {
method RemoveClass (line 134) | func (s *Selection) RemoveClass(class ...string) *Selection {
method ToggleClass (line 162) | func (s *Selection) ToggleClass(class ...string) *Selection {
function getNodeText (line 188) | func getNodeText(node *html.Node) string {
function getAttributePtr (line 203) | func getAttributePtr(attrName string, n *html.Node) *html.Attribute {
function getAttributeValue (line 217) | func getAttributeValue(attrName string, n *html.Node) option.Option[stri...
function getClassesAndAttr (line 225) | func getClassesAndAttr(n *html.Node, create bool) (classes string, attr ...
function getClassesSlice (line 247) | func getClassesSlice(classes string) []string {
function removeAttr (line 251) | func removeAttr(n *html.Node, attrName string) {
function setClasses (line 261) | func setClasses(n *html.Node, attr *html.Attribute, classes string) {
FILE: common/goquery/property_test.go
function TestAttrExists (line 9) | func TestAttrExists(t *testing.T) {
function TestAttrOr (line 17) | func TestAttrOr(t *testing.T) {
function TestAttrNotExist (line 30) | func TestAttrNotExist(t *testing.T) {
function TestRemoveAttr (line 36) | func TestRemoveAttr(t *testing.T) {
function TestSetAttr (line 46) | func TestSetAttr(t *testing.T) {
function TestSetAttr2 (line 61) | func TestSetAttr2(t *testing.T) {
function TestText (line 76) | func TestText(t *testing.T) {
function TestText2 (line 83) | func TestText2(t *testing.T) {
function TestText3 (line 93) | func TestText3(t *testing.T) {
function TestHtml (line 104) | func TestHtml(t *testing.T) {
function TestNbsp (line 118) | func TestNbsp(t *testing.T) {
function TestAddClass (line 141) | func TestAddClass(t *testing.T) {
function TestAddClassSimilar (line 151) | func TestAddClassSimilar(t *testing.T) {
function TestAddEmptyClass (line 160) | func TestAddEmptyClass(t *testing.T) {
function TestAddClasses (line 170) | func TestAddClasses(t *testing.T) {
function TestHasClass (line 180) | func TestHasClass(t *testing.T) {
function TestHasClassNone (line 187) | func TestHasClassNone(t *testing.T) {
function TestHasClassNotFirst (line 194) | func TestHasClassNotFirst(t *testing.T) {
function TestRemoveClass (line 201) | func TestRemoveClass(t *testing.T) {
function TestRemoveClassSimilar (line 210) | func TestRemoveClassSimilar(t *testing.T) {
function TestRemoveAllClasses (line 219) | func TestRemoveAllClasses(t *testing.T) {
function TestToggleClass (line 234) | func TestToggleClass(t *testing.T) {
FILE: common/goquery/query.go
method Is (line 7) | func (s *Selection) Is(selector string) bool {
method IsMatcher (line 17) | func (s *Selection) IsMatcher(m Matcher) bool {
method IsFunction (line 30) | func (s *Selection) IsFunction(f func(int, *Selection) bool) bool {
method IsSelection (line 36) | func (s *Selection) IsSelection(sel *Selection) bool {
method IsNodes (line 42) | func (s *Selection) IsNodes(nodes ...*html.Node) bool {
method Contains (line 51) | func (s *Selection) Contains(n *html.Node) bool {
FILE: common/goquery/query_test.go
function TestIs (line 7) | func TestIs(t *testing.T) {
function TestIsInvalid (line 14) | func TestIsInvalid(t *testing.T) {
function TestIsPositional (line 21) | func TestIsPositional(t *testing.T) {
function TestIsPositionalNot (line 28) | func TestIsPositionalNot(t *testing.T) {
function TestIsFunction (line 35) | func TestIsFunction(t *testing.T) {
function TestIsFunctionRollback (line 45) | func TestIsFunctionRollback(t *testing.T) {
function TestIsSelection (line 55) | func TestIsSelection(t *testing.T) {
function TestIsSelectionNot (line 64) | func TestIsSelectionNot(t *testing.T) {
function TestIsNodes (line 73) | func TestIsNodes(t *testing.T) {
function TestDocContains (line 82) | func TestDocContains(t *testing.T) {
function TestSelContains (line 89) | func TestSelContains(t *testing.T) {
function TestSelNotContains (line 97) | func TestSelNotContains(t *testing.T) {
FILE: common/goquery/traversal.go
type siblingType (line 5) | type siblingType
constant siblingPrevUntil (line 10) | siblingPrevUntil siblingType = iota - 3
constant siblingPrevAll (line 11) | siblingPrevAll
constant siblingPrev (line 12) | siblingPrev
constant siblingAll (line 13) | siblingAll
constant siblingNext (line 14) | siblingNext
constant siblingNextAll (line 15) | siblingNextAll
constant siblingNextUntil (line 16) | siblingNextUntil
constant siblingAllIncludingNonElements (line 17) | siblingAllIncludingNonElements
method Find (line 23) | func (s *Selection) Find(selector string) *Selection {
method FindMatcher (line 30) | func (s *Selection) FindMatcher(m Matcher) *Selection {
method FindSelection (line 37) | func (s *Selection) FindSelection(sel *Selection) *Selection {
method FindNodes (line 47) | func (s *Selection) FindNodes(nodes ...*html.Node) *Selection {
method Contents (line 59) | func (s *Selection) Contents() *Selection {
method ContentsFiltered (line 68) | func (s *Selection) ContentsFiltered(selector string) *Selection {
method ContentsMatcher (line 79) | func (s *Selection) ContentsMatcher(m Matcher) *Selection {
method Children (line 85) | func (s *Selection) Children() *Selection {
method ChildrenFiltered (line 92) | func (s *Selection) ChildrenFiltered(selector string) *Selection {
method ChildrenMatcher (line 99) | func (s *Selection) ChildrenMatcher(m Matcher) *Selection {
method Parent (line 105) | func (s *Selection) Parent() *Selection {
method ParentFiltered (line 111) | func (s *Selection) ParentFiltered(selector string) *Selection {
method ParentMatcher (line 117) | func (s *Selection) ParentMatcher(m Matcher) *Selection {
method Closest (line 123) | func (s *Selection) Closest(selector string) *Selection {
method ClosestMatcher (line 130) | func (s *Selection) ClosestMatcher(m Matcher) *Selection {
method ClosestNodes (line 145) | func (s *Selection) ClosestNodes(nodes ...*html.Node) *Selection {
method ClosestSelection (line 165) | func (s *Selection) ClosestSelection(sel *Selection) *Selection {
method Parents (line 174) | func (s *Selection) Parents() *Selection {
method ParentsFiltered (line 180) | func (s *Selection) ParentsFiltered(selector string) *Selection {
method ParentsMatcher (line 186) | func (s *Selection) ParentsMatcher(m Matcher) *Selection {
method ParentsUntil (line 193) | func (s *Selection) ParentsUntil(selector string) *Selection {
method ParentsUntilMatcher (line 200) | func (s *Selection) ParentsUntilMatcher(m Matcher) *Selection {
method ParentsUntilSelection (line 207) | func (s *Selection) ParentsUntilSelection(sel *Selection) *Selection {
method ParentsUntilNodes (line 217) | func (s *Selection) ParentsUntilNodes(nodes ...*html.Node) *Selection {
method ParentsFilteredUntil (line 224) | func (s *Selection) ParentsFilteredUntil(filterSelector, untilSelector s...
method ParentsFilteredUntilMatcher (line 230) | func (s *Selection) ParentsFilteredUntilMatcher(filter, until Matcher) *...
method ParentsFilteredUntilSelection (line 237) | func (s *Selection) ParentsFilteredUntilSelection(filterSelector string,...
method ParentsMatcherUntilSelection (line 244) | func (s *Selection) ParentsMatcherUntilSelection(filter Matcher, sel *Se...
method ParentsFilteredUntilNodes (line 254) | func (s *Selection) ParentsFilteredUntilNodes(filterSelector string, nod...
method ParentsMatcherUntilNodes (line 261) | func (s *Selection) ParentsMatcherUntilNodes(filter Matcher, nodes ...*h...
method Siblings (line 267) | func (s *Selection) Siblings() *Selection {
method SiblingsFiltered (line 274) | func (s *Selection) SiblingsFiltered(selector string) *Selection {
method SiblingsMatcher (line 281) | func (s *Selection) SiblingsMatcher(m Matcher) *Selection {
method Next (line 287) | func (s *Selection) Next() *Selection {
method NextFiltered (line 294) | func (s *Selection) NextFiltered(selector string) *Selection {
method NextMatcher (line 301) | func (s *Selection) NextMatcher(m Matcher) *Selection {
method NextAll (line 307) | func (s *Selection) NextAll() *Selection {
method NextAllFiltered (line 314) | func (s *Selection) NextAllFiltered(selector string) *Selection {
method NextAllMatcher (line 321) | func (s *Selection) NextAllMatcher(m Matcher) *Selection {
method Prev (line 327) | func (s *Selection) Prev() *Selection {
method PrevFiltered (line 334) | func (s *Selection) PrevFiltered(selector string) *Selection {
method PrevMatcher (line 341) | func (s *Selection) PrevMatcher(m Matcher) *Selection {
method PrevAll (line 347) | func (s *Selection) PrevAll() *Selection {
method PrevAllFiltered (line 354) | func (s *Selection) PrevAllFiltered(selector string) *Selection {
method PrevAllMatcher (line 361) | func (s *Selection) PrevAllMatcher(m Matcher) *Selection {
method NextUntil (line 368) | func (s *Selection) NextUntil(selector string) *Selection {
method NextUntilMatcher (line 376) | func (s *Selection) NextUntilMatcher(m Matcher) *Selection {
method NextUntilSelection (line 384) | func (s *Selection) NextUntilSelection(sel *Selection) *Selection {
method NextUntilNodes (line 394) | func (s *Selection) NextUntilNodes(nodes ...*html.Node) *Selection {
method PrevUntil (line 402) | func (s *Selection) PrevUntil(selector string) *Selection {
method PrevUntilMatcher (line 410) | func (s *Selection) PrevUntilMatcher(m Matcher) *Selection {
method PrevUntilSelection (line 418) | func (s *Selection) PrevUntilSelection(sel *Selection) *Selection {
method PrevUntilNodes (line 428) | func (s *Selection) PrevUntilNodes(nodes ...*html.Node) *Selection {
method NextFilteredUntil (line 436) | func (s *Selection) NextFilteredUntil(filterSelector, untilSelector stri...
method NextFilteredUntilMatcher (line 444) | func (s *Selection) NextFilteredUntilMatcher(filter, until Matcher) *Sel...
method NextFilteredUntilSelection (line 452) | func (s *Selection) NextFilteredUntilSelection(filterSelector string, se...
method NextMatcherUntilSelection (line 459) | func (s *Selection) NextMatcherUntilSelection(filter Matcher, sel *Selec...
method NextFilteredUntilNodes (line 469) | func (s *Selection) NextFilteredUntilNodes(filterSelector string, nodes ...
method NextMatcherUntilNodes (line 477) | func (s *Selection) NextMatcherUntilNodes(filter Matcher, nodes ...*html...
method PrevFilteredUntil (line 485) | func (s *Selection) PrevFilteredUntil(filterSelector, untilSelector stri...
method PrevFilteredUntilMatcher (line 493) | func (s *Selection) PrevFilteredUntilMatcher(filter, until Matcher) *Sel...
method PrevFilteredUntilSelection (line 501) | func (s *Selection) PrevFilteredUntilSelection(filterSelector string, se...
method PrevMatcherUntilSelection (line 508) | func (s *Selection) PrevMatcherUntilSelection(filter Matcher, sel *Selec...
method PrevFilteredUntilNodes (line 518) | func (s *Selection) PrevFilteredUntilNodes(filterSelector string, nodes ...
method PrevMatcherUntilNodes (line 526) | func (s *Selection) PrevMatcherUntilNodes(filter Matcher, nodes ...*html...
function filterAndPush (line 533) | func filterAndPush(srcSel *Selection, nodes []*html.Node, m Matcher) *Se...
function findWithMatcher (line 541) | func findWithMatcher(nodes []*html.Node, m Matcher) []*html.Node {
function getParentsNodes (line 556) | func getParentsNodes(nodes []*html.Node, stopm Matcher, stopNodes []*htm...
function getSiblingNodes (line 578) | func getSiblingNodes(nodes []*html.Node, st siblingType, untilm Matcher,...
function getChildrenNodes (line 605) | func getChildrenNodes(nodes []*html.Node, st siblingType) []*html.Node {
function getChildrenWithSiblingType (line 613) | func getChildrenWithSiblingType(parent *html.Node, st siblingType, skipN...
function getParentNodes (line 677) | func getParentNodes(nodes []*html.Node) []*html.Node {
function mapNodes (line 690) | func mapNodes(nodes []*html.Node, f func(int, *html.Node) []*html.Node) ...
FILE: common/goquery/traversal_test.go
function TestFind (line 8) | func TestFind(t *testing.T) {
function TestFindRollback (line 13) | func TestFindRollback(t *testing.T) {
function TestFindNotSelf (line 19) | func TestFindNotSelf(t *testing.T) {
function TestFindInvalid (line 24) | func TestFindInvalid(t *testing.T) {
function TestFindBig (line 29) | func TestFindBig(t *testing.T) {
function TestChainedFind (line 39) | func TestChainedFind(t *testing.T) {
function TestChainedFindInvalid (line 44) | func TestChainedFindInvalid(t *testing.T) {
function TestChildren (line 49) | func TestChildren(t *testing.T) {
function TestChildrenRollback (line 54) | func TestChildrenRollback(t *testing.T) {
function TestContents (line 60) | func TestContents(t *testing.T) {
function TestContentsRollback (line 65) | func TestContentsRollback(t *testing.T) {
function TestChildrenFiltered (line 71) | func TestChildrenFiltered(t *testing.T) {
function TestChildrenFilteredInvalid (line 76) | func TestChildrenFilteredInvalid(t *testing.T) {
function TestChildrenFilteredRollback (line 81) | func TestChildrenFilteredRollback(t *testing.T) {
function TestContentsFiltered (line 87) | func TestContentsFiltered(t *testing.T) {
function TestContentsFilteredInvalid (line 92) | func TestContentsFilteredInvalid(t *testing.T) {
function TestContentsFilteredRollback (line 97) | func TestContentsFilteredRollback(t *testing.T) {
function TestChildrenFilteredNone (line 103) | func TestChildrenFilteredNone(t *testing.T) {
function TestParent (line 108) | func TestParent(t *testing.T) {
function TestParentRollback (line 113) | func TestParentRollback(t *testing.T) {
function TestParentBody (line 119) | func TestParentBody(t *testing.T) {
function TestParentFiltered (line 124) | func TestParentFiltered(t *testing.T) {
function TestParentFilteredInvalid (line 130) | func TestParentFilteredInvalid(t *testing.T) {
function TestParentFilteredRollback (line 135) | func TestParentFilteredRollback(t *testing.T) {
function TestParents (line 141) | func TestParents(t *testing.T) {
function TestParentsOrder (line 146) | func TestParentsOrder(t *testing.T) {
function TestParentsRollback (line 152) | func TestParentsRollback(t *testing.T) {
function TestParentsFiltered (line 158) | func TestParentsFiltered(t *testing.T) {
function TestParentsFilteredInvalid (line 163) | func TestParentsFilteredInvalid(t *testing.T) {
function TestParentsFilteredRollback (line 168) | func TestParentsFilteredRollback(t *testing.T) {
function TestParentsUntil (line 174) | func TestParentsUntil(t *testing.T) {
function TestParentsUntilInvalid (line 179) | func TestParentsUntilInvalid(t *testing.T) {
function TestParentsUntilRollback (line 184) | func TestParentsUntilRollback(t *testing.T) {
function TestParentsUntilSelection (line 190) | func TestParentsUntilSelection(t *testing.T) {
function TestParentsUntilSelectionRollback (line 197) | func TestParentsUntilSelectionRollback(t *testing.T) {
function TestParentsUntilNodes (line 204) | func TestParentsUntilNodes(t *testing.T) {
function TestParentsUntilNodesRollback (line 211) | func TestParentsUntilNodesRollback(t *testing.T) {
function TestParentsFilteredUntil (line 218) | func TestParentsFilteredUntil(t *testing.T) {
function TestParentsFilteredUntilInvalid (line 223) | func TestParentsFilteredUntilInvalid(t *testing.T) {
function TestParentsFilteredUntilRollback (line 228) | func TestParentsFilteredUntilRollback(t *testing.T) {
function TestParentsFilteredUntilSelection (line 234) | func TestParentsFilteredUntilSelection(t *testing.T) {
function TestParentsFilteredUntilSelectionRollback (line 241) | func TestParentsFilteredUntilSelectionRollback(t *testing.T) {
function TestParentsFilteredUntilNodes (line 248) | func TestParentsFilteredUntilNodes(t *testing.T) {
function TestParentsFilteredUntilNodesRollback (line 255) | func TestParentsFilteredUntilNodesRollback(t *testing.T) {
function TestSiblings (line 262) | func TestSiblings(t *testing.T) {
function TestSiblingsRollback (line 267) | func TestSiblingsRollback(t *testing.T) {
function TestSiblings2 (line 273) | func TestSiblings2(t *testing.T) {
function TestSiblings3 (line 278) | func TestSiblings3(t *testing.T) {
function TestSiblingsFiltered (line 283) | func TestSiblingsFiltered(t *testing.T) {
function TestSiblingsFilteredInvalid (line 288) | func TestSiblingsFilteredInvalid(t *testing.T) {
function TestSiblingsFilteredRollback (line 293) | func TestSiblingsFilteredRollback(t *testing.T) {
function TestNext (line 299) | func TestNext(t *testing.T) {
function TestNextRollback (line 304) | func TestNextRollback(t *testing.T) {
function TestNext2 (line 310) | func TestNext2(t *testing.T) {
function TestNextNone (line 315) | func TestNextNone(t *testing.T) {
function TestNextFiltered (line 320) | func TestNextFiltered(t *testing.T) {
function TestNextFilteredInvalid (line 325) | func TestNextFilteredInvalid(t *testing.T) {
function TestNextFilteredRollback (line 330) | func TestNextFilteredRollback(t *testing.T) {
function TestNextFiltered2 (line 336) | func TestNextFiltered2(t *testing.T) {
function TestPrev (line 341) | func TestPrev(t *testing.T) {
function TestPrevRollback (line 347) | func TestPrevRollback(t *testing.T) {
function TestPrev2 (line 353) | func TestPrev2(t *testing.T) {
function TestPrevNone (line 358) | func TestPrevNone(t *testing.T) {
function TestPrevFiltered (line 363) | func TestPrevFiltered(t *testing.T) {
function TestPrevFilteredInvalid (line 368) | func TestPrevFilteredInvalid(t *testing.T) {
function TestPrevFilteredRollback (line 373) | func TestPrevFilteredRollback(t *testing.T) {
function TestNextAll (line 379) | func TestNextAll(t *testing.T) {
function TestNextAllRollback (line 384) | func TestNextAllRollback(t *testing.T) {
function TestNextAll2 (line 390) | func TestNextAll2(t *testing.T) {
function TestNextAllNone (line 395) | func TestNextAllNone(t *testing.T) {
function TestNextAllFiltered (line 400) | func TestNextAllFiltered(t *testing.T) {
function TestNextAllFilteredInvalid (line 405) | func TestNextAllFilteredInvalid(t *testing.T) {
function TestNextAllFilteredRollback (line 410) | func TestNextAllFilteredRollback(t *testing.T) {
function TestNextAllFiltered2 (line 416) | func TestNextAllFiltered2(t *testing.T) {
function TestPrevAll (line 421) | func TestPrevAll(t *testing.T) {
function TestPrevAllOrder (line 426) | func TestPrevAllOrder(t *testing.T) {
function TestPrevAllRollback (line 432) | func TestPrevAllRollback(t *testing.T) {
function TestPrevAll2 (line 438) | func TestPrevAll2(t *testing.T) {
function TestPrevAllFiltered (line 443) | func TestPrevAllFiltered(t *testing.T) {
function TestPrevAllFilteredInvalid (line 448) | func TestPrevAllFilteredInvalid(t *testing.T) {
function TestPrevAllFilteredRollback (line 453) | func TestPrevAllFilteredRollback(t *testing.T) {
function TestNextUntil (line 459) | func TestNextUntil(t *testing.T) {
function TestNextUntilInvalid (line 465) | func TestNextUntilInvalid(t *testing.T) {
function TestNextUntil2 (line 470) | func TestNextUntil2(t *testing.T) {
function TestNextUntilOrder (line 476) | func TestNextUntilOrder(t *testing.T) {
function TestNextUntilRollback (line 482) | func TestNextUntilRollback(t *testing.T) {
function TestNextUntilSelection (line 488) | func TestNextUntilSelection(t *testing.T) {
function TestNextUntilSelectionRollback (line 496) | func TestNextUntilSelectionRollback(t *testing.T) {
function TestNextUntilNodes (line 503) | func TestNextUntilNodes(t *testing.T) {
function TestNextUntilNodesRollback (line 511) | func TestNextUntilNodesRollback(t *testing.T) {
function TestPrevUntil (line 518) | func TestPrevUntil(t *testing.T) {
function TestPrevUntilInvalid (line 524) | func TestPrevUntilInvalid(t *testing.T) {
function TestPrevUntil2 (line 529) | func TestPrevUntil2(t *testing.T) {
function TestPrevUntilOrder (line 535) | func TestPrevUntilOrder(t *testing.T) {
function TestPrevUntilRollback (line 541) | func TestPrevUntilRollback(t *testing.T) {
function TestPrevUntilSelection (line 547) | func TestPrevUntilSelection(t *testing.T) {
function TestPrevUntilSelectionRollback (line 555) | func TestPrevUntilSelectionRollback(t *testing.T) {
function TestPrevUntilNodes (line 562) | func TestPrevUntilNodes(t *testing.T) {
function TestPrevUntilNodesRollback (line 570) | func TestPrevUntilNodesRollback(t *testing.T) {
function TestNextFilteredUntil (line 577) | func TestNextFilteredUntil(t *testing.T) {
function TestNextFilteredUntilInvalid (line 583) | func TestNextFilteredUntilInvalid(t *testing.T) {
function TestNextFilteredUntilRollback (line 588) | func TestNextFilteredUntilRollback(t *testing.T) {
function TestNextFilteredUntilSelection (line 594) | func TestNextFilteredUntilSelection(t *testing.T) {
function TestNextFilteredUntilSelectionRollback (line 602) | func TestNextFilteredUntilSelectionRollback(t *testing.T) {
function TestNextFilteredUntilNodes (line 609) | func TestNextFilteredUntilNodes(t *testing.T) {
function TestNextFilteredUntilNodesRollback (line 617) | func TestNextFilteredUntilNodesRollback(t *testing.T) {
function TestPrevFilteredUntil (line 624) | func TestPrevFilteredUntil(t *testing.T) {
function TestPrevFilteredUntilInvalid (line 630) | func TestPrevFilteredUntilInvalid(t *testing.T) {
function TestPrevFilteredUntilRollback (line 635) | func TestPrevFilteredUntilRollback(t *testing.T) {
function TestPrevFilteredUntilSelection (line 641) | func TestPrevFilteredUntilSelection(t *testing.T) {
function TestPrevFilteredUntilSelectionRollback (line 649) | func TestPrevFilteredUntilSelectionRollback(t *testing.T) {
function TestPrevFilteredUntilNodes (line 656) | func TestPrevFilteredUntilNodes(t *testing.T) {
function TestPrevFilteredUntilNodesRollback (line 664) | func TestPrevFilteredUntilNodesRollback(t *testing.T) {
function TestClosestItself (line 671) | func TestClosestItself(t *testing.T) {
function TestClosestNoDupes (line 678) | func TestClosestNoDupes(t *testing.T) {
function TestClosestNone (line 685) | func TestClosestNone(t *testing.T) {
function TestClosestInvalid (line 691) | func TestClosestInvalid(t *testing.T) {
function TestClosestMany (line 697) | func TestClosestMany(t *testing.T) {
function TestClosestRollback (line 704) | func TestClosestRollback(t *testing.T) {
function TestClosestSelectionItself (line 710) | func TestClosestSelectionItself(t *testing.T) {
function TestClosestSelectionNoDupes (line 716) | func TestClosestSelectionNoDupes(t *testing.T) {
function TestClosestSelectionNone (line 723) | func TestClosestSelectionNone(t *testing.T) {
function TestClosestSelectionMany (line 729) | func TestClosestSelectionMany(t *testing.T) {
function TestClosestSelectionRollback (line 736) | func TestClosestSelectionRollback(t *testing.T) {
function TestClosestNodesItself (line 742) | func TestClosestNodesItself(t *testing.T) {
function TestClosestNodesNoDupes (line 748) | func TestClosestNodesNoDupes(t *testing.T) {
function TestClosestNodesNone (line 755) | func TestClosestNodesNone(t *testing.T) {
function TestClosestNodesMany (line 761) | func TestClosestNodesMany(t *testing.T) {
function TestClosestNodesRollback (line 768) | func TestClosestNodesRollback(t *testing.T) {
function TestIssue26 (line 774) | func TestIssue26(t *testing.T) {
FILE: common/goquery/type.go
type Document (line 22) | type Document struct
function NewDocumentFromNode (line 30) | func NewDocumentFromNode(root *html.Node) *Document {
function NewDocument (line 37) | func NewDocument(url string) result.Result[*Document] {
function NewDocumentFromReader (line 49) | func NewDocumentFromReader(r io.Reader) result.Result[*Document] {
function NewDocumentFromResponse (line 60) | func NewDocumentFromResponse(res *http.Response) result.Result[*Document] {
function CloneDocument (line 78) | func CloneDocument(doc *Document) *Document {
function newDocument (line 83) | func newDocument(root *html.Node, url *url.URL) *Document {
type Selection (line 93) | type Selection struct
function newEmptySelection (line 100) | func newEmptySelection(doc *Document) *Selection {
function newSingleSelection (line 105) | func newSingleSelection(node *html.Node, doc *Document) *Selection {
type Matcher (line 112) | type Matcher interface
function compileMatcher (line 121) | func compileMatcher(s string) Matcher {
type invalidMatcher (line 130) | type invalidMatcher struct
method Match (line 132) | func (invalidMatcher) Match(n *html.Node) bool { return fa...
method MatchAll (line 133) | func (invalidMatcher) MatchAll(n *html.Node) []*html.Node { return nil }
method Filter (line 134) | func (invalidMatcher) Filter(ns []*html.Node) []*html.Node { return nil }
FILE: common/goquery/type_test.go
function Doc (line 20) | func Doc() *Document {
function Doc2 (line 27) | func Doc2() *Document {
function Doc2Clone (line 34) | func Doc2Clone() *Document {
function Doc3 (line 38) | func Doc3() *Document {
function Doc3Clone (line 45) | func Doc3Clone() *Document {
function DocB (line 49) | func DocB() *Document {
function DocW (line 56) | func DocW() *Document {
function assertLength (line 63) | func assertLength(t *testing.T, nodes []*html.Node, length int) {
function assertClass (line 72) | func assertClass(t *testing.T, sel *Selection, class string) {
function assertPanic (line 78) | func assertPanic(t *testing.T) {
function assertEqual (line 84) | func assertEqual(t *testing.T, s1 *Selection, s2 *Selection) {
function assertSelectionIs (line 90) | func assertSelectionIs(t *testing.T, sel *Selection, is ...string) {
function printSel (line 98) | func printSel(t *testing.T, sel *Selection) {
function loadDoc (line 108) | func loadDoc(page string) *Document {
function TestNewDocument (line 124) | func TestNewDocument(t *testing.T) {
function TestNewDocumentFromReader (line 137) | func TestNewDocumentFromReader(t *testing.T) {
function TestNewDocumentFromResponseNil (line 186) | func TestNewDocumentFromResponseNil(t *testing.T) {
function TestIssue103 (line 193) | func TestIssue103(t *testing.T) {
FILE: common/goquery/utilities.go
constant minNodesForSet (line 12) | minNodesForSet = 1000
function NodeName (line 34) | func NodeName(s *Selection) string {
function OuterHtml (line 56) | func OuterHtml(s *Selection) (string, error) {
function sliceContains (line 70) | func sliceContains(container []*html.Node, contained *html.Node) bool {
function nodeContains (line 81) | func nodeContains(container *html.Node, contained *html.Node) bool {
function isInSlice (line 93) | func isInSlice(slice []*html.Node, node *html.Node) bool {
function indexInSlice (line 98) | func indexInSlice(slice []*html.Node, node *html.Node) int {
function appendWithoutDuplicates (line 114) | func appendWithoutDuplicates(target []*html.Node, nodes []*html.Node, ta...
function grep (line 146) | func grep(sel *Selection, predicate func(i int, s *Selection) bool) (res...
function pushStack (line 157) | func pushStack(fromSel *Selection, nodes []*html.Node) *Selection {
FILE: common/goquery/utilities_test.go
function TestNodeName (line 27) | func TestNodeName(t *testing.T) {
function TestNodeNameMultiSel (line 66) | func TestNodeNameMultiSel(t *testing.T) {
function TestOuterHtml (line 86) | func TestOuterHtml(t *testing.T) {
FILE: common/kafka/kafka.go
type KafkaSender (line 25) | type KafkaSender struct
method SetTopic (line 54) | func (p *KafkaSender) SetTopic(topic string) {
method Push (line 59) | func (p *KafkaSender) Push(data map[string]interface{}) result.VoidRes...
function GetProducer (line 30) | func GetProducer() result.Result[sarama.SyncProducer] {
function Refresh (line 35) | func Refresh() {
function New (line 49) | func New() *KafkaSender {
FILE: common/kafka/kafka_test.go
type mockSyncProducer (line 11) | type mockSyncProducer struct
method SendMessage (line 17) | func (m *mockSyncProducer) SendMessage(msg *sarama.ProducerMessage) (i...
method SendMessages (line 24) | func (m *mockSyncProducer) SendMessages(msgs []*sarama.ProducerMessage...
method Close (line 31) | func (m *mockSyncProducer) Close() error {
function TestNew (line 38) | func TestNew(t *testing.T) {
function TestSetTopic (line 45) | func TestSetTopic(t *testing.T) {
function TestGetProducer_BeforeRefresh (line 53) | func TestGetProducer_BeforeRefresh(t *testing.T) {
function TestRefresh (line 60) | func TestRefresh(t *testing.T) {
function TestGetProducer_AfterRefresh (line 64) | func TestGetProducer_AfterRefresh(t *testing.T) {
function TestPush_NilProducer (line 76) | func TestPush_NilProducer(t *testing.T) {
function TestPush_Success (line 92) | func TestPush_Success(t *testing.T) {
function TestPush_SendError (line 105) | func TestPush_SendError(t *testing.T) {
function TestPush_EmptyData (line 121) | func TestPush_EmptyData(t *testing.T) {
FILE: common/mahonia/8bit.go
type eightBitInfo (line 10) | type eightBitInfo struct
method register (line 32) | func (info *eightBitInfo) register() {
method unpack (line 91) | func (info *eightBitInfo) unpack() {
constant asciiRepertoire (line 30) | asciiRepertoire = "\x00\x01\x02\x03\x04\x05\x06\a\b\t\n\v\f\r\x0e\x0f\x1...
function init (line 108) | func init() {
FILE: common/mahonia/ASCII.go
function init (line 5) | func init() {
function decodeASCIIRune (line 26) | func decodeASCIIRune(p []byte) (c rune, size int, status Status) {
function encodeASCIIRune (line 39) | func encodeASCIIRune(p []byte, c rune) (size int, status Status) {
function decodeLatin1Rune (line 54) | func decodeLatin1Rune(p []byte) (c rune, size int, status Status) {
function encodeLatin1Rune (line 63) | func encodeLatin1Rune(p []byte, c rune) (size int, status Status) {
FILE: common/mahonia/big5.go
function init (line 9) | func init() {
function decodeBig5Rune (line 23) | func decodeBig5Rune(p []byte) (r rune, size int, status Status) {
function encodeBig5Rune (line 48) | func encodeBig5Rune(p []byte, r rune) (size int, status Status) {
function reverseBig5Table (line 81) | func reverseBig5Table() {
FILE: common/mahonia/charset.go
type Status (line 13) | type Status
constant SUCCESS (line 17) | SUCCESS = Status(iota)
constant INVALID_CHAR (line 22) | INVALID_CHAR
constant NO_ROOM (line 27) | NO_ROOM
constant STATE_ONLY (line 31) | STATE_ONLY
type Decoder (line 36) | type Decoder
type Encoder (line 40) | type Encoder
type Charset (line 44) | type Charset struct
function simplifyName (line 66) | func simplifyName(name string) string {
function RegisterCharset (line 83) | func RegisterCharset(cs *Charset) {
function GetCharset (line 94) | func GetCharset(name string) *Charset {
function NewDecoder (line 100) | func NewDecoder(name string) Decoder {
function NewEncoder (line 109) | func NewEncoder(name string) Encoder {
FILE: common/mahonia/convert_string.go
method ConvertString (line 8) | func (e Encoder) ConvertString(s string) string {
method ConvertString (line 35) | func (d Decoder) ConvertString(s string) string {
method ConvertStringOK (line 65) | func (e Encoder) ConvertStringOK(s string) (result string, ok bool) {
method ConvertStringOK (line 106) | func (d Decoder) ConvertStringOK(s string) (result string, ok bool) {
FILE: common/mahonia/cp51932.go
function init (line 9) | func init() {
function decodeCP51932 (line 23) | func decodeCP51932(p []byte) (c rune, size int, status Status) {
function encodeCP51932 (line 50) | func encodeCP51932(p []byte, c rune) (size int, status Status) {
FILE: common/mahonia/entity.go
function EntityDecoder (line 12) | func EntityDecoder() Decoder {
FILE: common/mahonia/euc-jp.go
function init (line 9) | func init() {
function decodeEucJP (line 24) | func decodeEucJP(p []byte) (c rune, size int, status Status) {
function encodeEucJP (line 61) | func encodeEucJP(p []byte, c rune) (size int, status Status) {
FILE: common/mahonia/euc-kr-data.go
function reverseEucKrTable (line 11) | func reverseEucKrTable() {
FILE: common/mahonia/euc-kr.go
function init (line 9) | func init() {
function decodeEucKr (line 38) | func decodeEucKr(p []byte) (c rune, size int, status Status) {
function encodeEucKr (line 61) | func encodeEucKr(p []byte, c rune) (size int, status Status) {
FILE: common/mahonia/fallback.go
function FallbackDecoder (line 9) | func FallbackDecoder(decoders ...Decoder) Decoder {
FILE: common/mahonia/gb18030-data.go
function gb18030Linear (line 9) | func gb18030Linear(g uint32) uint32 {
constant maxGB18030Linear (line 38) | maxGB18030Linear = 39393
FILE: common/mahonia/gb18030.go
function init (line 9) | func init() {
function decodeGB18030Rune (line 23) | func decodeGB18030Rune(p []byte) (r rune, size int, status Status) {
function encodeGB18030Rune (line 83) | func encodeGB18030Rune(p []byte, r rune) (size int, status Status) {
function buildGB18030Tables (line 149) | func buildGB18030Tables() {
FILE: common/mahonia/gbk.go
function init (line 5) | func init() {
function decodeGBKRune (line 18) | func decodeGBKRune(p []byte) (r rune, size int, status Status) {
function encodeGBKRune (line 47) | func encodeGBKRune(p []byte, r rune) (size int, status Status) {
FILE: common/mahonia/iso2022jp.go
constant esc (line 9) | esc = 27
function init (line 11) | func init() {
FILE: common/mahonia/kuten.go
type kutenTable (line 11) | type kutenTable struct
method Reverse (line 25) | func (t *kutenTable) Reverse() {
method DecodeLow (line 38) | func (t *kutenTable) DecodeLow(p []byte) (c rune, size int, status Sta...
method DecodeHigh (line 55) | func (t *kutenTable) DecodeHigh(p []byte) (c rune, size int, status St...
method EncodeHigh (line 72) | func (t *kutenTable) EncodeHigh(p []byte, c rune) (size int, status St...
FILE: common/mahonia/mahonia_test.go
function TestCharset (line 10) | func TestCharset(t *testing.T) {
function TestSimplifyName (line 32) | func TestSimplifyName(t *testing.T) {
function TestNewDecoderEncoder (line 38) | func TestNewDecoderEncoder(t *testing.T) {
function TestConvertStringUTF8 (line 56) | func TestConvertStringUTF8(t *testing.T) {
function TestConvertStringGBK (line 68) | func TestConvertStringGBK(t *testing.T) {
function TestConvertStringOK (line 80) | func TestConvertStringOK(t *testing.T) {
function TestReader (line 94) | func TestReader(t *testing.T) {
function TestReaderReadRune (line 107) | func TestReaderReadRune(t *testing.T) {
function TestWriter (line 119) | func TestWriter(t *testing.T) {
function TestWriterWriteRune (line 135) | func TestWriterWriteRune(t *testing.T) {
function TestStatusConstants (line 148) | func TestStatusConstants(t *testing.T) {
function TestDecoderConvertStringEmpty (line 154) | func TestDecoderConvertStringEmpty(t *testing.T) {
function TestEncoderConvertStringEmpty (line 161) | func TestEncoderConvertStringEmpty(t *testing.T) {
function TestReaderEOF (line 168) | func TestReaderEOF(t *testing.T) {
function TestReaderLargeRead (line 178) | func TestReaderLargeRead(t *testing.T) {
function TestConvertStringLongString (line 192) | func TestConvertStringLongString(t *testing.T) {
function TestConvertStringBig5 (line 203) | func TestConvertStringBig5(t *testing.T) {
function TestConvertStringGB2312 (line 214) | func TestConvertStringGB2312(t *testing.T) {
function TestConvertStringGB18030 (line 225) | func TestConvertStringGB18030(t *testing.T) {
function TestConvertStringISO8859_1 (line 236) | func TestConvertStringISO8859_1(t *testing.T) {
function TestWriterPartialRune (line 247) | func TestWriterPartialRune(t *testing.T) {
function TestWriterWriteRuneMultiByte (line 258) | func TestWriterWriteRuneMultiByte(t *testing.T) {
function TestReaderSmallBuffer (line 268) | func TestReaderSmallBuffer(t *testing.T) {
function TestConvertStringOKInvalidUTF8 (line 278) | func TestConvertStringOKInvalidUTF8(t *testing.T) {
function TestSimplifyNameWithDigits (line 286) | func TestSimplifyNameWithDigits(t *testing.T) {
function TestConvertStringUTF16 (line 292) | func TestConvertStringUTF16(t *testing.T) {
function TestConvertStringUTF16BE (line 303) | func TestConvertStringUTF16BE(t *testing.T) {
function TestConvertStringUTF16LE (line 314) | func TestConvertStringUTF16LE(t *testing.T) {
function TestReaderPartialUTF8AtEOF (line 325) | func TestReaderPartialUTF8AtEOF(t *testing.T) {
function TestEncoderConvertStringOKInvalidRune (line 338) | func TestEncoderConvertStringOKInvalidRune(t *testing.T) {
function TestDecoderConvertStringOKNoRoom (line 346) | func TestDecoderConvertStringOKNoRoom(t *testing.T) {
function TestConvertStringShiftJIS (line 354) | func TestConvertStringShiftJIS(t *testing.T) {
function TestConvertStringEUCJP (line 365) | func TestConvertStringEUCJP(t *testing.T) {
function TestConvertStringEUCKR (line 376) | func TestConvertStringEUCKR(t *testing.T) {
function TestConvertStringASCII (line 387) | func TestConvertStringASCII(t *testing.T) {
function TestConvertStringLatin1 (line 398) | func TestConvertStringLatin1(t *testing.T) {
function TestEncoderConvertStringNoRoom (line 409) | func TestEncoderConvertStringNoRoom(t *testing.T) {
function TestConvertStringISO2022JP (line 418) | func TestConvertStringISO2022JP(t *testing.T) {
function TestConvertStringCP51932 (line 429) | func TestConvertStringCP51932(t *testing.T) {
function TestConvertStringTCVN3 (line 440) | func TestConvertStringTCVN3(t *testing.T) {
function TestReaderReadRuneEOF (line 451) | func TestReaderReadRuneEOF(t *testing.T) {
function TestWriterBufferedWrite (line 460) | func TestWriterBufferedWrite(t *testing.T) {
FILE: common/mahonia/mahoniconv/mahoniconv.go
function main (line 17) | func main() {
FILE: common/mahonia/mbcs.go
type mbcsTrie (line 8) | type mbcsTrie struct
type MBCSTable (line 17) | type MBCSTable struct
method AddCharacter (line 24) | func (table *MBCSTable) AddCharacter(c rune, bytes string) {
method Decoder (line 44) | func (table *MBCSTable) Decoder() Decoder {
method Encoder (line 74) | func (table *MBCSTable) Encoder() Encoder {
FILE: common/mahonia/reader.go
constant defaultBufSize (line 16) | defaultBufSize = 4096
type Reader (line 20) | type Reader struct
method fill (line 38) | func (b *Reader) fill() {
method Read (line 59) | func (b *Reader) Read(p []byte) (n int, err error) {
method ReadRune (line 125) | func (b *Reader) ReadRune() (c rune, size int, err error) {
method NewReader (line 29) | func (d Decoder) NewReader(rd io.Reader) *Reader {
FILE: common/mahonia/shiftjis-data.go
function reverseShiftJISTable (line 11) | func reverseShiftJISTable() {
FILE: common/mahonia/shiftjis.go
function init (line 9) | func init() {
function decodeSJIS (line 23) | func decodeSJIS(p []byte) (c rune, size int, status Status) {
function encodeSJIS (line 54) | func encodeSJIS(p []byte, c rune) (size int, status Status) {
FILE: common/mahonia/tcvn3.go
function init (line 20) | func init() {
function decodeTCVN3 (line 34) | func decodeTCVN3(p []byte) (rune, int, Status) {
function encodeTCVN3 (line 53) | func encodeTCVN3(p []byte, c rune) (int, Status) {
function buildTCVN3Tables (line 78) | func buildTCVN3Tables() {
FILE: common/mahonia/translate.go
method Translate (line 6) | func (d Decoder) Translate(data []byte, eof bool) (n int, cdata []byte, ...
function doubleLength (line 46) | func doubleLength(b []byte) []byte {
FILE: common/mahonia/utf16.go
function init (line 7) | func init() {
function decodeUTF16beRune (line 72) | func decodeUTF16beRune(p []byte) (r rune, size int, status Status) {
function encodeUTF16beRune (line 99) | func encodeUTF16beRune(p []byte, c rune) (size int, status Status) {
function decodeUTF16leRune (line 122) | func decodeUTF16leRune(p []byte) (r rune, size int, status Status) {
function encodeUTF16leRune (line 149) | func encodeUTF16leRune(p []byte, c rune) (size int, status Status) {
FILE: common/mahonia/utf8.go
function init (line 5) | func init() {
function decodeUTF8Rune (line 13) | func decodeUTF8Rune(p []byte) (c rune, size int, status Status) {
function encodeUTF8Rune (line 38) | func encodeUTF8Rune(p []byte, c rune) (size int, status Status) {
FILE: common/mahonia/writer.go
type Writer (line 9) | type Writer struct
method Write (line 25) | func (w *Writer) Write(p []byte) (n int, err error) {
method WriteRune (line 79) | func (w *Writer) WriteRune(c rune) (size int, err error) {
method NewWriter (line 17) | func (e Encoder) NewWriter(wr io.Writer) *Writer {
FILE: common/mgo/count.go
type Count (line 13) | type Count struct
method Exec (line 19) | func (cnt *Count) Exec(resultPtr interface{}) (r result.Result[any]) {
FILE: common/mgo/find.go
type Find (line 13) | type Find struct
method Exec (line 23) | func (f *Find) Exec(resultPtr interface{}) (r result.Result[any]) {
FILE: common/mgo/insert.go
type Insert (line 11) | type Insert struct
method Exec (line 21) | func (i *Insert) Exec(resultPtr interface{}) (r result.Result[any]) {
constant MaxLen (line 18) | MaxLen int = 5000
FILE: common/mgo/list.go
type List (line 9) | type List struct
method Exec (line 13) | func (l *List) Exec(resultPtr interface{}) (r result.Result[any]) {
FILE: common/mgo/mgo.go
type sessionProvider (line 17) | type sessionProvider interface
type dbProvider (line 21) | type dbProvider interface
type collectionProvider (line 25) | type collectionProvider interface
type queryProvider (line 33) | type queryProvider interface
type MgoSrc (line 44) | type MgoSrc struct
method Usable (line 170) | func (ms *MgoSrc) Usable() bool {
method Reset (line 178) | func (*MgoSrc) Reset() {}
method Close (line 181) | func (ms *MgoSrc) Close() {
type mgoSessionAdapter (line 48) | type mgoSessionAdapter struct
method DB (line 50) | func (m *mgoSessionAdapter) DB(name string) dbProvider {
method DatabaseNames (line 54) | func (m *mgoSessionAdapter) DatabaseNames() ([]string, error) {
type mgoDbAdapter (line 62) | type mgoDbAdapter struct
method C (line 64) | func (m *mgoDbAdapter) C(name string) collectionProvider {
method CollectionNames (line 68) | func (m *mgoDbAdapter) CollectionNames() ([]string, error) {
type mgoCollectionAdapter (line 72) | type mgoCollectionAdapter struct
method Find (line 74) | func (m *mgoCollectionAdapter) Find(query interface{}) queryProvider {
method Insert (line 78) | func (m *mgoCollectionAdapter) Insert(docs ...interface{}) error {
method Remove (line 82) | func (m *mgoCollectionAdapter) Remove(selector interface{}) error {
method Update (line 86) | func (m *mgoCollectionAdapter) Update(selector, update interface{}) er...
method UpdateAll (line 90) | func (m *mgoCollectionAdapter) UpdateAll(selector, update interface{})...
method Upsert (line 94) | func (m *mgoCollectionAdapter) Upsert(selector, update interface{}) (*...
type mgoQueryAdapter (line 98) | type mgoQueryAdapter struct
method Count (line 100) | func (m *mgoQueryAdapter) Count() (int, error) {
method Sort (line 104) | func (m *mgoQueryAdapter) Sort(fields ...string) queryProvider {
method Skip (line 108) | func (m *mgoQueryAdapter) Skip(n int) queryProvider {
method Limit (line 112) | func (m *mgoQueryAdapter) Limit(n int) queryProvider {
method Select (line 116) | func (m *mgoQueryAdapter) Select(selector interface{}) queryProvider {
method All (line 120) | func (m *mgoQueryAdapter) All(result interface{}) error {
function getPool (line 150) | func getPool() pool.Pool {
function Refresh (line 158) | func Refresh() {
function Error (line 189) | func Error() error {
function Call (line 194) | func Call(fn func(pool.Src) error) result.VoidResult {
function Close (line 199) | func Close() {
function Len (line 204) | func Len() int {
function DatabaseNames (line 209) | func DatabaseNames() result.Result[[]string] {
function CollectionNames (line 223) | func CollectionNames(dbname string) result.Result[[]string] {
FILE: common/mgo/mgo_test.go
type mockQuery (line 14) | type mockQuery struct
method Count (line 21) | func (m *mockQuery) Count() (int, error) {
method Sort (line 28) | func (m *mockQuery) Sort(_ ...string) queryProvider { return m }
method Skip (line 30) | func (m *mockQuery) Skip(_ int) queryProvider { return m }
method Limit (line 32) | func (m *mockQuery) Limit(_ int) queryProvider { return m }
method Select (line 34) | func (m *mockQuery) Select(_ interface{}) queryProvider { return m }
method All (line 36) | func (m *mockQuery) All(result interface{}) error {
type mockCollection (line 44) | type mockCollection struct
method Find (line 55) | func (m *mockCollection) Find(query interface{}) queryProvider {
method Insert (line 62) | func (m *mockCollection) Insert(_ ...interface{}) error { return m.ins...
method Remove (line 64) | func (m *mockCollection) Remove(_ interface{}) error { return m.remove...
method Update (line 66) | func (m *mockCollection) Update(_, _ interface{}) error { return m.upd...
method UpdateAll (line 68) | func (m *mockCollection) UpdateAll(_, _ interface{}) (*mgo.ChangeInfo,...
method Upsert (line 72) | func (m *mockCollection) Upsert(_, _ interface{}) (*mgo.ChangeInfo, er...
type mockDatabase (line 76) | type mockDatabase struct
method C (line 82) | func (m *mockDatabase) C(name string) collectionProvider {
method CollectionNames (line 91) | func (m *mockDatabase) CollectionNames() ([]string, error) {
type mockSession (line 95) | type mockSession struct
method Usable (line 103) | func (m *mockSession) Usable() bool {
method Reset (line 109) | func (m *mockSession) Reset() {}
method Close (line 111) | func (m *mockSession) Close() {}
method DB (line 113) | func (m *mockSession) DB(name string) dbProvider {
method DatabaseNames (line 124) | func (m *mockSession) DatabaseNames() ([]string, error) {
type mockPool (line 130) | type mockPool struct
method Call (line 134) | func (m *mockPool) Call(fn func(pool.Src) error) result.VoidResult {
method Close (line 138) | func (m *mockPool) Close() {}
method Len (line 140) | func (m *mockPool) Len() int { return 0 }
function setupMockPool (line 142) | func setupMockPool(s sessionProvider) {
function teardownMockPool (line 147) | func teardownMockPool() {
function TestGetOperator (line 154) | func TestGetOperator(t *testing.T) {
function TestMgo_InvalidOp (line 177) | func TestMgo_InvalidOp(t *testing.T) {
function TestMgo_UnknownOptionField (line 184) | func TestMgo_UnknownOptionField(t *testing.T) {
function TestMgo_OptionReflection (line 210) | func TestMgo_OptionReflection(t *testing.T) {
function TestList_Exec (line 238) | func TestList_Exec(t *testing.T) {
function TestList_Exec_AllDbs (line 265) | func TestList_Exec_AllDbs(t *testing.T) {
function TestCount_Exec (line 286) | func TestCount_Exec(t *testing.T) {
function TestCount_Exec_ObjectId (line 316) | func TestCount_Exec_ObjectId(t *testing.T) {
function TestCount_Exec_InvalidIdType (line 344) | func TestCount_Exec_InvalidIdType(t *testing.T) {
function TestFind_Exec (line 360) | func TestFind_Exec(t *testing.T) {
function TestFind_Exec_InvalidIdType (line 398) | func TestFind_Exec_InvalidIdType(t *testing.T) {
function TestInsert_Exec (line 414) | func TestInsert_Exec(t *testing.T) {
function TestInsert_Exec_NilResultPtr (line 451) | func TestInsert_Exec_NilResultPtr(t *testing.T) {
function TestRemove_Exec (line 475) | func TestRemove_Exec(t *testing.T) {
function TestRemove_Exec_ObjectId (line 499) | func TestRemove_Exec_ObjectId(t *testing.T) {
function TestRemove_Exec_InvalidIdType (line 523) | func TestRemove_Exec_InvalidIdType(t *testing.T) {
function TestUpdate_Exec (line 538) | func TestUpdate_Exec(t *testing.T) {
function TestUpdate_Exec_InvalidIdType (line 563) | func TestUpdate_Exec_InvalidIdType(t *testing.T) {
function TestUpdateAll_Exec (line 579) | func TestUpdateAll_Exec(t *testing.T) {
function TestUpdateAll_Exec_InvalidIdType (line 610) | func TestUpdateAll_Exec_InvalidIdType(t *testing.T) {
function TestUpsert_Exec (line 627) | func TestUpsert_Exec(t *testing.T) {
function TestUpsert_Exec_InvalidIdType (line 658) | func TestUpsert_Exec_InvalidIdType(t *testing.T) {
function TestMgoSrc_Usable (line 675) | func TestMgoSrc_Usable(t *testing.T) {
function TestMgoSrc_Close (line 682) | func TestMgoSrc_Close(t *testing.T) {
function TestMgoSrc_Reset (line 687) | func TestMgoSrc_Reset(t *testing.T) {
function TestError (line 692) | func TestError(t *testing.T) {
function TestDatabaseNames (line 696) | func TestDatabaseNames(t *testing.T) {
function TestDatabaseNames_Err (line 714) | func TestDatabaseNames_Err(t *testing.T) {
function TestCollectionNames_Err (line 728) | func TestCollectionNames_Err(t *testing.T) {
function TestCollectionNames (line 744) | func TestCollectionNames(t *testing.T) {
function TestLen (line 763) | func TestLen(t *testing.T) {
function TestClose (line 773) | func TestClose(t *testing.T) {
function TestList_DatabaseNamesErr (line 781) | func TestList_DatabaseNamesErr(t *testing.T) {
function TestList_CollectionNamesErr (line 796) | func TestList_CollectionNamesErr(t *testing.T) {
function TestCount_QueryErr (line 814) | func TestCount_QueryErr(t *testing.T) {
function TestFind_CountErr (line 839) | func TestFind_CountErr(t *testing.T) {
function TestFind_AllErr (line 864) | func TestFind_AllErr(t *testing.T) {
function TestInsert_InsertErr (line 889) | func TestInsert_InsertErr(t *testing.T) {
function TestRemove_RemoveErr (line 913) | func TestRemove_RemoveErr(t *testing.T) {
function TestUpdate_UpdateErr (line 935) | func TestUpdate_UpdateErr(t *testing.T) {
function TestUpdateAll_UpdateAllErr (line 958) | func TestUpdateAll_UpdateAllErr(t *testing.T) {
function TestUpsert_UpsertErr (line 982) | func TestUpsert_UpsertErr(t *testing.T) {
FILE: common/mgo/operator.go
function Mgo (line 13) | func Mgo(resultPtr interface{}, operate string, option map[string]interf...
type Operator (line 32) | type Operator interface
function getOperator (line 37) | func getOperator(operate string) Operator {
FILE: common/mgo/remove.go
type Remove (line 13) | type Remove struct
method Exec (line 19) | func (r *Remove) Exec(_ interface{}) (res result.Result[any]) {
FILE: common/mgo/update.go
type Update (line 13) | type Update struct
method Exec (line 20) | func (u *Update) Exec(_ interface{}) (r result.Result[any]) {
FILE: common/mgo/update_all.go
type UpdateAll (line 13) | type UpdateAll struct
method Exec (line 20) | func (ua *UpdateAll) Exec(resultPtr interface{}) (r result.Result[any]) {
FILE: common/mgo/upsert.go
type Upsert (line 13) | type Upsert struct
method Exec (line 20) | func (us *Upsert) Exec(resultPtr interface{}) (r result.Result[any]) {
FILE: common/mysql/mysql.go
type Table (line 18) | type Table struct
method Clone (line 91) | func (m *Table) Clone() *Table {
method SetTableName (line 100) | func (t *Table) SetTableName(name string) *Table {
method AddColumn (line 106) | func (t *Table) AddColumn(names ...string) *Table {
method CustomPrimaryKey (line 116) | func (t *Table) CustomPrimaryKey(primaryKeyCode string) *Table {
method Create (line 123) | func (t *Table) Create() (r result.VoidResult) {
method Truncate (line 150) | func (t *Table) Truncate() (r result.VoidResult) {
method addRow (line 162) | func (t *Table) addRow(value []string) *Table {
method AutoInsert (line 171) | func (t *Table) AutoInsert(value []string) *Table {
method FlushInsert (line 194) | func (t *Table) FlushInsert() (r result.VoidResult) {
method SelectAll (line 235) | func (t *Table) SelectAll() result.Result[*sql.Rows] {
type mysqlConst (line 28) | type mysqlConst struct
function getMysqlConst (line 46) | func getMysqlConst() *mysqlConst {
function DB (line 51) | func DB() (*sql.DB, error) {
function SetDBForTest (line 56) | func SetDBForTest(d *sql.DB) func() {
function Refresh (line 70) | func Refresh() {
function New (line 86) | func New() result.Result[*Table] {
function wrapSQLKey (line 249) | func wrapSQLKey(s string) string {
FILE: common/mysql/mysql_test.go
function TestNew (line 12) | func TestNew(t *testing.T) {
function TestTable_Clone (line 23) | func TestTable_Clone(t *testing.T) {
function TestTable_SetTableName (line 42) | func TestTable_SetTableName(t *testing.T) {
function TestTable_AddColumn (line 62) | func TestTable_AddColumn(t *testing.T) {
function TestTable_CustomPrimaryKey (line 79) | func TestTable_CustomPrimaryKey(t *testing.T) {
function TestDB (line 92) | func TestDB(t *testing.T) {
function TestTable_Create_EmptyColumns (line 102) | func TestTable_Create_EmptyColumns(t *testing.T) {
function TestTable_FlushInsert_NoRows (line 113) | func TestTable_FlushInsert_NoRows(t *testing.T) {
function TestTable_FlushInsert_NoColumns (line 121) | func TestTable_FlushInsert_NoColumns(t *testing.T) {
function TestTable_SelectAll_EmptyTableName (line 130) | func TestTable_SelectAll_EmptyTableName(t *testing.T) {
function TestTable_AutoInsert_OversizedPacket (line 141) | func TestTable_AutoInsert_OversizedPacket(t *testing.T) {
function TestTable_AutoInsert_SmallValue (line 152) | func TestTable_AutoInsert_SmallValue(t *testing.T) {
function TestTable_AutoInsert_MultipleColumns (line 163) | func TestTable_AutoInsert_MultipleColumns(t *testing.T) {
function TestNew_Unwrap (line 174) | func TestNew_Unwrap(t *testing.T) {
function TestTable_Clone_PreservesCustomPrimaryKey (line 185) | func TestTable_Clone_PreservesCustomPrimaryKey(t *testing.T) {
function TestTable_SetTableName_Chain (line 193) | func TestTable_SetTableName_Chain(t *testing.T) {
function TestTable_AddColumn_EmptyTrimmed (line 200) | func TestTable_AddColumn_EmptyTrimmed(t *testing.T) {
function setupMockDB (line 211) | func setupMockDB(t *testing.T) (*sql.DB, sqlmock.Sqlmock, func()) {
function TestTable_Create_WithMock (line 224) | func TestTable_Create_WithMock(t *testing.T) {
function TestTable_Create_CustomPrimaryKey_WithMock (line 241) | func TestTable_Create_CustomPrimaryKey_WithMock(t *testing.T) {
function TestTable_Truncate_WithMock (line 258) | func TestTable_Truncate_WithMock(t *testing.T) {
function TestTable_FlushInsert_WithMock (line 275) | func TestTable_FlushInsert_WithMock(t *testing.T) {
function TestTable_SelectAll_WithMock (line 294) | func TestTable_SelectAll_WithMock(t *testing.T) {
function TestTable_Create_ExecError (line 316) | func TestTable_Create_ExecError(t *testing.T) {
function TestTable_Truncate_ExecError (line 333) | func TestTable_Truncate_ExecError(t *testing.T) {
function TestTable_FlushInsert_ExecError (line 350) | func TestTable_FlushInsert_ExecError(t *testing.T) {
function TestTable_SelectAll_QueryError (line 368) | func TestTable_SelectAll_QueryError(t *testing.T) {
FILE: common/ping/ping.go
constant icmpv4EchoRequest (line 25) | icmpv4EchoRequest = 8
constant icmpv4EchoReply (line 26) | icmpv4EchoReply = 0
constant icmpv6EchoRequest (line 27) | icmpv6EchoRequest = 128
constant icmpv6EchoReply (line 28) | icmpv6EchoReply = 129
type icmpMessage (line 31) | type icmpMessage struct
method Marshal (line 45) | func (m *icmpMessage) Marshal() ([]byte, error) {
type icmpMessageBody (line 38) | type icmpMessageBody interface
function parseICMPMessage (line 76) | func parseICMPMessage(b []byte) (*icmpMessage, error) {
type icmpEcho (line 96) | type icmpEcho struct
method Len (line 102) | func (p *icmpEcho) Len() int {
method Marshal (line 111) | func (p *icmpEcho) Marshal() ([]byte, error) {
function parseICMPEcho (line 120) | func parseICMPEcho(b []byte) (*icmpEcho, error) {
type PingResult (line 131) | type PingResult struct
function Ping (line 137) | func Ping(address string, timeoutSecond int) result.Result[PingResult] {
function Pinger (line 147) | func Pinger(address string, timeoutSecond int) (r result.VoidResult) {
function ipv4Payload (line 181) | func ipv4Payload(b []byte) []byte {
FILE: common/ping/ping_test.go
type emptyBody (line 16) | type emptyBody struct
method Len (line 18) | func (*emptyBody) Len() int { return 0 }
method Marshal (line 19) | func (*emptyBody) Marshal() ([]byte, error) { return nil, nil }
type errorBody (line 21) | type errorBody struct
method Len (line 23) | func (*errorBody) Len() int { return 4 }
method Marshal (line 24) | func (*errorBody) Marshal() ([]byte, error) { return nil, errors.New("...
function TestPing (line 26) | func TestPing(t *testing.T) {
function TestIcmpEchoLen (line 30) | func TestIcmpEchoLen(t *testing.T) {
function TestIcmpEchoMarshal (line 55) | func TestIcmpEchoMarshal(t *testing.T) {
function TestParseICMPEcho (line 78) | func TestParseICMPEcho(t *testing.T) {
function TestParseICMPMessage (line 101) | func TestParseICMPMessage(t *testing.T) {
function TestIcmpMessageMarshal (line 192) | func TestIcmpMessageMarshal(t *testing.T) {
function TestIcmpMessageMarshalBodyError (line 260) | func TestIcmpMessageMarshalBodyError(t *testing.T) {
function TestIcmpMessageMarshalRoundtrip (line 268) | func TestIcmpMessageMarshalRoundtrip(t *testing.T) {
function TestIpv4Payload (line 292) | func TestIpv4Payload(t *testing.T) {
function TestIpv4PayloadHdrlen (line 313) | func TestIpv4PayloadHdrlen(t *testing.T) {
FILE: common/pinyin/example_test.go
function TestExamplePinyin_default (line 9) | func TestExamplePinyin_default(t *testing.T) {
function TestExamplePinyin_normal (line 16) | func TestExamplePinyin_normal(t *testing.T) {
function TestExamplePinyin_tone (line 24) | func TestExamplePinyin_tone(t *testing.T) {
function TestExamplePinyin_tone2 (line 32) | func TestExamplePinyin_tone2(t *testing.T) {
function TestExamplePinyin_initials (line 40) | func TestExamplePinyin_initials(t *testing.T) {
function TestExamplePinyin_firstLetter (line 48) | func TestExamplePinyin_firstLetter(t *testing.T) {
function TestExamplePinyin_finals (line 56) | func TestExamplePinyin_finals(t *testing.T) {
function TestExamplePinyin_finalsTone (line 64) | func TestExamplePinyin_finalsTone(t *testing.T) {
function TestExamplePinyin_finalsTone2 (line 72) | func TestExamplePinyin_finalsTone2(t *testing.T) {
function TestExamplePinyin_heteronym (line 80) | func TestExamplePinyin_heteronym(t *testing.T) {
function TestExampleLazyPinyin (line 89) | func TestExampleLazyPinyin(t *testing.T) {
function TestExampleSlug (line 96) | func TestExampleSlug(t *testing.T) {
FILE: common/pinyin/initials_sort.go
function SortInitials (line 8) | func SortInitials(strs []string) {
FILE: common/pinyin/pinyin.go
constant Version (line 13) | Version = "0.2.1"
constant Author (line 14) | Author = "mozillazg, 闲耘"
constant License (line 15) | License = "MIT"
constant Copyright (line 16) | Copyright = "Copyright (c) 2014 mozillazg, 闲耘"
constant Normal (line 21) | Normal = 0
constant Tone (line 22) | Tone = 1
constant Tone2 (line 23) | Tone2 = 2
constant Initials (line 24) | Initials = 3
constant FirstLetter (line 25) | FirstLetter = 4
constant Finals (line 26) | Finals = 5
constant FinalsTone (line 27) | FinalsTone = 6
constant FinalsTone2 (line 28) | FinalsTone2 = 7
constant NORMAL (line 33) | NORMAL = 0
constant TONE (line 34) | TONE = 1
constant TONE2 (line 35) | TONE2 = 2
constant INITIALS (line 36) | INITIALS = 3
constant FIRST_LETTER (line 37) | FIRST_LETTER = 4
constant FINALS (line 38) | FINALS = 5
constant FINALS_TONE (line 39) | FINALS_TONE = 6
constant FINALS_TONE2 (line 40) | FINALS_TONE2 = 7
type Args (line 61) | type Args struct
function NewArgs (line 77) | func NewArgs() Args {
function initial (line 82) | func initial(p string) string {
function final (line 94) | func final(p string) string {
function toFixed (line 102) | func toFixed(p string, a Args) string {
function applyStyle (line 129) | func applyStyle(p []string, a Args) []string {
function SinglePinyin (line 138) | func SinglePinyin(r rune, a Args) []string {
function Pinyin (line 153) | func Pinyin(s string, a Args) [][]string {
function LazyPinyin (line 164) | func LazyPinyin(s string, a Args) []string {
function Slug (line 174) | func Slug(s string, a Args) string {
FILE: common/pinyin/pinyin_test.go
function TestSortInitials (line 8) | func TestSortInitials(t *testing.T) {
function TestFinalEmptyInitial (line 17) | func TestFinalEmptyInitial(t *testing.T) {
FILE: common/pool/pool.go
type Pool (line 16) | type Pool interface
type classic (line 22) | type classic struct
method Call (line 68) | func (c *classic) Call(callback func(Src) error) result.VoidResult {
method Close (line 104) | func (c *classic) Close() {
method Len (line 119) | func (c *classic) Len() int {
method gc (line 126) | func (c *classic) gc() {
method incAuto (line 141) | func (c *classic) incAuto() error {
method del (line 156) | func (c *classic) del(src Src) {
method recover (line 163) | func (c *classic) recover(src Src) {
method isClosed (line 173) | func (c *classic) isClosed() bool {
type Src (line 33) | type Src interface
type Factory (line 39) | type Factory
constant GC_TIME (line 43) | GC_TIME = 60e9
function ClassicPool (line 51) | func ClassicPool(capacity, maxIdle int, factory Factory, gctime ...time....
FILE: common/pool/pool_test.go
type mockSrc (line 10) | type mockSrc struct
method Usable (line 17) | func (m *mockSrc) Usable() bool {
method Reset (line 23) | func (m *mockSrc) Reset() {
method Close (line 29) | func (m *mockSrc) Close() {
function newMockSrc (line 35) | func newMockSrc(usable bool) *mockSrc {
function TestClassicPool_Creation (line 39) | func TestClassicPool_Creation(t *testing.T) {
function TestClassicPool_Call (line 78) | func TestClassicPool_Call(t *testing.T) {
function TestClassicPool_Len (line 163) | func TestClassicPool_Len(t *testing.T) {
function TestClassicPool_UsableFalse_Retry (line 202) | func TestClassicPool_UsableFalse_Retry(t *testing.T) {
function TestClassicPool_Call_PanicRecovery (line 223) | func TestClassicPool_Call_PanicRecovery(t *testing.T) {
function TestClassicPool_ConcurrentCalls (line 238) | func TestClassicPool_ConcurrentCalls(t *testing.T) {
FILE: common/queue/queue.go
type Queue (line 5) | type Queue struct
method Init (line 19) | func (this *Queue) Init(size int) *Queue {
method Push (line 26) | func (this *Queue) Push(i interface{}) bool {
method PushSlice (line 35) | func (this *Queue) PushSlice(s []interface{}) {
method Pull (line 42) | func (this *Queue) Pull() interface{} {
method Exchange (line 47) | func (this *Queue) Exchange(num int) (add int) {
function NewQueue (line 11) | func NewQueue(size int) *Queue {
FILE: common/queue/queue_test.go
function TestNewQueue (line 7) | func TestNewQueue(t *testing.T) {
function TestPushAndPull (line 17) | func TestPushAndPull(t *testing.T) {
function TestPushSlice (line 42) | func TestPushSlice(t *testing.T) {
function TestInit (line 50) | func TestInit(t *testing.T) {
function TestExchange (line 65) | func TestExchange(t *testing.T) {
FILE: common/session/sess_cookie.go
type CookieSessionStore (line 31) | type CookieSessionStore struct
method Set (line 38) | func (st *CookieSessionStore) Set(key, value interface{}) {
method Get (line 45) | func (st *CookieSessionStore) Get(key interface{}) option.Option[inter...
method Delete (line 53) | func (st *CookieSessionStore) Delete(key interface{}) {
method Flush (line 60) | func (st *CookieSessionStore) Flush() {
method SessionID (line 67) | func (st *CookieSessionStore) SessionID() string {
method SessionRelease (line 72) | func (st *CookieSessionStore) SessionRelease(w http.ResponseWriter) {
type cookieConfig (line 90) | type cookieConfig struct
type CookieProvider (line 100) | type CookieProvider struct
method SessionInit (line 111) | func (pder *CookieProvider) SessionInit(maxlifetime int64, config stri...
method SessionRead (line 132) | func (pder *CookieProvider) SessionRead(sid string) (Store, error) {
method SessionExist (line 145) | func (pder *CookieProvider) SessionExist(sid string) bool {
method SessionRegenerate (line 150) | func (pder *CookieProvider) SessionRegenerate(oldsid, sid string) (Sto...
method SessionDestroy (line 155) | func (pder *CookieProvider) SessionDestroy(sid string) error {
method SessionGC (line 160) | func (pder *CookieProvider) SessionGC() {
method SessionAll (line 165) | func (pder *CookieProvider) SessionAll() int {
method SessionUpdate (line 170) | func (pder *CookieProvider) SessionUpdate(sid string) error {
function init (line 174) | func init() {
FILE: common/session/sess_cookie_test.go
function TestCookie (line 24) | func TestCookie(t *testing.T) {
function TestCookieSessionStore (line 51) | func TestCookieSessionStore(t *testing.T) {
function TestCookieProvider (line 73) | func TestCookieProvider(t *testing.T) {
function TestDestorySessionCookie (line 91) | func TestDestorySessionCookie(t *testing.T) {
FILE: common/session/sess_file.go
type FileSessionStore (line 39) | type FileSessionStore struct
method Set (line 46) | func (fs *FileSessionStore) Set(key, value interface{}) {
method Get (line 53) | func (fs *FileSessionStore) Get(key interface{}) option.Option[interfa...
method Delete (line 61) | func (fs *FileSessionStore) Delete(key interface{}) {
method Flush (line 68) | func (fs *FileSessionStore) Flush() {
method SessionID (line 75) | func (fs *FileSessionStore) SessionID() string {
method SessionRelease (line 80) | func (fs *FileSessionStore) SessionRelease(w http.ResponseWriter) {
type FileProvider (line 104) | type FileProvider struct
method SessionInit (line 112) | func (fp *FileProvider) SessionInit(maxlifetime int64, savePath string...
method SessionRead (line 119) | func (fp *FileProvider) SessionRead(sid string) (Store, error) {
method SessionExist (line 156) | func (fp *FileProvider) SessionExist(sid string) bool {
method SessionDestroy (line 168) | func (fp *FileProvider) SessionDestroy(sid string) error {
method SessionGC (line 176) | func (fp *FileProvider) SessionGC() {
method SessionAll (line 185) | func (fp *FileProvider) SessionAll() int {
method SessionRegenerate (line 198) | func (fp *FileProvider) SessionRegenerate(oldsid, sid string) (Store, ...
function gcpath (line 251) | func gcpath(path string, info os.FileInfo, err error) error {
type activeSession (line 264) | type activeSession struct
method visit (line 268) | func (as *activeSession) visit(paths string, f os.FileInfo, err error)...
function init (line 279) | func init() {
FILE: common/session/sess_file_test.go
function TestFileProvider_InitAndRead (line 11) | func TestFileProvider_InitAndRead(t *testing.T) {
function TestFileProvider_Exist (line 28) | func TestFileProvider_Exist(t *testing.T) {
function TestFileProvider_Destroy (line 44) | func TestFileProvider_Destroy(t *testing.T) {
function TestFileProvider_GC (line 61) | func TestFileProvider_GC(t *testing.T) {
function TestFileProvider_SessionAll (line 81) | func TestFileProvider_SessionAll(t *testing.T) {
function TestFileSessionStore_SetGetDelete (line 97) | func TestFileSessionStore_SetGetDelete(t *testing.T) {
function TestFileSessionStore_Flush (line 118) | func TestFileSessionStore_Flush(t *testing.T) {
function TestFileProvider_Regenerate (line 132) | func TestFileProvider_Regenerate(t *testing.T) {
function TestFileSessionStore_SessionRelease (line 157) | func TestFileSessionStore_SessionRelease(t *testing.T) {
FILE: common/session/sess_mem.go
type MemSessionStore (line 29) | type MemSessionStore struct
method Set (line 37) | func (st *MemSessionStore) Set(key, value interface{}) {
method Get (line 44) | func (st *MemSessionStore) Get(key interface{}) option.Option[interfac...
method Delete (line 52) | func (st *MemSessionStore) Delete(key interface{}) {
method Flush (line 59) | func (st *MemSessionStore) Flush() {
method SessionID (line 66) | func (st *MemSessionStore) SessionID() string {
method SessionRelease (line 71) | func (st *MemSessionStore) SessionRelease(w http.ResponseWriter) {
type MemProvider (line 75) | type MemProvider struct
method SessionInit (line 84) | func (pder *MemProvider) SessionInit(maxlifetime int64, savePath strin...
method SessionRead (line 91) | func (pder *MemProvider) SessionRead(sid string) (Store, error) {
method SessionExist (line 109) | func (pder *MemProvider) SessionExist(sid string) bool {
method SessionRegenerate (line 117) | func (pder *MemProvider) SessionRegenerate(oldsid, sid string) (Store,...
method SessionDestroy (line 140) | func (pder *MemProvider) SessionDestroy(sid string) error {
method SessionGC (line 153) | func (pder *MemProvider) SessionGC() {
method SessionAll (line 175) | func (pder *MemProvider) SessionAll() int {
method SessionUpdate (line 180) | func (pder *MemProvider) SessionUpdate(sid string) error {
function init (line 192) | func init() {
FILE: common/session/sess_mem_test.go
function TestMem (line 24) | func TestMem(t *testing.T) {
function TestMemSessionStore (line 48) | func TestMemSessionStore(t *testing.T) {
function TestMemProvider (line 77) | func TestMemProvider(t *testing.T) {
function TestMemProviderGC (line 135) | func TestMemProviderGC(t *testing.T) {
FILE: common/session/sess_test.go
function Test_gob (line 23) | func Test_gob(t *testing.T) {
function TestEncodeGobDecodeGob (line 50) | func TestEncodeGobDecodeGob(t *testing.T) {
function TestDecodeGobInvalid (line 82) | func TestDecodeGobInvalid(t *testing.T) {
function TestRandomCreateBytes (line 89) | func TestRandomCreateBytes(t *testing.T) {
type User (line 106) | type User struct
function TestGenerate (line 111) | func TestGenerate(t *testing.T) {
function TestCookieEncodeDecode (line 118) | func TestCookieEncodeDecode(t *testing.T) {
function TestParseConfig (line 146) | func TestParseConfig(t *testing.T) {
function TestNewManager (line 190) | func TestNewManager(t *testing.T) {
FILE: common/session/sess_utils.go
function init (line 34) | func init() {
function EncodeGob (line 46) | func EncodeGob(obj map[interface{}]interface{}) ([]byte, error) {
function DecodeGob (line 60) | func DecodeGob(encoded []byte) (map[interface{}]interface{}, error) {
function generateRandomKey (line 72) | func generateRandomKey(strength int) []byte {
function encrypt (line 86) | func encrypt(block cipher.Block, value []byte) ([]byte, error) {
function decrypt (line 102) | func decrypt(block cipher.Block, value []byte) ([]byte, error) {
function encodeCookie (line 117) | func encodeCookie(block cipher.Block, hashKey, name string, value map[in...
function decodeCookie (line 136) | func decodeCookie(block cipher.Block, hashKey, name, value string, gcmax...
function encode (line 181) | func encode(value []byte) []byte {
function decode (line 188) | func decode(value []byte) ([]byte, error) {
function RandomCreateBytes (line 198) | func RandomCreateBytes(n int, alphabets ...byte) []byte {
FILE: common/session/session.go
type Store (line 51) | type Store interface
type Provider (line 62) | type Provider interface
function Register (line 80) | func Register(name string, provide Provider) {
type managerConfig (line 90) | type managerConfig struct
type Manager (line 106) | type Manager struct
method getSid (line 165) | func (manager *Manager) getSid(r *http.Request) (string, error) {
method SessionStart (line 195) | func (manager *Manager) SessionStart(w http.ResponseWriter, r *http.Re...
method SessionDestroy (line 241) | func (manager *Manager) SessionDestroy(w http.ResponseWriter, r *http....
method GetSessionStore (line 268) | func (manager *Manager) GetSessionStore(sid string) result.Result[Stor...
method GC (line 273) | func (manager *Manager) GC() {
method SessionRegenerateID (line 279) | func (manager *Manager) SessionRegenerateID(w http.ResponseWriter, r *...
method GetActiveSession (line 339) | func (manager *Manager) GetActiveSession() int {
method SetSecure (line 344) | func (manager *Manager) SetSecure(secure bool) {
method sessionID (line 348) | func (manager *Manager) sessionID() (string, error) {
method isSecure (line 358) | func (manager *Manager) isSecure(req *http.Request) bool {
function NewManager (line 114) | func NewManager(provideName, config string) (*Manager, error) {
type Log (line 372) | type Log struct
function NewSessionLog (line 377) | func NewSessionLog(out io.Writer) *Log {
FILE: common/session/session_manager_test.go
function TestManagerGetSid (line 10) | func TestManagerGetSid(t *testing.T) {
function TestManagerSessionStartExisting (line 24) | func TestManagerSessionStartExisting(t *testing.T) {
function TestManagerGetSessionStore (line 46) | func TestManagerGetSessionStore(t *testing.T) {
function TestManagerSessionDestroy (line 67) | func TestManagerSessionDestroy(t *testing.T) {
function TestManagerSessionDestroyNoCookie (line 83) | func TestManagerSessionDestroyNoCookie(t *testing.T) {
function TestManagerSessionRegenerateID (line 93) | func TestManagerSessionRegenerateID(t *testing.T) {
function TestManagerGetActiveSession (line 128) | func TestManagerGetActiveSession(t *testing.T) {
function TestManagerSetSecure (line 142) | func TestManagerSetSecure(t *testing.T) {
function TestManagerIsSecure (line 150) | func TestManagerIsSecure(t *testing.T) {
function TestManagerEnableSidInUrlQuery (line 180) | func TestManagerEnableSidInUrlQuery(t *testing.T) {
FILE: common/simplejson/simplejson.go
function Version (line 18) | func Version() string {
type Json (line 23) | type Json struct
method Interface (line 51) | func (j *Json) Interface() interface{} {
method Encode (line 56) | func (j *Json) Encode() ([]byte, error) {
method EncodePretty (line 61) | func (j *Json) EncodePretty() ([]byte, error) {
method MarshalJSON (line 66) | func (j *Json) MarshalJSON() ([]byte, error) {
method UnmarshalJSON (line 71) | func (j *Json) UnmarshalJSON(p []byte) error {
method Set (line 79) | func (j *Json) Set(key string, val interface{}) {
method SetPath (line 89) | func (j *Json) SetPath(branch []string, val interface{}) {
method Del (line 127) | func (j *Json) Del(key string) {
method Get (line 141) | func (j *Json) Get(key string) *Json {
method GetPath (line 156) | func (j *Json) GetPath(branch ...string) *Json {
method GetIndex (line 171) | func (j *Json) GetIndex(index int) *Json {
method CheckGet (line 189) | func (j *Json) CheckGet(key string) option.Option[*Json] {
method Map (line 201) | func (j *Json) Map() result.Result[map[string]interface{}] {
method Array (line 209) | func (j *Json) Array() result.Result[[]interface{}] {
method Float64 (line 217) | func (j *Json) Float64() result.Result[float64] {
method Int (line 233) | func (j *Json) Int() result.Result[int] {
method Int64 (line 249) | func (j *Json) Int64() result.Result[int64] {
method Uint64 (line 264) | func (j *Json) Uint64() result.Result[uint64] {
method Bool (line 280) | func (j *Json) Bool() result.Result[bool] {
method String (line 288) | func (j *Json) String() result.Result[string] {
method Bytes (line 296) | func (j *Json) Bytes() result.Result[[]byte] {
method StringArray (line 304) | func (j *Json) StringArray() result.Result[[]string] {
method IntArray (line 325) | func (j *Json) IntArray() result.Result[[]int] {
method Int64Array (line 347) | func (j *Json) Int64Array() result.Result[[]int64] {
method MustArray (line 375) | func (j *Json) MustArray(args ...[]interface{}) []interface{} {
method MustMap (line 396) | func (j *Json) MustMap(args ...map[string]interface{}) map[string]inte...
method MustString (line 415) | func (j *Json) MustString(args ...string) string {
method MustStringArray (line 436) | func (j *Json) MustStringArray(args ...[]string) []string {
method MustInt (line 455) | func (j *Json) MustInt(args ...int) int {
method MustFloat64 (line 474) | func (j *Json) MustFloat64(args ...float64) float64 {
method MustBool (line 493) | func (j *Json) MustBool(args ...bool) bool {
method MustInt64 (line 512) | func (j *Json) MustInt64(args ...int64) int64 {
method MustUint64 (line 531) | func (j *Json) MustUint64(args ...uint64) uint64 {
function NewJson (line 28) | func NewJson(body []byte) result.Result[*Json] {
function NewFromReader (line 35) | func NewFromReader(r io.Reader) result.Result[*Json] {
function New (line 44) | func New() *Json {
FILE: common/simplejson/simplejson_test.go
constant sampleJSON (line 11) | sampleJSON = `{"name":"John","age":30,"scores":[90,85,92],"address":{"ci...
function sampleJson (line 13) | func sampleJson(t *testing.T) *Json {
function TestVersion (line 22) | func TestVersion(t *testing.T) {
function TestNewJson (line 29) | func TestNewJson(t *testing.T) {
function TestNewFromReader (line 54) | func TestNewFromReader(t *testing.T) {
function TestNew (line 76) | func TestNew(t *testing.T) {
function TestInterface (line 87) | func TestInterface(t *testing.T) {
function TestGet (line 94) | func TestGet(t *testing.T) {
function TestGetPath (line 120) | func TestGetPath(t *testing.T) {
function TestGetIndex (line 145) | func TestGetIndex(t *testing.T) {
function TestCheckGet (line 173) | func TestCheckGet(t *testing.T) {
function TestSet (line 198) | func TestSet(t *testing.T) {
function TestSetPath (line 207) | func TestSetPath(t *testing.T) {
function TestDel (line 221) | func TestDel(t *testing.T) {
function TestMap (line 229) | func TestMap(t *testing.T) {
function TestArray (line 255) | func TestArray(t *testing.T) {
function TestFloat64 (line 280) | func TestFloat64(t *testing.T) {
function TestInt (line 309) | func TestInt(t *testing.T) {
function TestInt64 (line 338) | func TestInt64(t *testing.T) {
function TestUint64 (line 349) | func TestUint64(t *testing.T) {
function TestBool (line 360) | func TestBool(t *testing.T) {
function TestString (line 388) | func TestString(t *testing.T) {
function TestBytes (line 417) | func TestBytes(t *testing.T) {
function TestStringArray (line 428) | func TestStringArray(t *testing.T) {
function TestIntArray (line 454) | func TestIntArray(t *testing.T) {
function TestInt64Array (line 472) | func TestInt64Array(t *testing.T) {
function TestMustArray (line 490) | func TestMustArray(t *testing.T) {
function TestMustMap (line 502) | func TestMustMap(t *testing.T) {
function TestMustString (line 515) | func TestMustString(t *testing.T) {
function TestMustStringArray (line 527) | func TestMustStringArray(t *testing.T) {
function TestMustInt (line 543) | func TestMustInt(t *testing.T) {
function TestMustFloat64 (line 555) | func TestMustFloat64(t *testing.T) {
function TestMustBool (line 567) | func TestMustBool(t *testing.T) {
function TestMustInt64 (line 579) | func TestMustInt64(t *testing.T) {
function TestMustUint64 (line 591) | func TestMustUint64(t *testing.T) {
function TestEncode (line 603) | func TestEncode(t *testing.T) {
function TestEncodePretty (line 618) | func TestEncodePretty(t *testing.T) {
function TestMarshalJSON (line 633) | func TestMarshalJSON(t *testing.T) {
function TestUnmarshalJSON (line 644) | func TestUnmarshalJSON(t *testing.T) {
function TestSetPathEmptyBranch (line 655) | func TestSetPathEmptyBranch(t *testing.T) {
function TestSetPathOverwriteNonMap (line 663) | func TestSetPathOverwriteNonMap(t *testing.T) {
FILE: common/util/util.go
constant USE_KEYIN (line 32) | USE_KEYIN = "\r\t\n"
function JSONPToJSON (line 43) | func JSONPToJSON(json string) string {
function Mkdir (line 59) | func Mkdir(filePath string) result.VoidResult {
function GetWDPath (line 74) | func GetWDPath() string {
function IsDirExists (line 83) | func IsDirExists(path string) bool {
function IsFileExists (line 92) | func IsFileExists(path string) bool {
function walkPath (line 101) | func walkPath(targpath string) result.Result[string] {
function WalkFiles (line 109) | func WalkFiles(targpath string, suffixes ...string) (ret result.Result[[...
function WalkDir (line 135) | func WalkDir(targpath string, suffixes ...string) (ret result.Result[[]s...
function WalkRelFiles (line 161) | func WalkRelFiles(targpath string, suffixes ...string) (ret result.Resul...
function WalkRelDir (line 188) | func WalkRelDir(targpath string, suffixes ...string) (ret result.Result[...
function RelPath (line 215) | func RelPath(targpath string) (ret result.Result[string]) {
function IsNum (line 223) | func IsNum(a string) bool {
function XML2MapStr (line 228) | func XML2MapStr(xmldoc string) map[string]string {
function MakeHash (line 252) | func MakeHash(s string) string {
function HashString (line 259) | func HashString(encode string) uint64 {
function MakeUnique (line 266) | func MakeUnique(obj interface{}) string {
function MakeMd5 (line 274) | func MakeMd5(obj interface{}, length int) string {
function JSONString (line 286) | func JSONString(obj interface{}) string {
function CheckErr (line 295) | func CheckErr(err error) {
function CheckErrPanic (line 300) | func CheckErrPanic(err error) {
function FileNameReplace (line 307) | func FileNameReplace(fileName string) string {
function ExcelSheetNameReplace (line 342) | func ExcelSheetNameReplace(fileName string) string {
function Atoa (line 355) | func Atoa(str interface{}) option.Option[string] {
function Atoi (line 363) | func Atoi(str interface{}) option.Option[int] {
function Atoui (line 372) | func Atoui(str interface{}) option.Option[uint] {
function RandomCreateBytes (line 381) | func RandomCreateBytes(n int, alphabets ...byte) []byte {
function KeyinsParse (line 408) | func KeyinsParse(keyins string) []string {
function Bytes2String (line 437) | func Bytes2String(b []byte) string {
function String2Bytes (line 444) | func String2Bytes(s string) []byte {
FILE: common/util/util_test.go
function TestJSONPToJSON (line 11) | func TestJSONPToJSON(t *testing.T) {
function TestMkdir (line 34) | func TestMkdir(t *testing.T) {
function TestIsDirExists (line 68) | func TestIsDirExists(t *testing.T) {
function TestIsFileExists (line 93) | func TestIsFileExists(t *testing.T) {
function TestWalkFiles (line 118) | func TestWalkFiles(t *testing.T) {
function TestWalkDir (line 158) | func TestWalkDir(t *testing.T) {
function TestIsNum (line 182) | func TestIsNum(t *testing.T) {
function TestMakeHash (line 205) | func TestMakeHash(t *testing.T) {
function TestHashString (line 215) | func TestHashString(t *testing.T) {
function TestMakeUnique (line 225) | func TestMakeUnique(t *testing.T) {
function TestMakeMd5 (line 235) | func TestMakeMd5(t *testing.T) {
function TestJSONString (line 249) | func TestJSONString(t *testing.T) {
function TestFileNameReplace (line 260) | func TestFileNameReplace(t *testing.T) {
function TestExcelSheetNameReplace (line 284) | func TestExcelSheetNameReplace(t *testing.T) {
function TestAtoa (line 294) | func TestAtoa(t *testing.T) {
function TestAtoi (line 308) | func TestAtoi(t *testing.T) {
function TestAtoui (line 322) | func TestAtoui(t *testing.T) {
function TestBytes2String_String2Bytes (line 336) | func TestBytes2String_String2Bytes(t *testing.T) {
function TestKeyinsParse (line 351) | func TestKeyinsParse(t *testing.T) {
function TestRandomCreateBytes (line 381) | func TestRandomCreateBytes(t *testing.T) {
function TestXML2MapStr (line 403) | func TestXML2MapStr(t *testing.T) {
function TestRelPath (line 414) | func TestRelPath(t *testing.T) {
function TestWalkRelFiles (line 432) | func TestWalkRelFiles(t *testing.T) {
function TestWalkRelDir (line 461) | func TestWalkRelDir(t *testing.T) {
FILE: common/websocket/client.go
type DialError (line 17) | type DialError struct
method Error (line 22) | func (e *DialError) Error() string {
function NewConfig (line 27) | func NewConfig(server, origin string) (config *Config, err error) {
function NewClient (line 43) | func NewClient(config *Config, rwc io.ReadWriteCloser) (ws *Conn, err er...
function Dial (line 56) | func Dial(url_, protocol, origin string) (ws *Conn, err error) {
function DialConfig (line 68) | func DialConfig(config *Config) (ws *Conn, err error) {
FILE: common/websocket/client_test.go
function TestDialError_Error (line 11) | func TestDialError_Error(t *testing.T) {
function TestNewConfig (line 20) | func TestNewConfig(t *testing.T) {
function TestDialConfig (line 53) | func TestDialConfig(t *testing.T) {
function TestNewClient (line 105) | func TestNewClient(t *testing.T) {
function TestDial_WithProtocol (line 128) | func TestDial_WithProtocol(t *testing.T) {
function dialConn (line 143) | func dialConn(t *testing.T, httpURL string) io.ReadWriteCloser {
FILE: common/websocket/hybi.go
constant websocketGUID (line 26) | websocketGUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
constant closeStatusNormal (line 28) | closeStatusNormal = 1000
constant closeStatusGoingAway (line 29) | closeStatusGoingAway = 1001
constant closeStatusProtocolError (line 30) | closeStatusProtocolError = 1002
constant closeStatusUnsupportedData (line 31) | closeStatusUnsupportedData = 1003
constant closeStatusFrameTooLarge (line 32) | closeStatusFrameTooLarge = 1004
constant closeStatusNoStatusRcvd (line 33) | closeStatusNoStatusRcvd = 1005
constant closeStatusAbnormalClosure (line 34) | closeStatusAbnormalClosure = 1006
constant closeStatusBadMessageData (line 35) | closeStatusBadMessageData = 1007
constant closeStatusPolicyViolation (line 36) | closeStatusPolicyViolation = 1008
constant closeStatusTooBigData (line 37) | closeStatusTooBigData = 1009
constant closeStatusExtensionMismatch (line 38) | closeStatusExtensionMismatch = 1010
constant maxControlFramePayloadLength (line 40) | maxControlFramePayloadLength = 125
type hybiFrameHeader (line 63) | type hybiFrameHeader struct
type hybiFrameReader (line 74) | type hybiFrameReader struct
method Read (line 82) | func (frame *hybiFrameReader) Read(msg []byte) (n int, err error) {
method PayloadType (line 96) | func (frame *hybiFrameReader) PayloadType() byte { return frame.header...
method HeaderReader (line 98) | func (frame *hybiFrameReader) HeaderReader() io.Reader {
method TrailerReader (line 108) | func (frame *hybiFrameReader) TrailerReader() io.Reader { return nil }
method Len (line 110) | func (frame *hybiFrameReader) Len() (n int) { return frame.length }
type hybiFrameReaderFactory (line 113) | type hybiFrameReaderFactory struct
method NewFrameReader (line 120) | func (buf hybiFrameReaderFactory) NewFrameReader() (frame frameReader,...
type hybiFrameWriter (line 181) | type hybiFrameWriter struct
method Write (line 187) | func (frame *hybiFrameWriter) Write(msg []byte) (n int, err error) {
method Close (line 244) | func (frame *hybiFrameWriter) Close() error { return nil }
type hybiFrameWriterFactory (line 246) | type hybiFrameWriterFactory struct
method NewFrameWriter (line 251) | func (buf hybiFrameWriterFactory) NewFrameWriter(p
Condensed preview — 391 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (7,292K chars).
[
{
"path": ".gitattributes",
"chars": 24,
"preview": "*.* linguist-language=go"
},
{
"path": ".gitignore",
"chars": 471,
"preview": "*.o\r\n*.a\r\n*.so\r\n_obj\r\n_test\r\n*.[568vq]\r\n[568vq].out\r\n*.cgo1.go\r\n*.cgo2.c\r\n_cgo_defun.c\r\n_cgo_gotypes.go\r\n_cgo_export.*\r\n"
},
{
"path": "LICENSE",
"chars": 11338,
"preview": " Apache License\n Version 2.0, January 2004\n "
},
{
"path": "README.md",
"chars": 11161,
"preview": "<div align=\"center\">\r\n <img src=\"https://github.com/andeya/pholcus/raw/master/doc/icon.png\" width=\"120\" alt=\"Pholcus Lo"
},
{
"path": "app/aid/history/failure.go",
"chars": 2726,
"preview": "package history\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"os\"\n\t\"sync\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/andeya/"
},
{
"path": "app/aid/history/failure_test.go",
"chars": 3732,
"preview": "package history\n\nimport (\n\t\"net/http\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\n\t\"github.com/andeya/pholcus/app/downloader/requ"
},
{
"path": "app/aid/history/history.go",
"chars": 9287,
"preview": "// Package history provides persistence and inheritance of success and failure request records.\npackage history\n\nimport "
},
{
"path": "app/aid/history/history_test.go",
"chars": 10753,
"preview": "package history\n\nimport (\n\t\"database/sql\"\n\t\"encoding/json\"\n\t\"net/http\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\n\tsqlmock \"gith"
},
{
"path": "app/aid/history/success.go",
"chars": 2985,
"preview": "package history\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"os\"\n\t\"sync\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/andeya/ph"
},
{
"path": "app/aid/history/success_test.go",
"chars": 3580,
"preview": "package history\n\nimport (\n\t\"encoding/json\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\n\t\"github.com/andeya/pholcus/common/util\"\n\t"
},
{
"path": "app/aid/proxy/host.go",
"chars": 657,
"preview": "package proxy\n\nimport (\n\t\"sync\"\n\t\"time\"\n)\n\n// ProxyForHost manages proxy IPs for a host, sorted by response time.\ntype P"
},
{
"path": "app/aid/proxy/host_test.go",
"chars": 1299,
"preview": "package proxy\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestProxyForHost_Len(t *testing.T) {\n\ttests := []struct {\n\t\tproxys []"
},
{
"path": "app/aid/proxy/proxy.go",
"chars": 7024,
"preview": "// Package proxy provides proxy IP pool management and online filtering.\npackage proxy\n\nimport (\n\t\"io\"\n\t\"log\"\n\t\"net/http"
},
{
"path": "app/aid/proxy/proxy_test.go",
"chars": 5303,
"preview": "package proxy\n\nimport (\n\t\"net/http\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/andeya/gust/result"
},
{
"path": "app/app.go",
"chars": 19564,
"preview": "// app interface for graphical user interface.\n// Basic execution order: New() --> [SetLog(io.Writer) -->] Init() --> Sp"
},
{
"path": "app/app_test.go",
"chars": 5223,
"preview": "package app\n\nimport (\n\t\"bytes\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/andeya/pholcus/app/spider\"\n\t\"github.com/andeya/pholcus/r"
},
{
"path": "app/crawler/crawler.go",
"chars": 4948,
"preview": "// Package crawler provides the core crawler engine for request scheduling and page downloading.\npackage crawler\n\nimport"
},
{
"path": "app/crawler/crawler_test.go",
"chars": 4146,
"preview": "package crawler\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/andeya/pholcus/app/downloader/request\"\n\t\"g"
},
{
"path": "app/crawler/crawlerpool.go",
"chars": 2913,
"preview": "package crawler\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/andeya/gust/option\"\n\t\"github.com/andeya/pholcus/app/downloader\"\n"
},
{
"path": "app/crawler/crawlerpool_test.go",
"chars": 3380,
"preview": "package crawler\n\nimport (\n\t\"testing\"\n\n\t\"github.com/andeya/pholcus/app/downloader\"\n\t\"github.com/andeya/pholcus/app/downlo"
},
{
"path": "app/crawler/spiderqueue.go",
"chars": 3185,
"preview": "package crawler\n\nimport (\n\t\"github.com/andeya/gust/option\"\n\tspider \"github.com/andeya/pholcus/app/spider\"\n\t\"github.com/a"
},
{
"path": "app/crawler/spiderqueue_test.go",
"chars": 4573,
"preview": "package crawler\n\nimport (\n\t\"testing\"\n\n\tspider \"github.com/andeya/pholcus/app/spider\"\n)\n\nfunc makeSpider(name string, key"
},
{
"path": "app/distribute/integration_test.go",
"chars": 1102,
"preview": "package distribute\n\nimport (\n\t\"net\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/andeya/pholcus/app/distribute/te"
},
{
"path": "app/distribute/interface.go",
"chars": 323,
"preview": "package distribute\n\n// Distributor is the distributed interface.\ntype Distributor interface {\n\t// Send sends a task from"
},
{
"path": "app/distribute/master_api.go",
"chars": 1102,
"preview": "package distribute\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/andeya/pholcus/app/distribu"
},
{
"path": "app/distribute/master_api_test.go",
"chars": 1804,
"preview": "package distribute\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/andeya/pholcus/app/distribute/teleport\"\n)\n\ntype m"
},
{
"path": "app/distribute/slave_api.go",
"chars": 729,
"preview": "package distribute\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/andeya/pholcus/app/distribu"
},
{
"path": "app/distribute/slave_api_test.go",
"chars": 1135,
"preview": "package distribute\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/andeya/pholcus/app/distribute/teleport\"\n)\n\nfunc T"
},
{
"path": "app/distribute/task.go",
"chars": 1117,
"preview": "// Package distribute provides distributed task scheduling and master-slave node communication.\npackage distribute\n\n// T"
},
{
"path": "app/distribute/task_test.go",
"chars": 752,
"preview": "package distribute\n\nimport (\n\t\"testing\"\n)\n\nfunc TestTask_Fields(t *testing.T) {\n\ttests := []struct {\n\t\tname strin"
},
{
"path": "app/distribute/taskjar.go",
"chars": 926,
"preview": "package distribute\n\n// TaskJar is the task storage.\ntype TaskJar struct {\n\tTasks chan *Task\n}\n\n// NewTaskJar creates a t"
},
{
"path": "app/distribute/taskjar_test.go",
"chars": 1952,
"preview": "package distribute\n\nimport (\n\t\"sync\"\n\t\"testing\"\n)\n\nfunc TestNewTaskJar(t *testing.T) {\n\ttj := NewTaskJar()\n\tif tj == nil"
},
{
"path": "app/distribute/teleport/client.go",
"chars": 2621,
"preview": "package teleport\n\nimport (\n\t\"log\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/andeya/gust/result\"\n)\n\n// tpClient holds client-only stat"
},
{
"path": "app/distribute/teleport/conn.go",
"chars": 692,
"preview": "package teleport\n\nimport (\n\t\"net\"\n)\n\n// Connect wraps a network connection.\ntype Connect struct {\n\tnet.Conn\n\tUsable b"
},
{
"path": "app/distribute/teleport/conn_test.go",
"chars": 876,
"preview": "package teleport\n\nimport (\n\t\"net\"\n\t\"testing\"\n)\n\nfunc TestNewConnect(t *testing.T) {\n\tclient, server := net.Pipe()\n\tdefer"
},
{
"path": "app/distribute/teleport/debug.go",
"chars": 324,
"preview": "package teleport\n\nimport (\n\t\"log\"\n)\n\nvar Debug bool\n\nfunc debugPrintf(format string, v ...interface{}) {\n\tif !Debug {\n\t\t"
},
{
"path": "app/distribute/teleport/netdata.go",
"chars": 534,
"preview": "package teleport\n\nconst (\n\tSUCCESS = 0\n\tFAILURE = -1\n\tLLLEGAL = -2\n)\n\n// NetData is the data transfer structure.\ntype Ne"
},
{
"path": "app/distribute/teleport/netdata_test.go",
"chars": 854,
"preview": "package teleport\n\nimport (\n\t\"testing\"\n)\n\nfunc TestNewNetData(t *testing.T) {\n\ttests := []struct {\n\t\tfrom, to, op, flag s"
},
{
"path": "app/distribute/teleport/protocol.go",
"chars": 1897,
"preview": "package teleport\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n)\n\nconst (\n\tDataLengthOfLenth = 4\n)\n\n// Protocol handles packet f"
},
{
"path": "app/distribute/teleport/protocol_test.go",
"chars": 3536,
"preview": "package teleport\n\nimport (\n\t\"bytes\"\n\t\"testing\"\n)\n\nfunc TestNewProtocol(t *testing.T) {\n\ttests := []struct {\n\t\theader str"
},
{
"path": "app/distribute/teleport/return_func.go",
"chars": 851,
"preview": "package teleport\n\n// ReturnData builds an API response. If OpAndToAndFrom[0] is empty, use peer operation; if [1] is emp"
},
{
"path": "app/distribute/teleport/return_func_test.go",
"chars": 1817,
"preview": "package teleport\n\nimport (\n\t\"testing\"\n)\n\nfunc TestReturnData(t *testing.T) {\n\ttests := []struct {\n\t\tbody "
},
{
"path": "app/distribute/teleport/server.go",
"chars": 3184,
"preview": "package teleport\n\nimport (\n\t\"encoding/json\"\n\t\"log\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/andeya/gust/result\"\n)\n\n// tpServer holds"
},
{
"path": "app/distribute/teleport/teleport.go",
"chars": 9571,
"preview": "// Package teleport provides a high-concurrency API framework for distributed systems.\n// It uses socket duplex communic"
},
{
"path": "app/distribute/teleport/teleport_test.go",
"chars": 4726,
"preview": "package teleport\n\nimport (\n\t\"encoding/json\"\n\t\"net\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc freePort(t *testing.T) "
},
{
"path": "app/distribute/teleport/util.go",
"chars": 1020,
"preview": "package teleport\n\nimport (\n\t\"crypto/md5\"\n\t\"encoding/hex\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"hash/crc32\"\n\t\"hash/fnv\"\n\t\"strconv\"\n)\n"
},
{
"path": "app/distribute/teleport/util_test.go",
"chars": 1242,
"preview": "package teleport\n\nimport (\n\t\"testing\"\n)\n\nfunc TestMakeHash(t *testing.T) {\n\ttests := []struct {\n\t\ts string\n\t\twant str"
},
{
"path": "app/downloader/downloader.go",
"chars": 478,
"preview": "// Package downloader defines the page downloader interface.\npackage downloader\n\nimport (\n\t\"github.com/andeya/pholcus/ap"
},
{
"path": "app/downloader/downloader_surfer.go",
"chars": 1627,
"preview": "package downloader\n\nimport (\n\t\"errors\"\n\t\"net/http\"\n\t\"net/http/cookiejar\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/"
},
{
"path": "app/downloader/downloader_test.go",
"chars": 2676,
"preview": "package downloader\n\nimport (\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/andeya/pholcus/app/downloader/req"
},
{
"path": "app/downloader/request/request.go",
"chars": 9490,
"preview": "// Package request provides encapsulation and deduplication of crawl requests.\npackage request\n\nimport (\n\t\"crypto/md5\"\n\t"
},
{
"path": "app/downloader/request/request_test.go",
"chars": 8254,
"preview": "package request\n\nimport (\n\t\"encoding/json\"\n\t\"net/http\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestReqTemp(t *testing.T) {\n\tvar a = &"
},
{
"path": "app/downloader/request/temp.go",
"chars": 962,
"preview": "package request\n\nimport (\n\t\"encoding/json\"\n\t\"reflect\"\n\n\t\"github.com/andeya/pholcus/common/util\"\n\t\"github.com/andeya/phol"
},
{
"path": "app/downloader/surfer/agent/agent.go",
"chars": 8417,
"preview": "// Package agent generates user agents strings for well known browsers\n// and for custom browsers.\n//\n// When submitting"
},
{
"path": "app/downloader/surfer/agent/agent_bsd.go",
"chars": 499,
"preview": "//go:build darwin || dragonfly || freebsd || netbsd || openbsd\n\n// Package agent provides system User-Agent information."
},
{
"path": "app/downloader/surfer/agent/agent_linux.go",
"chars": 731,
"preview": "//go:build linux && !arm\n\npackage agent\n\nimport (\n\t\"runtime\"\n\t\"syscall\"\n)\n\n// osName returns the name of the OS.\nfunc os"
},
{
"path": "app/downloader/surfer/agent/agent_linux_arm.go",
"chars": 732,
"preview": "//go:build linux && arm\n\npackage agent\n\nimport (\n\t\"runtime\"\n\t\"syscall\"\n)\n\n// osName returns the name of the OS.\nfunc osN"
},
{
"path": "app/downloader/surfer/agent/agent_test.go",
"chars": 3746,
"preview": "package agent\n\nimport (\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestUserAgentsPopulated(t *testing.T) {\n\tif len(UserAgents[\"all\"])"
},
{
"path": "app/downloader/surfer/agent/agent_windows.go",
"chars": 436,
"preview": "//go:build windows\n\n// Package agent provides system User-Agent information.\npackage agent\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n\t"
},
{
"path": "app/downloader/surfer/chrome.go",
"chars": 6312,
"preview": "//go:build !cover\n\npackage surfer\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"net/http\"\n\t\"net/http/cookiejar\"\n\t\"strings\"\n"
},
{
"path": "app/downloader/surfer/chrome_stub.go",
"chars": 520,
"preview": "//go:build cover\n\npackage surfer\n\nimport (\n\t\"errors\"\n\t\"net/http\"\n\t\"net/http/cookiejar\"\n\n\t\"github.com/andeya/gust/result\""
},
{
"path": "app/downloader/surfer/chrome_test.go",
"chars": 2448,
"preview": "package surfer\n\nimport (\n\t\"io\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestChromeDownloaderBaiduSearch(t *testing.T) {\n\tif testing"
},
{
"path": "app/downloader/surfer/example/example.go",
"chars": 2319,
"preview": "package main\n\nimport (\n\t\"io\"\n\t\"log\"\n\t\"time\"\n\n\t\"github.com/andeya/pholcus/app/downloader/surfer\"\n)\n\nfunc main() {\n\tvar va"
},
{
"path": "app/downloader/surfer/param.go",
"chars": 3773,
"preview": "// Copyright 2015 andeya Author. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "app/downloader/surfer/param_test.go",
"chars": 4013,
"preview": "package surfer\n\nimport (\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestNewParam(t *testing.T) {\n\ttests"
},
{
"path": "app/downloader/surfer/phantom.go",
"chars": 9979,
"preview": "//go:build !cover\n\n// Copyright 2015 andeya Author. All Rights Reserved.\n//\n// Licensed under the Apache License, Versio"
},
{
"path": "app/downloader/surfer/phantom_stub.go",
"chars": 601,
"preview": "//go:build cover\n\npackage surfer\n\nimport (\n\t\"errors\"\n\t\"net/http\"\n\t\"net/http/cookiejar\"\n\n\t\"github.com/andeya/gust/result\""
},
{
"path": "app/downloader/surfer/request.go",
"chars": 5252,
"preview": "// Copyright 2015 andeya Author. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "app/downloader/surfer/request_test.go",
"chars": 4840,
"preview": "package surfer\n\nimport (\n\t\"net/http\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestDefaultRequestPrepare(t *testing.T) {\n\ttests := []st"
},
{
"path": "app/downloader/surfer/surf.go",
"chars": 4722,
"preview": "// Copyright 2015 andeya Author. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "app/downloader/surfer/surf_stub_test.go",
"chars": 588,
"preview": "//go:build cover\n\npackage surfer\n\nimport \"testing\"\n\nfunc TestDownloadPhantomJsIDStub(t *testing.T) {\n\treq := &mockReques"
},
{
"path": "app/downloader/surfer/surf_test.go",
"chars": 9894,
"preview": "package surfer\n\nimport (\n\t\"bytes\"\n\t\"compress/flate\"\n\t\"compress/gzip\"\n\t\"compress/zlib\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/http/cooki"
},
{
"path": "app/downloader/surfer/surfer.go",
"chars": 2555,
"preview": "// Copyright 2015 andeya Author. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "app/downloader/surfer/util.go",
"chars": 3314,
"preview": "// Copyright 2015 andeya Author. All Rights Reserved.\n//\n// Licensed under the Apache License, Version 2.0 (the \"License"
},
{
"path": "app/downloader/surfer/util_test.go",
"chars": 2833,
"preview": "package surfer\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n)\n"
},
{
"path": "app/pipeline/collector/collector.go",
"chars": 4347,
"preview": "// Package collector implements result collection and output.\npackage collector\n\nimport (\n\t\"runtime/debug\"\n\t\"sync\"\n\t\"syn"
},
{
"path": "app/pipeline/collector/collector_test.go",
"chars": 9268,
"preview": "package collector\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/and"
},
{
"path": "app/pipeline/collector/data/data.go",
"chars": 1738,
"preview": "// Package data provides storage structure definitions for data and file cells.\npackage data\n\nimport (\n\t\"sync\"\n)\n\nconst "
},
{
"path": "app/pipeline/collector/data/data_test.go",
"chars": 2116,
"preview": "package data\n\nimport (\n\t\"testing\"\n)\n\nfunc TestGetDataCell(t *testing.T) {\n\td := map[string]interface{}{\"key\": \"value\"}\n\t"
},
{
"path": "app/pipeline/collector/output_beanstalkd.go",
"chars": 1486,
"preview": "//go:build !coverage\n\npackage collector\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/url\"\n\t\"time\"\n\n\t\"github.com/andeya/gust/r"
},
{
"path": "app/pipeline/collector/output_beanstalkd_stub.go",
"chars": 191,
"preview": "//go:build coverage\n\npackage collector\n\nimport (\n\t\"github.com/andeya/gust/result\"\n)\n\nfunc init() {\n\tDataOutput[\"beanstal"
},
{
"path": "app/pipeline/collector/output_csv.go",
"chars": 2048,
"preview": "package collector\n\nimport (\n\t\"encoding/csv\"\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/andeya/pholcus/c"
},
{
"path": "app/pipeline/collector/output_data.go",
"chars": 2130,
"preview": "package collector\n\nimport (\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/andeya/pholcus/logs\"\n)\n\n// Refresher is an opt"
},
{
"path": "app/pipeline/collector/output_data_test.go",
"chars": 747,
"preview": "package collector\n\nimport (\n\t\"testing\"\n\n\t\"github.com/andeya/gust/result\"\n)\n\nfunc TestRegister(t *testing.T) {\n\tRegister("
},
{
"path": "app/pipeline/collector/output_excel.go",
"chars": 2124,
"preview": "package collector\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/andeya/pholcus/common/util\"\n\t\"gi"
},
{
"path": "app/pipeline/collector/output_file.go",
"chars": 2168,
"preview": "package collector\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sync/atomic\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"git"
},
{
"path": "app/pipeline/collector/output_kafka.go",
"chars": 2165,
"preview": "//go:build !coverage\n\npackage collector\n\nimport (\n\t\"regexp\"\n\t\"sync\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/andey"
},
{
"path": "app/pipeline/collector/output_kafka_stub.go",
"chars": 186,
"preview": "//go:build coverage\n\npackage collector\n\nimport (\n\t\"github.com/andeya/gust/result\"\n)\n\nfunc init() {\n\tDataOutput[\"kafka\"] "
},
{
"path": "app/pipeline/collector/output_mgo.go",
"chars": 1981,
"preview": "//go:build !coverage\n\npackage collector\n\nimport (\n\tmgov2 \"gopkg.in/mgo.v2\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.co"
},
{
"path": "app/pipeline/collector/output_mgo_stub.go",
"chars": 184,
"preview": "//go:build coverage\n\npackage collector\n\nimport (\n\t\"github.com/andeya/gust/result\"\n)\n\nfunc init() {\n\tDataOutput[\"mgo\"] = "
},
{
"path": "app/pipeline/collector/output_mysql.go",
"chars": 2269,
"preview": "//go:build !coverage\n\npackage collector\n\nimport (\n\t\"sync\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/andeya/pholcus/"
},
{
"path": "app/pipeline/collector/output_mysql_stub.go",
"chars": 186,
"preview": "//go:build coverage\n\npackage collector\n\nimport (\n\t\"github.com/andeya/gust/result\"\n)\n\nfunc init() {\n\tDataOutput[\"mysql\"] "
},
{
"path": "app/pipeline/collector/output_util.go",
"chars": 1104,
"preview": "package collector\n\nimport (\n\t\"github.com/andeya/pholcus/logs\"\n)\n\n// namespace returns the main namespace (relative to DB"
},
{
"path": "app/pipeline/collector/output_util_test.go",
"chars": 2653,
"preview": "package collector\n\nimport (\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/andeya/pholcus/app/spider\"\n)\n\nfunc TestJoinNamespaces(t "
},
{
"path": "app/pipeline/output.go",
"chars": 1512,
"preview": "package pipeline\n\nimport (\n\t\"sort\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/andeya/pholcus/app/pipeline/collector\""
},
{
"path": "app/pipeline/pipeline.go",
"chars": 675,
"preview": "// Package pipeline provides the data collection and output pipeline.\npackage pipeline\n\nimport (\n\t\"github.com/andeya/gus"
},
{
"path": "app/pipeline/pipeline_test.go",
"chars": 2787,
"preview": "package pipeline\n\nimport (\n\t\"testing\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/andeya/pholcus/app/pipeline/collect"
},
{
"path": "app/scheduler/matrix.go",
"chars": 6348,
"preview": "package scheduler\n\nimport (\n\t\"runtime/debug\"\n\t\"sort\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/andeya/pholcus/app/aid"
},
{
"path": "app/scheduler/scheduler.go",
"chars": 2766,
"preview": "// Package scheduler provides crawl task scheduling and resource allocation.\npackage scheduler\n\nimport (\n\t\"runtime/debug"
},
{
"path": "app/scheduler/scheduler_test.go",
"chars": 6492,
"preview": "package scheduler\n\nimport (\n\t\"testing\"\n\n\t\"github.com/andeya/pholcus/app/downloader/request\"\n\t\"github.com/andeya/pholcus/"
},
{
"path": "app/spider/common/common.go",
"chars": 8455,
"preview": "// Package common provides HTML cleaning, form parsing, and other utility functions for spider rules.\npackage common\n\nim"
},
{
"path": "app/spider/common/common_test.go",
"chars": 8998,
"preview": "package common\n\nimport (\n\t\"testing\"\n)\n\nfunc TestCleanHtml(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tstr st"
},
{
"path": "app/spider/common/form.go",
"chars": 4250,
"preview": "package common\n\nimport (\n\t\"net/url\"\n\t\"strings\"\n\n\t\"github.com/andeya/pholcus/common/goquery\"\n\n\t\"github.com/andeya/pholcus"
},
{
"path": "app/spider/common/form_test.go",
"chars": 3264,
"preview": "package common\n\nimport (\n\t\"strings\"\n\t\"testing\"\n\n\tspider \"github.com/andeya/pholcus/app/spider\"\n\t\"github.com/andeya/pholc"
},
{
"path": "app/spider/context.go",
"chars": 20644,
"preview": "package spider\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\n\t\"mime\"\n\t\"net/http\"\n\t\"path\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n\t\"unsafe\"\n\n\t\"golang.org/"
},
{
"path": "app/spider/parsejs.go",
"chars": 5515,
"preview": "package spider\n\nimport (\n\t\"encoding/xml\"\n\t\"log\"\n\t\"os\"\n\t\"path\"\n\t\"path/filepath\"\n\t\"regexp\"\n\t\"runtime/debug\"\n\t\"strings\"\n\t\"s"
},
{
"path": "app/spider/species.go",
"chars": 1651,
"preview": "// Package spider provides spider rule definition, species registration, and parsing.\npackage spider\n\nimport (\n\t\"fmt\"\n\n\t"
},
{
"path": "app/spider/species_test.go",
"chars": 6995,
"preview": "package spider\n\nimport (\n\t\"testing\"\n)\n\nfunc TestSpiderSpecies_Add(t *testing.T) {\n\tss := &SpiderSpecies{\n\t\tlist: []*Spid"
},
{
"path": "app/spider/spider.go",
"chars": 10170,
"preview": "package spider\n\nimport (\n\t\"errors\"\n\t\"math\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/andeya/pholcus/app/downloader/request\"\n\t\"github"
},
{
"path": "app/spider/timer.go",
"chars": 3450,
"preview": "package spider\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/andeya/pholcus/logs\"\n)\n\n// Timer manages a collection of named cl"
},
{
"path": "app/spider/timer_test.go",
"chars": 515,
"preview": "package spider\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestTimer1(t *testing.T) {\n\tt.Log(time.Now())\n\tctx := GetContext(new"
},
{
"path": "cmd/cmd_test.go",
"chars": 213,
"preview": "package cmd\n\nimport (\n\t\"flag\"\n\t\"testing\"\n)\n\nfunc TestFlag(t *testing.T) {\n\tflag.CommandLine = flag.NewFlagSet(\"cmd_test\""
},
{
"path": "cmd/pholcus-cmd.go",
"chars": 4367,
"preview": "// Package cmd implements the command-line interface for Pholcus.\npackage cmd\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"os\"\n\t\"os/signal"
},
{
"path": "common/beanstalkd/beanstalkd.go",
"chars": 1751,
"preview": "// Package beanstalkd provides a client wrapper for Beanstalkd job queue.\npackage beanstalkd\n\nimport (\n\t\"net/url\"\n\n\t\"git"
},
{
"path": "common/beanstalkd/beanstalkd_test.go",
"chars": 2497,
"preview": "package beanstalkd\n\nimport (\n\t\"net\"\n\t\"net/url\"\n\t\"testing\"\n\n\t\"github.com/andeya/pholcus/config\"\n\t\"github.com/kr/beanstalk"
},
{
"path": "common/bytes/bytes.go",
"chars": 1834,
"preview": "// Package bytes provides byte unit conversion and parsing.\npackage bytes\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\t\"strconv\"\n)\n\ntype "
},
{
"path": "common/bytes/bytes_test.go",
"chars": 1707,
"preview": "package bytes\n\nimport (\n\t\"testing\"\n)\n\nfunc TestFormat(t *testing.T) {\n\ttests := []struct {\n\t\tinput uint64\n\t\twant string"
},
{
"path": "common/closer/closer.go",
"chars": 608,
"preview": "// Package closer provides utilities for closing resources with error logging.\npackage closer\n\nimport (\n\t\"errors\"\n\t\"io\"\n"
},
{
"path": "common/closer/closer_test.go",
"chars": 596,
"preview": "package closer\n\nimport (\n\t\"errors\"\n\t\"testing\"\n)\n\ntype mockCloser struct {\n\terr error\n}\n\nfunc (m *mockCloser) Close() err"
},
{
"path": "common/gc/gc.go",
"chars": 660,
"preview": "// Package gc provides manual garbage collection to release heap memory.\npackage gc\n\nimport (\n\t\"runtime\"\n\t\"runtime/debug"
},
{
"path": "common/gc/gc_test.go",
"chars": 369,
"preview": "package gc\n\nimport (\n\t\"testing\"\n)\n\nfunc TestGCSizeConstant(t *testing.T) {\n\texpected := 50 << 20\n\tif GC_SIZE != expected"
},
{
"path": "common/goquery/.gitattributes",
"chars": 29,
"preview": "testdata/* linguist-vendored\n"
},
{
"path": "common/goquery/.gitignore",
"chars": 151,
"preview": "# editor temporary files\n*.sublime-*\n.DS_Store\n*.swp\n#*.*#\ntags\n\n# direnv config\n.env*\n\n# test binaries\n*.test\n\n# covera"
},
{
"path": "common/goquery/.travis.yml",
"chars": 98,
"preview": "language: go\n\ngo:\n - 1.1\n - 1.2\n - 1.3\n - 1.4\n - 1.5\n - 1.6\n - 1.7\n - tip\n"
},
{
"path": "common/goquery/LICENSE",
"chars": 1491,
"preview": "Copyright (c) 2012-2016, Martin Angers & Contributors\nAll rights reserved.\n\nRedistribution and use in source and binary "
},
{
"path": "common/goquery/README.md",
"chars": 9296,
"preview": "# goquery - a little like that j-thing, only in Go [![build status](https://secure.travis-ci.org/PuerkitoBio/goquery.png"
},
{
"path": "common/goquery/array.go",
"chars": 3016,
"preview": "package goquery\n\nimport (\n\t\"golang.org/x/net/html\"\n)\n\n// First reduces the set of matched elements to the first in the s"
},
{
"path": "common/goquery/array_test.go",
"chars": 4831,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n)\n\nfunc TestFirst(t *testing.T) {\n\tsel := Doc().Find(\".pvk-content\").First()\n\tasser"
},
{
"path": "common/goquery/bench/v0.1.0",
"chars": 18552,
"preview": "PASS\nBenchmarkFirst\t20000000\t 92.9 ns/op\nBenchmarkLast\t20000000\t 91.6 ns/op\nBenchmarkEq\t20000000\t 9"
},
{
"path": "common/goquery/bench/v0.1.1",
"chars": 18647,
"preview": "PASS\nBenchmarkFirst\t20000000\t 96.2 ns/op\nBenchmarkLast\t20000000\t 95.8 ns/op\nBenchmarkEq\t20000000\t 9"
},
{
"path": "common/goquery/bench/v0.2.0",
"chars": 19592,
"preview": "PASS\nBenchmarkFirst\t20000000\t 94.3 ns/op\nBenchmarkLast\t20000000\t 94.7 ns/op\nBenchmarkEq\t20000000\t 9"
},
{
"path": "common/goquery/bench/v0.2.1-go1.1rc1",
"chars": 20136,
"preview": "PASS\nBenchmarkFirst\t20000000\t 96.3 ns/op\nBenchmarkLast\t20000000\t 95.7 ns/op\nBenchmarkEq\t20000000\t 9"
},
{
"path": "common/goquery/bench/v0.3.0",
"chars": 20367,
"preview": "PASS\nBenchmarkFirst\t20000000\t 95.5 ns/op\nBenchmarkLast\t20000000\t 94.9 ns/op\nBenchmarkEq\t20000000\t 9"
},
{
"path": "common/goquery/bench/v0.3.2-go1.2",
"chars": 20440,
"preview": "PASS\nBenchmarkFirst\t20000000\t 88.4 ns/op\nBenchmarkLast\t20000000\t 88.2 ns/op\nBenchmarkEq\t20000000\t 8"
},
{
"path": "common/goquery/bench/v0.3.2-go1.2-take2",
"chars": 20400,
"preview": "PASS\nBenchmarkFirst\t20000000\t 88.3 ns/op\nBenchmarkLast\t20000000\t 88.9 ns/op\nBenchmarkEq\t20000000\t 8"
},
{
"path": "common/goquery/bench/v0.3.2-go1.2rc1",
"chars": 20400,
"preview": "PASS\nBenchmarkFirst\t20000000\t 91.0 ns/op\nBenchmarkLast\t20000000\t 90.5 ns/op\nBenchmarkEq\t20000000\t 9"
},
{
"path": "common/goquery/bench/v1.0.0-go1.7",
"chars": 8639,
"preview": "BenchmarkFirst-4 \t30000000\t 50.7 ns/op\t 48 B/op\t 1 allocs/op\nBenchmarkLast-4"
},
{
"path": "common/goquery/bench/v1.0.1a-go1.7",
"chars": 8639,
"preview": "BenchmarkFirst-4 \t30000000\t 50.9 ns/op\t 48 B/op\t 1 allocs/op\nBenchmarkLast-4"
},
{
"path": "common/goquery/bench/v1.0.1b-go1.7",
"chars": 8639,
"preview": "BenchmarkFirst-4 \t30000000\t 51.8 ns/op\t 48 B/op\t 1 allocs/op\nBenchmarkLast-4"
},
{
"path": "common/goquery/bench/v1.0.1c-go1.7",
"chars": 8742,
"preview": "BenchmarkFirst-4 \t30000000\t 51.7 ns/op\t 48 B/op\t 1 allocs/op\nBenchmarkLast-4"
},
{
"path": "common/goquery/bench_array_test.go",
"chars": 1922,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n)\n\nfunc BenchmarkFirst(b *testing.B) {\n\tb.StopTimer()\n\tsel := DocB().Find(\"dd\")\n\tb."
},
{
"path": "common/goquery/bench_example_test.go",
"chars": 922,
"preview": "package goquery\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nfunc BenchmarkMetalReviewExample(b *testing.B) {\n\tvar"
},
{
"path": "common/goquery/bench_expand_test.go",
"chars": 1677,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n)\n\nfunc BenchmarkAdd(b *testing.B) {\n\tvar n int\n\n\tb.StopTimer()\n\tsel := DocB().Find"
},
{
"path": "common/goquery/bench_filter_test.go",
"chars": 3919,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n)\n\nfunc BenchmarkFilter(b *testing.B) {\n\tvar n int\n\n\tb.StopTimer()\n\tsel := DocW().F"
},
{
"path": "common/goquery/bench_iteration_test.go",
"chars": 965,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n)\n\nfunc BenchmarkEach(b *testing.B) {\n\tvar tmp, n int\n\n\tb.StopTimer()\n\tsel := DocW("
},
{
"path": "common/goquery/bench_property_test.go",
"chars": 773,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n)\n\nfunc BenchmarkAttr(b *testing.B) {\n\tvar s string\n\n\tb.StopTimer()\n\tsel := DocW()."
},
{
"path": "common/goquery/bench_query_test.go",
"chars": 1718,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n)\n\nfunc BenchmarkIs(b *testing.B) {\n\tvar y bool\n\n\tb.StopTimer()\n\tsel := DocW().Find"
},
{
"path": "common/goquery/bench_traversal_test.go",
"chars": 14049,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n)\n\nfunc BenchmarkFind(b *testing.B) {\n\tvar n int\n\n\tfor i := 0; i < b.N; i++ {\n\t\tif "
},
{
"path": "common/goquery/doc/tips.md",
"chars": 2063,
"preview": "# Tips and tricks\n\n## Handle Non-UTF8 html Pages\n\nThe `go.net/html` package used by `goquery` requires that the html doc"
},
{
"path": "common/goquery/doc.go",
"chars": 4609,
"preview": "// Copyright (c) 2012-2016, Martin Angers & Contributors\n// All rights reserved.\n//\n// Redistribution and use in source "
},
{
"path": "common/goquery/example_test.go",
"chars": 889,
"preview": "package goquery_test\n\nimport (\n\t\"fmt\"\n\t\"log\"\n\n\t\"github.com/andeya/pholcus/common/goquery\"\n)\n\n// This example scrapes the"
},
{
"path": "common/goquery/expand.go",
"chars": 1673,
"preview": "package goquery\n\nimport \"golang.org/x/net/html\"\n\n// Add adds the selector string's matching nodes to those in the curren"
},
{
"path": "common/goquery/expand_test.go",
"chars": 2222,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n)\n\nfunc TestAdd(t *testing.T) {\n\tsel := Doc().Find(\"div.row-fluid\").Add(\"a\")\n\tasser"
},
{
"path": "common/goquery/filter.go",
"chars": 5922,
"preview": "package goquery\n\nimport \"golang.org/x/net/html\"\n\n// Filter reduces the set of matched elements to those that match the s"
},
{
"path": "common/goquery/filter_test.go",
"chars": 5039,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n)\n\nfunc TestFilter(t *testing.T) {\n\tsel := Doc().Find(\".span12\").Filter(\".alert\")\n\t"
},
{
"path": "common/goquery/iteration.go",
"chars": 1444,
"preview": "package goquery\n\n// Each iterates over a Selection object, executing a function for each\n// matched element. It returns "
},
{
"path": "common/goquery/iteration_test.go",
"chars": 1924,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n\n\t\"golang.org/x/net/html\"\n)\n\nfunc TestEach(t *testing.T) {\n\tvar cnt int\n\n\tsel := Do"
},
{
"path": "common/goquery/manipulation.go",
"chars": 17737,
"preview": "// Package goquery provides jQuery-like HTML document parsing and manipulation.\npackage goquery\n\nimport (\n\t\"strings\"\n\n\t\""
},
{
"path": "common/goquery/manipulation_test.go",
"chars": 11474,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n)\n\nconst (\n\twrapHtml = \"<div id=\\\"ins\\\">test string<div><p><em><b></b></em></p></di"
},
{
"path": "common/goquery/misc/git/pre-commit",
"chars": 1104,
"preview": "#!/bin/sh\n\necho \">>> golint\"\nfor dir in $(go list ./... | grep -v /vendor/)\ndo\n golint \"${dir}\"\ndone\necho \"<<< golint"
},
{
"path": "common/goquery/property.go",
"chars": 6661,
"preview": "package goquery\n\nimport (\n\t\"bytes\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/andeya/gust/option\"\n\t\"golang.org/x/net/html\"\n)\n\nva"
},
{
"path": "common/goquery/property_test.go",
"chars": 6166,
"preview": "package goquery\n\nimport (\n\t\"regexp\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestAttrExists(t *testing.T) {\n\tif val := Doc().Find(\""
},
{
"path": "common/goquery/query.go",
"chars": 1772,
"preview": "package goquery\n\nimport \"golang.org/x/net/html\"\n\n// Is checks the current matched set of elements against a selector and"
},
{
"path": "common/goquery/query_test.go",
"chars": 2315,
"preview": "package goquery\n\nimport (\n\t\"testing\"\n)\n\nfunc TestIs(t *testing.T) {\n\tsel := Doc().Find(\".footer p:nth-child(1)\")\n\tif !se"
},
{
"path": "common/goquery/testdata/gotesting.html",
"chars": 21393,
"preview": "<!DOCTYPE html>\n<html>\n<head>\n<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n\n <title>testing - Th"
},
{
"path": "common/goquery/testdata/gowiki.html",
"chars": 127818,
"preview": "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\""
},
{
"path": "common/goquery/testdata/metalreview.html",
"chars": 40234,
"preview": "\n\n<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dt"
},
{
"path": "common/goquery/testdata/page.html",
"chars": 5591,
"preview": "<!DOCTYPE html>\n<html lang=\"en\" ng-app=\"app\">\n <head>\n <meta charset=\"utf-8\">\n <meta http-equiv=\"X-UA-C"
},
{
"path": "common/goquery/testdata/page2.html",
"chars": 747,
"preview": "<!DOCTYPE html>\n<html>\n <head>\n <title>Tests for siblings</title>\n </head>\n <BODY>\n <div id=\"main\">\n <div "
},
{
"path": "common/goquery/testdata/page3.html",
"chars": 756,
"preview": "<!DOCTYPE html>\n<html>\n <head>\n <title>Tests for siblings</title>\n </head>\n <BODY>\n <div id=\"main\">\n <div "
},
{
"path": "common/goquery/traversal.go",
"chars": 28271,
"preview": "package goquery\n\nimport \"golang.org/x/net/html\"\n\ntype siblingType int\n\n// Sibling type, used internally when iterating o"
},
{
"path": "common/goquery/traversal_test.go",
"chars": 20893,
"preview": "package goquery\n\nimport (\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestFind(t *testing.T) {\n\tsel := Doc().Find(\"div.row-fluid\")\n\tas"
},
{
"path": "common/goquery/type.go",
"chars": 4306,
"preview": "package goquery\n\nimport (\n\t\"errors\"\n\t\"io\"\n\t\"log\"\n\t\"net/http\"\n\t\"net/url\"\n\n\t\"github.com/andeya/gust/result\"\n\t\"github.com/a"
},
{
"path": "common/goquery/type_test.go",
"chars": 3777,
"preview": "package goquery\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"golang.org/x/net/html\"\n)\n\n// Test helper functi"
},
{
"path": "common/goquery/utilities.go",
"chars": 4706,
"preview": "package goquery\n\nimport (\n\t\"bytes\"\n\n\t\"golang.org/x/net/html\"\n)\n\n// used to determine if a set (map[*html.Node]bool) shou"
},
{
"path": "common/goquery/utilities_test.go",
"chars": 2742,
"preview": "package goquery\n\nimport (\n\t\"reflect\"\n\t\"sort\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"golang.org/x/net/html\"\n)\n\nvar allNodes = `<!doctyp"
},
{
"path": "common/kafka/kafka.go",
"chars": 1667,
"preview": "// Package kafka provides Kafka message queue sending wrapper.\npackage kafka\n\nimport (\n\t\"errors\"\n\t\"strings\"\n\t\"sync\"\n\n\t\"g"
},
{
"path": "common/kafka/kafka_test.go",
"chars": 2594,
"preview": "package kafka\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/Shopify/sarama\"\n)\n\ntype mockSyncProducer struct {\n\tse"
},
{
"path": "common/mahonia/8bit.go",
"chars": 145756,
"preview": "package mahonia\n\nimport (\n\t\"fmt\"\n\t\"sync\"\n)\n\n// Converters for simple 8-bit character sets.\n\ntype eightBitInfo struct {\n\t"
},
{
"path": "common/mahonia/ASCII.go",
"chars": 1544,
"preview": "package mahonia\n\n// Converters for ASCII and ISO-8859-1\n\nfunc init() {\n\tfor i := 0; i < len(asciiCharsets); i++ {\n\t\tRegi"
},
{
"path": "common/mahonia/README.md",
"chars": 545,
"preview": "# mahonia\n\ncharacter-set conversion library implemented in Go.\n\nMahonia is a character-set conversion library implemente"
},
{
"path": "common/mahonia/big5-data.go",
"chars": 249968,
"preview": "package mahonia\n\nvar big5ToUnicode = [65536]uint16{\n\t0xA140: 0x3000, // IDEOGRAPHIC SPACE\n\t0xA141: 0xFF0C, // FULLWIDTH "
},
{
"path": "common/mahonia/big5.go",
"chars": 1301,
"preview": "package mahonia\n\n// Converters for Big 5 encoding.\n\nimport (\n\t\"sync\"\n)\n\nfunc init() {\n\tRegisterCharset(&Charset{\n\t\tName:"
},
{
"path": "common/mahonia/charset.go",
"chars": 3521,
"preview": "// This package is a character-set conversion library for Go.\n//\n// (DEPRECATED: use code.google.com/p/go.text/encoding,"
},
{
"path": "common/mahonia/convert_string.go",
"chars": 2477,
"preview": "package mahonia\n\nimport (\n\t\"unicode/utf8\"\n)\n\n// ConvertString converts a string from UTF-8 to e's encoding.\nfunc (e Enc"
},
{
"path": "common/mahonia/cp51932.go",
"chars": 1255,
"preview": "package mahonia\n\nimport (\n\t\"unicode/utf8\"\n)\n\n// Converters for Microsoft's version of the EUC-JP encoding\n\nfunc init() {"
},
{
"path": "common/mahonia/entity.go",
"chars": 3622,
"preview": "package mahonia\n\n// decoding HTML entities\n\nimport (\n\t\"sort\"\n)\n\n// EntityDecoder returns a Decoder that decodes HTML cha"
},
{
"path": "common/mahonia/entity_data.go",
"chars": 71219,
"preview": "package mahonia\n\n// Taken from /src/pkg/html/entity.go in the Go source code.\n\n// Copyright 2010 The Go Authors. All rig"
},
{
"path": "common/mahonia/euc-jp.go",
"chars": 1740,
"preview": "package mahonia\n\nimport (\n\t\"unicode/utf8\"\n)\n\n// Converters for the EUC-JP encoding\n\nfunc init() {\n\tRegisterCharset(&Char"
},
{
"path": "common/mahonia/euc-kr-data.go",
"chars": 290302,
"preview": "package mahonia\n\nimport (\n\t\"sync\"\n)\n\nvar eucKrOnce sync.Once\n\nvar unicodeToEucKr []uint16\n\nfunc reverseEucKrTable() {\n\tu"
},
{
"path": "common/mahonia/euc-kr.go",
"chars": 1291,
"preview": "package mahonia\n\n// Converters for the EUC-KR encoding.\n\nimport (\n\t\"unicode/utf8\"\n)\n\nfunc init() {\n\tRegisterCharset(&Cha"
},
{
"path": "common/mahonia/fallback.go",
"chars": 691,
"preview": "package mahonia\n\n// FallbackDecoder combines a series of Decoders into one.\n// If the first Decoder returns a status of "
},
{
"path": "common/mahonia/gb18030-data.go",
"chars": 158117,
"preview": "package mahonia\n\n// Data tables for 4-byte characters in GB18030 encoding.\n// Based on http://source.icu-project.org/rep"
},
{
"path": "common/mahonia/gb18030.go",
"chars": 2866,
"preview": "package mahonia\n\nimport (\n\t\"sync\"\n)\n\n// Converters for GB18030 encoding.\n\nfunc init() {\n\tRegisterCharset(&Charset{\n\t\tNam"
},
{
"path": "common/mahonia/gbk-data.go",
"chars": 814791,
"preview": "package mahonia\n\n// Data for GBK 2-byte codes.\n// GBK codes are represented in big-endian order.\n// Based on the 2-byte "
},
{
"path": "common/mahonia/gbk.go",
"chars": 1133,
"preview": "package mahonia\n\n// Converters for GBK encoding.\n\nfunc init() {\n\tRegisterCharset(&Charset{\n\t\tName: \"GBK\",\n\t\tAliases: "
},
{
"path": "common/mahonia/iso2022jp.go",
"chars": 2253,
"preview": "package mahonia\n\nimport (\n\t\"unicode/utf8\"\n)\n\n// converters for ISO-2022-JP encoding\n\nconst esc = 27\n\nfunc init() {\n\ttype"
},
{
"path": "common/mahonia/jis0201-data.go",
"chars": 6498,
"preview": "package mahonia\n\nvar jis0201ToUnicode = [256]uint16{\n\t0x20: 0x0020, // SPACE\n\t0x21: 0x0021, // EXCLAMATION MARK\n\t0x22: 0"
},
{
"path": "common/mahonia/jis0208-data.go",
"chars": 118945,
"preview": "package mahonia\n\nvar jis0208Table = kutenTable{\n\tData: [94][94]uint16{\n\t\t0x00: [94]uint16{\n\t\t\t0x00: 0x3000,\n\t\t\t0x01: 0x3"
},
{
"path": "common/mahonia/jis0212-data.go",
"chars": 104916,
"preview": "package mahonia\n\nvar jis0212Table = kutenTable{\n\tData: [94][94]uint16{\n\t\t0x01: [94]uint16{\n\t\t\t0x0e: 0x02d8,\n\t\t\t0x0f: 0x0"
}
]
// ... and 191 more files (download for full content)
About this extraction
This page contains the full source code of the andeya/pholcus GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 391 files (6.2 MB), approximately 1.6M tokens, and a symbol index with 3148 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.