fingerprint) {
this.fingerprint = fingerprint;
}
}
================================================
FILE: src/main/java/model/HttpMsgInfo.java
================================================
package model;
import burp.*;
import java.nio.charset.StandardCharsets;
import java.util.zip.CRC32;
import static utils.CastUtils.isNotEmptyObj;
//创建一个类用于存储 代理 流量的解析结果
public class HttpMsgInfo {
private static final IExtensionHelpers helpers = BurpExtender.getHelpers();
private byte[] reqBytes;
private byte[] respBytes;
private String reqMethod;
private HttpUrlInfo urlInfo;
private HttpRespInfo respInfo;
private int respStatusCode;
private String respTitle;
private String msgHash;
// 构造函数
public HttpMsgInfo(IInterceptedProxyMessage iInterceptedProxyMessage) {
IHttpRequestResponse messageInfo = iInterceptedProxyMessage.getMessageInfo();
reqBytes = messageInfo.getRequest();
//请求方法
IRequestInfo requestInfoBetter = helpers.analyzeRequest(messageInfo);
reqMethod = requestInfoBetter.getMethod();
//从请求URL解析部分信息 //直接从请求体是没有办法获取到请求URL信息的, URL此时只能从外部传入
String reqUrl = requestInfoBetter.getUrl().toString();
urlInfo = new HttpUrlInfo(reqUrl);
//从响应结果解析部分信息
respBytes = messageInfo.getResponse();
respInfo = new HttpRespInfo(respBytes);
//响应码是常用的
respStatusCode = respInfo.getStatusCode();
respTitle = respInfo.getRespTitle();
//请求响应信息的简单hash值
msgHash = calcMsgHash(urlInfo.getUrlToFileUsual(),reqMethod,respStatusCode,respInfo.getBodyLenVague());
}
// 构造函数
public HttpMsgInfo(IHttpRequestResponse iHttpRequestResponse) {
//请求信息
reqBytes = iHttpRequestResponse.getRequest();
//请求方法
IHttpService httpService = iHttpRequestResponse.getHttpService();
IRequestInfo requestInfoBetter = helpers.analyzeRequest(httpService,reqBytes);
reqMethod = requestInfoBetter.getMethod();
//从请求URL解析部分信息
String reqUrl = requestInfoBetter.getUrl().toString();
urlInfo = new HttpUrlInfo(reqUrl);
//从响应结果解析部分信息
respBytes = iHttpRequestResponse.getResponse();
respInfo = new HttpRespInfo(respBytes);
//响应码是常用的
respStatusCode = respInfo.getStatusCode();
respTitle = respInfo.getRespTitle();
//请求响应信息的简单hash值
msgHash = calcMsgHash(urlInfo.getUrlToFileUsual(),reqMethod,respStatusCode,respInfo.getBodyLenVague());
}
// 构造函数
public HttpMsgInfo(String requestUrl, byte[] requestBytes, byte[] responseBytes, String msgInfoHash) {
//请求信息
reqBytes = requestBytes;
//请求方法
IRequestInfo requestInfoSimple = helpers.analyzeRequest(reqBytes);
reqMethod = requestInfoSimple.getMethod();
//从请求URL解析部分信息
String reqUrl = requestUrl;
urlInfo = new HttpUrlInfo(reqUrl);
//从响应结果解析部分信息
respBytes = responseBytes;
respInfo = new HttpRespInfo(respBytes);
//响应码是常用的
respStatusCode = respInfo.getStatusCode();
respTitle = respInfo.getRespTitle();
//请求响应信息的简单hash值 因为中间可能截断了超大的响应体 , 因此最好手动传入 msgHash
msgHash = msgInfoHash;
}
/**
* 计算消息Hash
*/
private String calcMsgHash(String urlToFileUsual, String reqMethod, int respStatusCode, int respBodyLenVague) {
return calcCRC32(String.format("%s|%s|%s|%s", urlToFileUsual, reqMethod, respStatusCode, respBodyLenVague));
}
/**
* 计算给定字符串的CRC32校验和,并以十六进制字符串形式返回。
* @param string 要计算CRC32的字符串
* @return 字符串的CRC32校验和的十六进制表示
*/
private static String calcCRC32(String string) {
// 使用 UTF-8 编码将字符串转换为字节数组
byte[] inputBytes = string.getBytes(StandardCharsets.UTF_8);
// 初始化CRC32对象
CRC32 crc32 = new CRC32();
// 更新CRC值
crc32.update(inputBytes, 0, inputBytes.length);
// 将计算后的CRC32值转换为十六进制字符串并返回
return Long.toHexString(crc32.getValue()).toLowerCase();
}
public String getReqMethod() {
return reqMethod;
}
public byte[] getRespBytes() {
return respBytes;
}
public byte[] getReqBytes() {
return reqBytes;
}
public String getMsgHash() {
return msgHash;
}
public void setRespBytes(byte[] respBytes) {
this.respBytes = respBytes;
}
public HttpUrlInfo getUrlInfo() {
return urlInfo;
}
public HttpRespInfo getRespInfo() {
return respInfo;
}
public int getRespStatusCode() {
return respStatusCode;
}
public String getRespTitle() {
return respTitle;
}
}
================================================
FILE: src/main/java/model/HttpRespInfo.java
================================================
package model;
import burp.BurpExtender;
import burp.IExtensionHelpers;
import burp.IResponseInfo;
import utils.RespHashUtils;
import utils.RespTitleUtils;
import java.util.Arrays;
public class HttpRespInfo {
private static final IExtensionHelpers helpers = BurpExtender.getHelpers();
private byte[] respBytes = "".getBytes();
private int statusCode = -1;
private int respLength = -1;
private int bodyLength = -1;
private int bodyLenVague = -1;
private String inferredMimeType = "";
private String statedMimeType = "";
private int bodyOffset = -1;
private String respTitle = "";
private String iconHash = ""; //记录响应体的hash值
public String getIconHash() {
return iconHash;
}
HttpRespInfo(byte[] responseBytes) {
if (responseBytes == null || responseBytes.length <= 0){
// Warning: That response body is empty !!!
return;
}
respBytes = responseBytes;
//响应长度
respLength = respBytes.length;
//响应信息
IResponseInfo responseInfo = helpers.analyzeResponse(respBytes);
//响应状态码
statusCode = responseInfo.getStatusCode();
//获取响应类型
inferredMimeType = responseInfo.getInferredMimeType(); //根据响应的内容自动推断出的 MIME 类型
statedMimeType = responseInfo.getStatedMimeType(); //由服务器明确声明的内容类型
//响应体分割标记
bodyOffset = responseInfo.getBodyOffset();
bodyLength = getBodyBytes().length;
//大致的响应长度
bodyLenVague = bodyLength / 200;
//响应文本标题
respTitle = RespTitleUtils.parseTextTitle(respBytes);
//当响应类型是 ico 类型时计算一下hash值
if (getStatedMimeType() != null && getStatedMimeType().contains("ico")){
iconHash = RespHashUtils.getFaviconHash(getBodyBytes());
}
}
/**
* 获取 请求体或响应体的body部分
*/
public byte[] getBodyBytes() {
// 确保 bodyOffset 不会导致数组越界
int bodyLength = Math.max(0, respBytes.length - bodyOffset);
// 从 bytes 数组中复制 body 的部分
return Arrays.copyOfRange(respBytes, bodyOffset, bodyOffset + bodyLength);
}
/**
* 获取 请求或响应的头部信息部分
*/
public byte[] getHeaderBytes() {
// 确保 headerOffset 不会导致数组越界,并且至少是从0开始
int headerLength = Math.max(0, bodyOffset);
// 从 bytes 数组中复制 header 的部分
return Arrays.copyOfRange(respBytes, 0, headerLength);
}
public int getStatusCode() {
return statusCode;
}
public int getRespLength() {
return respLength;
}
public int getBodyLength() {
return bodyLength;
}
public int getBodyLenVague() {
return bodyLenVague;
}
public String getInferredMimeType() {
return inferredMimeType;
}
public String getStatedMimeType() {
return statedMimeType;
}
public int getBodyOffset() {
return bodyOffset;
}
public byte[] getRespBytes() {
return respBytes;
}
public String getRespTitle() {
return respTitle;
}
}
================================================
FILE: src/main/java/model/HttpUrlInfo.java
================================================
package model;
import utilbox.DomainUtils;
import java.net.MalformedURLException;
import java.net.URL;
import static utils.BurpPrintUtils.stderr_println;
import static utils.CastUtils.*;
//创建一个类用于存储 URL解析结果的类
public class HttpUrlInfo {
private String rawUrl;
private String rawUrlUsual;
private String proto = null;
private String host = null;
private int port = -1;
private String file = null;
private String query = null;
private String ref = null;
private String hostPort = null;
private String hostPortUsual = null;
private String pathToFile = null;
private String pathToDir = null;
private String pathToEnd = null;
private String suffix = null;
private String suffixUsual = null;
private String rootDomain = null;
private String rootUrl = null;
private String rootUrlUsual = null;
private String rootUrlNotSlash = null;
private String urlToFile = null;
private String urlToPath = null;
private String urlToFileUsual = null;
private String urlToPathUsual = null;
public HttpUrlInfo(String requestUrl){
rawUrl = requestUrl;
//基于URL获取其他请求信息
try {
URL urlObj = new URL(rawUrl);
//协议 (protocol):如 http 或 https
proto = urlObj.getProtocol(); //协议 (protocol):如 http 或 https
//主机 (host):如 www.example.com
host = urlObj.getHost();
//端口 (port):如 80 或 443(默认情况下,如果未指定,http 默认为 80,https 默认为 443) 同时 检查reqPort为-1的情况
port = urlObj.getPort() < 0 ? urlObj.getDefaultPort() : urlObj.getPort();
//文件 resource
file = urlObj.getFile();
//查询参数 (query):如 ?key=value&anotherKey=anotherValue
query = urlObj.getQuery();
//片段标识符 (fragment):如 #section1
ref = urlObj.getRef();
//添加个HostPort对象 www.baidu.com:80 | www.baidu.com:8080
hostPort = String.format("%s:%s", host, port);
//获取没有默认端口的请求头 www.baidu.com | www.baidu.com:8080
hostPortUsual = removeHostDefaultPort(hostPort,host,port);
//获取前缀URL // http://www.baidu.com:80/
rootUrl = String.format("%s://%s/", proto, hostPort);
//获取前缀URL // http://www.baidu.com/
rootUrlUsual = String.format("%s://%s/", proto, hostPortUsual);
//获取前缀URL // http://www.baidu.com
rootUrlNotSlash = String.format("%s://%s", proto, hostPortUsual);
//解析请求文件的后缀 php html
suffix = parseUrlExtStrict(file); //严重错误,域名中是有.符号的,因此不能直接截断域名
//解析请求文件的后缀 .php .html
suffixUsual = isEmptyObj(suffix) ? suffix:"." + suffix;
//获取主域名 baidu.com
rootDomain = DomainUtils.getRootDomain(host);
//路径 (path):如 /path/to/resource
pathToFile = urlObj.getPath();
// 重新构造基本URL,不包含查询参数 http://www.baidu.com/path/to/resource
urlToFile = new URL(proto, host, port, pathToFile).toString();
urlToFileUsual = removeUrlDefaultPort(urlToFile);
//获取请求路径的目录部分 /path/to/
pathToDir = parseReqPathDir(pathToFile);
//构造基本URL, 不包含请求文件 http://www.baidu.com/path/to/
urlToPath = new URL(proto, host, port, pathToDir).toString();
urlToPathUsual = removeUrlDefaultPort(urlToPath);
//获取带有参数的完整Path 不带http信息 /path/to/resource?key=value#section1
pathToEnd = genFullPath(pathToFile, query, ref);
//格式化URL 不显示默认端口
rawUrlUsual = removeUrlDefaultPort(rawUrl);
} catch (MalformedURLException e) {
stderr_println(String.format("Invalid URL: %s -> Error: %s", rawUrl, e.getMessage()));
e.printStackTrace();
}
}
/**
* 拼接 Path路径、?查询字符串、#索引
*/
private String genFullPath(String pathToFile,String query,String ref) {
StringBuilder fullPart = new StringBuilder(pathToFile);
if (isNotEmptyObj(query)) {
fullPart.append("?").append(query);
}
if (isNotEmptyObj(ref)) {
fullPart.append("#").append(ref);
}
return fullPart.toString();
}
/**
* 从 path 解析请求后缀 严格模式 处理 # 和 ?
*/
private String parseUrlExtStrict(String path) {
//忽略为空的情况
if (isEmptyObj(path)) return "";
int queryIndex = path.indexOf('?');
int fragmentIndex = path.indexOf('#');
int endIndex = -1;
// 计算有效部分的结束索引
if (queryIndex > 0 && fragmentIndex >0){
endIndex = Math.min(queryIndex, fragmentIndex);
} else if (queryIndex > 0 || fragmentIndex >0){
endIndex = Math.max(queryIndex, fragmentIndex);
} else {
endIndex = path.length();
}
// 截取有效部分;否则使用整个URL
String pureUrl = path.substring(0, endIndex);
// 查找最后一个`.`的位置
int lastDotIndex = pureUrl.lastIndexOf('.');
// 如果有扩展名,提取它;否则返回空字符串
String extension = lastDotIndex > -1 ? pureUrl.substring(lastDotIndex + 1) : "";
// 将扩展名转换为小写
return extension.toLowerCase();
}
/**
* 从给定的URL字符串中提取请求的目录部分。
* @param reqPath 完整的URL字符串。
* @return 请求的目录路径,不包含最后一个路径分隔符。
*/
public static String parseReqPathDir(String reqPath) {
// 去除最后一个路径分隔符后面的文件名部分,如果有的话
int lastPathSepIndex = reqPath.lastIndexOf('/');
// 如果找到了路径分隔符(lastPathSepIndex 不等于 -1)
if (lastPathSepIndex != -1) {
// 从原始路径中截取出从开头到最后一个路径分隔符(包括该分隔符)的部分 +1是为了保留最后一个路径分隔符
return reqPath.substring(0, lastPathSepIndex + 1);
}
return "/";
}
public String getRawUrlUsual() {
return rawUrlUsual;
}
public String getProto() {
return proto;
}
public String getHost() {
return host;
}
public String getHostPort() {
return hostPort;
}
public String getRootUrlUsual() {
return rootUrlUsual;
}
private String getRootUrl() {
return rootUrl;
}
public String getRootDomain() {
return rootDomain;
}
public int getPort() {
return port;
}
public String getQuery() {
return query;
}
public String getRef() {
return ref;
}
public String getPathToFile() {
return pathToFile;
}
public String getPathToDir() {
return pathToDir;
}
public String getSuffix() {
return suffix;
}
public String getSuffixUsual() {
return suffixUsual;
}
public String getUrlToFileUsual() {
return urlToFileUsual;
}
public String getUrlToPathUsual() {
return urlToPathUsual;
}
public String getPathToEnd() {
return pathToEnd;
}
public String getHostPortUsual() {
return hostPortUsual;
}
public String getRawUrl() {
return rawUrl;
}
public String getFile() {
return file;
}
public String getUrlToFile() {
return urlToFile;
}
public String getUrlToPath() {
return urlToPath;
}
public String getRootUrlNotSlash() {
return rootUrlNotSlash;
}
/**
* 1.remove default port(80\443) from the url
* 2.add default path(/) to the url,if it's empty
* 这个函数的目的是让URL的格式和通常从浏览器中复制的格式一致:
* 在浏览器中,我们看到的是 baidu.com, 复制粘贴得到的是 https://www.baidu.com/
*
* 比如
* http://bit4woo.com:80/ ---> http://bit4woo.com/
* https://bit4woo.com:443 ---> https://bit4woo.com/
*/
private String removeUrlDefaultPort(String urlString) {
try {
URL url = new URL(urlString);
String proto = url.getProtocol();
String host = url.getHost();
int port = url.getPort(); //不包含端口时返回-1
String path = url.getPath();
if (port < 0 ||
(port == 80 && proto.equalsIgnoreCase("http")) ||
(port == 443 && proto.equalsIgnoreCase("https"))
) {
String oldHost = url.getHost() + ":" + url.getPort();
urlString = urlString.replaceFirst(oldHost, host);
}
if (path.equals("")) {
urlString = urlString + "/";
}
return new URL(urlString).toString();
} catch (MalformedURLException e) {
e.printStackTrace();
return urlString;
}
}
private String removeHostDefaultPort(String hostPort, String host, int port) {
if (port < 0
|| (port == 80 && proto.equalsIgnoreCase("http"))
|| (port == 443 && proto.equalsIgnoreCase("https"))){
return host;
}
return hostPort;
}
/**
* 1、这个函数的目的是:在【浏览器URL】的基础上,加上默认端口。
*
* https://www.baidu.com/ ---> https://www.baidu.com:443/
* http://www.baidu.com ---> http://www.baidu.com:80/
*
* 在浏览器中,我们看到的是 baidu.com, 复制粘贴得到的是 https://www.baidu.com/
* let url String contains default port(80\443) and default path(/)
*
* burp中获取到的URL是包含默认端口的,但是平常浏览器中的URL格式都是不包含默认端口的。
* 应该尽量和平常使用习惯保存一致!所以尽量避免使用该函数。
*
* @param urlStr
* @return
*/
private String addUrlDefaultPort(String urlStr) {
try {
URL url = new URL(urlStr);
String host = url.getHost();
int port = url.getPort();
String path = url.getPath();
if (port == -1) {
String newHost = url.getHost() + ":" + url.getDefaultPort();
urlStr = urlStr.replaceFirst(host, newHost);
}
if (path.equals("")) {
urlStr = urlStr + "/";
}
return new URL(urlStr).toString();
} catch (MalformedURLException e) {
e.printStackTrace();
return urlStr;
}
}
}
================================================
FILE: src/main/java/model/PathToUrlsModel.java
================================================
package model;
import com.alibaba.fastjson2.JSONArray;
import utils.CastUtils;
import java.util.List;
public class PathToUrlsModel {
private int id;
private int basicPathNum;
private List pathToUrls;
private List unvisitedUrls;
public PathToUrlsModel(int id, int basic_path_num, JSONArray pathToUrls, JSONArray unvisitedUrls) {
this.id = id;
this.basicPathNum = basic_path_num;
this.pathToUrls = CastUtils.toStringList(pathToUrls);
this.unvisitedUrls = CastUtils.toStringList(unvisitedUrls);
}
public PathToUrlsModel(int id, int basic_path_num, String pathToUrls, String unvisitedUrls) {
this.id = id;
this.basicPathNum = basic_path_num;
this.pathToUrls = CastUtils.toStringList(pathToUrls);
this.unvisitedUrls = CastUtils.toStringList(unvisitedUrls);
}
public int getId() {
return id;
}
public void setId(int id) {
this.id = id;
}
public int getBasicPathNum() {
return basicPathNum;
}
public void setBasicPathNum(int basicPathNum) {
this.basicPathNum = basicPathNum;
}
public List getPathToUrls() {
return pathToUrls;
}
public void setPathToUrls(List pathToUrls) {
this.pathToUrls = pathToUrls;
}
public List getUnvisitedUrls() {
return unvisitedUrls;
}
public void setUnvisitedUrls(List unvisitedUrls) {
this.unvisitedUrls = unvisitedUrls;
}
}
================================================
FILE: src/main/java/model/PathTreeModel.java
================================================
package model;
import com.alibaba.fastjson2.JSONObject;
import utils.CastUtils;
public class PathTreeModel {
private String rootUrl;
private Integer basicPathNum;
private JSONObject pathTree;
public PathTreeModel(String rootUrl, Integer basicPathNum, JSONObject pathTree) {
this.rootUrl = rootUrl;
this.basicPathNum = basicPathNum;
this.pathTree = pathTree;
}
public PathTreeModel(String rootUrl, int basicPathNum, String pathTree) {
this.rootUrl = rootUrl;
this.basicPathNum = basicPathNum;
this.pathTree = CastUtils.toJsonObject(pathTree);
}
public Integer getBasicPathNum() {
return basicPathNum;
}
public JSONObject getPathTree() {
return pathTree;
}
public String getRootUrl() {
return rootUrl;
}
}
================================================
FILE: src/main/java/model/RecordHashMap.java
================================================
package model;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
public class RecordHashMap {
private final ConcurrentHashMap countMap;
public RecordHashMap() {
this.countMap = new ConcurrentHashMap<>();
}
public Map getStringMap() {
return this.countMap;
}
public Integer get(String key) {
Integer ret = this.countMap.get(key);
if (ret == null) {
return 0;
} else {
return ret;
}
}
public void add(String key) {
if (key == null || key.length() <= 0) {
throw new IllegalArgumentException("Key 不能为空");
}
synchronized (this.getStringMap()) {
this.countMap.put(key, (this.get(key) + 1));
}
}
public void del(String key) {
if (this.countMap.get(key) != null) {
this.countMap.remove(key);
}
}
}
================================================
FILE: src/main/java/model/RecordPathDirsModel.java
================================================
package model;
public class RecordPathDirsModel {
private String rootUrl;
private String reqPathDirs;
// 构造函数
public RecordPathDirsModel(String rootUrl, String reqPathDirs) {
this.rootUrl = rootUrl;
this.reqPathDirs = reqPathDirs;
}
public String getRootUrl() {
return rootUrl;
}
public String getReqPathDirs() {
return reqPathDirs;
}
}
================================================
FILE: src/main/java/model/RecordPathModel.java
================================================
package model;
import java.nio.charset.StandardCharsets;
import java.util.zip.CRC32;
public class RecordPathModel {
private String reqHash;
private String rootUrl;
private String reqPathDir;
private int respStatusCode;
public RecordPathModel(String rootUrl, String reqPathDir, int respStatusCode) {
this.rootUrl = rootUrl;
this.reqPathDir = reqPathDir;
this.respStatusCode = respStatusCode;
this.reqHash = getCalcCRC32();
}
public RecordPathModel(HttpUrlInfo urlInfo, int respStatusCode) {
this.rootUrl = urlInfo.getRootUrlUsual();
this.reqPathDir = urlInfo.getPathToDir();
this.respStatusCode = respStatusCode;
this.reqHash = getCalcCRC32();
}
private String getCalcCRC32() {
return calcCRC32(String.format("%s|%s|%s", this.rootUrl, this.reqPathDir, this.respStatusCode));
}
/**
* 计算给定字符串的CRC32校验和,并以十六进制字符串形式返回。
* @param string 要计算CRC32的字符串
* @return 字符串的CRC32校验和的十六进制表示
*/
private String calcCRC32(String string) {
// 使用 UTF-8 编码将字符串转换为字节数组
byte[] inputBytes = string.getBytes(StandardCharsets.UTF_8);
// 初始化CRC32对象
CRC32 crc32 = new CRC32();
// 更新CRC值
crc32.update(inputBytes, 0, inputBytes.length);
// 将计算后的CRC32值转换为十六进制字符串并返回
return Long.toHexString(crc32.getValue()).toLowerCase();
}
public String getReqPathDir() {
return reqPathDir;
}
public int getRespStatusCode() {
return respStatusCode;
}
public String getReqHash() {
return reqHash;
}
public String getRootUrl() {
return rootUrl;
}
}
================================================
FILE: src/main/java/model/ReqMsgDataModel.java
================================================
package model;
public class ReqMsgDataModel {
private String msgHash;
private String reqUrl;
private byte[] reqBytes;
private byte[] respBytes;
public ReqMsgDataModel(String msgHash, String reqUrl, byte[] reqBytes, byte[] respBytes) {
this.msgHash = msgHash;
this.reqUrl = reqUrl;
this.reqBytes = reqBytes;
this.respBytes = respBytes;
}
public String getMsgHash() {
return msgHash;
}
public String getReqUrl() {
return reqUrl;
}
public byte[] getReqBytes() {
return reqBytes;
}
public byte[] getRespBytes() {
return respBytes;
}
}
================================================
FILE: src/main/java/model/ReqUrlRespStatusModel.java
================================================
package model;
public class ReqUrlRespStatusModel {
private Integer id;
private String reqUrl;
private String reqMethod;
private Integer respStatusCode;
private Integer respLength;
// 有参构造函数
public ReqUrlRespStatusModel(Integer id, String reqUrl, String reqMethod, Integer respStatusCode, Integer respLength) {
this.id = id;
this.reqUrl = reqUrl;
this.reqMethod = reqMethod;
this.respStatusCode = respStatusCode;
this.respLength = respLength;
}
public Integer getId() {
return id;
}
public String getReqUrl() {
return reqUrl;
}
public String getReqMethod() {
return reqMethod;
}
public Integer getRespStatusCode() {
return respStatusCode;
}
public Integer getRespLength() {
return respLength;
}
}
================================================
FILE: src/main/java/model/RespFieldsModel.java
================================================
package model;
import com.alibaba.fastjson2.JSON;
import utils.CastUtils;
import utils.RespHashUtils;
import java.util.*;
/**
* 用于响应信息对比的数据模型
*/
public class RespFieldsModel {
private Integer statusCode; // 响应状态码,需要忽略 200 的情况
private Integer respLength; // 响应头中的长度 需要忽略小于0的情况
private Integer respBodyLength; // 响应内容大小
private String respTextTitle; // 响应文本标题
private String respHashContent; // 响应内容HASH
private String respRedirectUrl; // 响应重定向URL
public RespFieldsModel(HttpRespInfo respInfo) {
this.statusCode = respInfo.getStatusCode();
this.respLength = respInfo.getRespLength();
this.respBodyLength = respInfo.getBodyLength();
this.respTextTitle = respInfo.getRespTitle();
this.respRedirectUrl = CastUtils.parseRespRedirectUrl(respInfo.getHeaderBytes());
this.respHashContent = RespHashUtils.calcCRC32(respInfo.getBodyBytes());
}
public String toJSONString(){
return JSON.toJSONString(getAllFieldsAsMap());
}
// 新增方法:获取所有属性的名称和值
public Map getAllFieldsAsMap() {
Map fieldMap = new HashMap<>();
fieldMap.put("StatusCode", statusCode);
fieldMap.put("RespLength", respLength);
fieldMap.put("BodyLength", respBodyLength);
fieldMap.put("RespTitle", respTextTitle);
fieldMap.put("RespHash", respHashContent);
fieldMap.put("RedirectUrl", respRedirectUrl);
return fieldMap;
}
}
================================================
FILE: src/main/java/model/UnVisitedUrlsModel.java
================================================
package model;
import utils.CastUtils;
import java.util.List;
public class UnVisitedUrlsModel {
private int id;
private String rootUrl;
private List unvisitedUrls;
public UnVisitedUrlsModel(int id, String rootUrl, String unvisitedUrl) {
this.id = id;
this.rootUrl = rootUrl;
this.unvisitedUrls = CastUtils.toStringList(unvisitedUrl);
}
public int getId() {
return id;
}
public String getRootUrl() {
return rootUrl;
}
public List getUnvisitedUrls() {
return unvisitedUrls;
}
public void setUnvisitedUrls(List unvisitedUrls) {
this.unvisitedUrls = unvisitedUrls;
}
}
================================================
FILE: src/main/java/sqlUtils/CommonDeleteLine.java
================================================
package sqlUtils;
import database.DBService;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.List;
import static utils.BurpPrintUtils.stderr_println;
import static utils.CastUtils.isEmptyObj;
public class CommonDeleteLine {
/**
* 执行删除数据行的SQL语句
*/
private static int runDeleteByStringsSQL(String tableName, List stringList, String deleteSQL) {
int totalRowsAffected = 0;
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmt = conn.prepareStatement(deleteSQL)) {
// 设置SQL语句中的参数值 i+1表示从第一个?号开始设置
for (int i = 0; i < stringList.size(); i++) {
stmt.setString(i + 1, stringList.get(i));
}
// 执行删除操作
totalRowsAffected = stmt.executeUpdate();
} catch (Exception e) {
stderr_println(String.format("[-] runDeleteSql: [%s] -> Error: %s", tableName, e.getMessage()));
e.printStackTrace();
}
return totalRowsAffected;
}
//基于 rootUrls 列表 同时删除多行
public static synchronized int deleteLineByRootUrls(String tableName, List rootUrls) {
if (isEmptyObj(rootUrls)) return 0;
// 构建SQL语句,使用占位符 ? 来代表每个ID
String deleteSQL = ("DELETE FROM "+ tableName +" WHERE root_url IN $buildInParamList$;")
.replace("$buildInParamList$", buildSQL.buildInParamList(rootUrls.size()));
return runDeleteByStringsSQL(tableName, rootUrls, deleteSQL);
}
//基于 msgHash 列表 同时删除多个 行
public static synchronized int deleteLineByMsgHashList(String tableName, List msgHashList) {
if (isEmptyObj(msgHashList)) return 0;
// 构建SQL语句,使用占位符 ? 来代表每个ID
String deleteSQL = ("DELETE FROM "+ tableName + " WHERE msg_hash IN $buildInParamList$;")
.replace("$buildInParamList$", buildSQL.buildInParamList(msgHashList.size()));
return runDeleteByStringsSQL(tableName, msgHashList, deleteSQL);
}
/**
* 基于 id 列表 同时删除多个 行
*/
public static synchronized int deleteLineByIds(String tableName, List ids) {
int totalRowsAffected = 0;
if (ids.isEmpty()) return totalRowsAffected;
// 构建SQL语句,使用占位符 ? 来代表每个ID
String deleteSQL = ("DELETE FROM "+ tableName + " WHERE id IN $buildInParamList$;")
.replace("$buildInParamList$", buildSQL.buildInParamList(ids.size()));
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmt = conn.prepareStatement(deleteSQL)) {
// 设置SQL语句中的参数值 i+1表示从第一个?号开始设置
for (int i = 0; i < ids.size(); i++) {
stmt.setInt(i + 1, ids.get(i));
}
// 执行删除操作
totalRowsAffected = stmt.executeUpdate();
} catch (Exception e) {
stderr_println(String.format("[-] Error deleting Data By Ids On Table [%s] -> Error:[%s]", tableName, e.getMessage()));
e.printStackTrace();
}
return totalRowsAffected;
}
/**
* 基于 多个url前缀 列表 删除行
*/
public static synchronized int deleteLineByUrlLikeRootUrls(String tableName, List rootUrlList) {
if (isEmptyObj(rootUrlList)) return 0;
int totalRowsAffected = 0;
String deleteSQL = "DELETE FROM "+ tableName + " WHERE req_url LIKE ?;";
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmt = conn.prepareStatement(deleteSQL)) {
// 开启批处理
conn.setAutoCommit(false);
// 遍历rootUrlList,为每个rootUrl准备并添加到批处理队列
for (String rootUrl : rootUrlList) {
stmt.setString(1, rootUrl + "%");
stmt.addBatch();
}
// 执行批处理
int[] rowsAffected = stmt.executeBatch();
conn.commit();
// 计算受影响的总行数
for (int row : rowsAffected) {
totalRowsAffected += row;
}
} catch (Exception e) {
stderr_println(String.format("[-] Error deleting [%s] Data By Starts With rootUrl List: %s", tableName, e.getMessage()));
}
return totalRowsAffected;
}
}
================================================
FILE: src/main/java/sqlUtils/CommonFetchData.java
================================================
package sqlUtils;
import database.DBService;
import database.ReqDataTable;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.List;
import static utils.BurpPrintUtils.*;
public class CommonFetchData {
/**
* 统计数据表行数大小
*/
public static synchronized int fetchTableCounts(String tableName) {
int count = 0;
String selectSQL = "SELECT COUNT(*) FROM "+ tableName +" ;";
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmt = conn.prepareStatement(selectSQL);
ResultSet rs = stmt.executeQuery()) {
if (rs.next()) {
count = rs.getInt(1); // 获取第一列的值,即 COUNT(*) 的结果
}
} catch (Exception e) {
stderr_println(String.format("Error Counts [%s]: %s",tableName, e.getMessage() ));
}
return count;
}
/**
* 统计所有已经识别完成的URL的数量
* @return
*/
public static synchronized int fetchTableCountsByStatus(String analyseStatus) {
int count = 0;
String selectSQL = "SELECT COUNT(*) FROM "+ ReqDataTable.tableName + " WHERE run_status = ?;";
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmt = conn.prepareStatement(selectSQL)){
stmt.setString(1, analyseStatus);
ResultSet rs = stmt.executeQuery();
if (rs.next()) {
count = rs.getInt(1); // 获取第一列的值,即 COUNT(*) 的结果
}
} catch (Exception e) {
stderr_println(String.format("Counts Table [%s] Error: %s", ReqDataTable.tableName, e.getMessage() ));
}
return count;
}
/**
* 根据运行状态取获取对应 ID list
*/
public static synchronized List fetchIdsByRunStatus(String tableName, String analyseStatus, int limit) {
List ids = new ArrayList<>();
String selectSQL = "SELECT id FROM " + tableName + " WHERE run_status = ? ORDER BY id ASC LIMIT ?;";
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmt = conn.prepareStatement(selectSQL)) {
stmt.setString(1, analyseStatus);
stmt.setInt(2, limit); // Set the limit parameter
ResultSet rs = stmt.executeQuery();
while (rs.next()) {
int id = rs.getInt("id");
ids.add(id);
}
} catch (Exception e) {
stderr_println(LOG_DEBUG, String.format("[-] Error fetching [%s] ids: %s", tableName, e.getMessage()));
}
return ids;
}
/**
* 根据运行状态取获取对应请求 msghash list
* @return
*/
public static synchronized List fetchMsgHashByRunStatus(String tableName, String analyseStatus, int limit) {
List msgHashList = new ArrayList<>();
String selectSQL = "SELECT msg_hash FROM " + tableName + " WHERE run_status = ? ORDER BY id ASC LIMIT ?;";
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmt = conn.prepareStatement(selectSQL)) {
stmt.setString(1, analyseStatus);
stmt.setInt(2, limit); // Set the limit parameter
ResultSet rs = stmt.executeQuery();
while (rs.next()) {
String msgHash = rs.getString("msg_hash");
msgHashList.add(msgHash);
}
} catch (Exception e) {
stderr_println(LOG_DEBUG, String.format("[-] Error fetching [%s] MsgHash List from Analysis: %s",tableName, e.getMessage()));
}
return msgHashList;
}
/////////////////////////////
/**
* 获取任意表的任意列的字符串列表 【基于msgHashList】
*/
public static synchronized List fetchColumnStrListByMsgHashList(String tableName, String columnName, List msgHashList) {
List stringList = new ArrayList<>();
if (msgHashList.isEmpty())
return stringList;
String selectSQL = ("SELECT " + columnName + " FROM "+ tableName +" WHERE msg_hash IN $buildInParameterList$;")
.replace("$buildInParameterList$", buildSQL.buildInParamList(msgHashList.size()));
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmt = conn.prepareStatement(selectSQL)) {
for (int i = 0; i < msgHashList.size(); i++) {
stmt.setString(i + 1, msgHashList.get(i));
}
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
stringList.add(rs.getString(columnName));
}
}
} catch (Exception e) {
stderr_println(LOG_ERROR, String.format("[-] fetchColumnStrListByMsgHashList: [%s] [%s] -> Error: %s",tableName, columnName, e.getMessage()));
}
return stringList;
}
/////////////////////////////
/**
* 获取任意表的任意列的字符串拼接 【获取所有行】
*/
public static synchronized String fetchColumnGroupConcatString(String tableName, String columnName) {
String concatenatedURLs = null;
String concatSQL = ("SELECT GROUP_CONCAT($columnName$,',') AS concatenated_urls FROM "+ tableName +";")
.replace("$columnName$",columnName);
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmt = conn.prepareStatement(concatSQL)) {
ResultSet rs = stmt.executeQuery();
if (rs.next()) {
concatenatedURLs = rs.getString("concatenated_urls");
}
} catch (Exception e) {
stderr_println(LOG_ERROR, String.format("[-] Error fetching [%s] concatenating [%s]: %s",tableName, columnName, e.getMessage()));
e.printStackTrace();
}
return concatenatedURLs;
}
/**
* 获取任意表的任意列的字符串拼接 InRootUrls
*/
public static synchronized String fetchColumnGroupConcatStringInRootUrls(String tableName, String columnName, List rootUrls) {
String concatenatedURLs = null;
String concatSQL = ("SELECT GROUP_CONCAT($columnName$,',') AS concatenated_urls FROM "+ tableName +
" WHERE root_url IN $buildInParameterList$;")
.replace("$columnName$",columnName)
.replace("$buildInParameterList$", buildSQL.buildInParamList(rootUrls.size()));
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmt = conn.prepareStatement(concatSQL)) {
for (int i = 0; i < rootUrls.size(); i++) {
stmt.setString(i + 1, rootUrls.get(i));
}
ResultSet rs = stmt.executeQuery();
if (rs.next()) {
concatenatedURLs = rs.getString("concatenated_urls");
}
} catch (Exception e) {
stderr_println(LOG_ERROR, String.format("[-] Error fetch [%s] [%s] Column Group Concat String In RootUrls concatenating: %s",tableName, columnName, e.getMessage()));
e.printStackTrace();
}
return concatenatedURLs;
}
/**
* 获取任意表的任意列的字符串拼接 NotInRootUrls
*/
public static synchronized String fetchColumnGroupConcatStringNotInRootUrls(String tableName, String columnName, List rootUrls) {
String concatenatedURLs = null;
String concatSQL = ("SELECT GROUP_CONCAT($columnName$,',') AS concatenated_urls FROM "+ tableName +
" WHERE root_url NOT IN $buildInParameterList$;")
.replace("$columnName$",columnName)
.replace("$buildInParameterList$", buildSQL.buildInParamList(rootUrls.size()));
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmt = conn.prepareStatement(concatSQL)) {
for (int i = 0; i < rootUrls.size(); i++) {
stmt.setString(i + 1, rootUrls.get(i));
}
ResultSet rs = stmt.executeQuery();
if (rs.next()) {
concatenatedURLs = rs.getString("concatenated_urls");
}
} catch (Exception e) {
stderr_println(LOG_ERROR, String.format("[-] Error fetch [%s] [%s] Column Group Concat String Not In RootUrls concatenating: %s",tableName, columnName, e.getMessage()));
e.printStackTrace();
}
return concatenatedURLs;
}
}
================================================
FILE: src/main/java/sqlUtils/CommonUpdateStatus.java
================================================
package sqlUtils;
import database.DBService;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.List;
import static utils.BurpPrintUtils.*;
public class CommonUpdateStatus {
/**
* 更新多个 ID列表 的状态
*/
public static synchronized int updateStatusByIds(String tableName, List ids, String updateStatus) {
int updatedCount = -1;
String updateSQL = ("UPDATE " + tableName + " SET run_status = ? WHERE id IN $buildInParamList$;")
.replace("$buildInParamList$", buildSQL.buildInParamList(ids.size()));
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmtUpdate = conn.prepareStatement(updateSQL)) {
stmtUpdate.setString(1, updateStatus);
for (int i = 0; i < ids.size(); i++) {
stmtUpdate.setInt(i + 2, ids.get(i));
}
updatedCount = stmtUpdate.executeUpdate();
if (updatedCount != ids.size()) {
stderr_println(LOG_DEBUG, "[!] Number of updated rows does not match number of selected rows.");
}
} catch (Exception e) {
stderr_println(LOG_ERROR, String.format("[-] Error updating [%s] Data Status: %s", tableName, e.getMessage()));
}
return updatedCount;
}
/**
* 更新多个 msgHash 的状态
*/
public static synchronized int updateStatusByMsgHashList(String tableName, List msgHashList, String updateStatus) {
int updatedCount = -1;
String updateSQL = ("UPDATE " + tableName + " SET run_status = ? WHERE msg_hash IN $buildInParamList$;")
.replace("$buildInParamList$", buildSQL.buildInParamList(msgHashList.size()));
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmtUpdate = conn.prepareStatement(updateSQL)) {
stmtUpdate.setString(1, updateStatus);
for (int i = 0; i < msgHashList.size(); i++) {
stmtUpdate.setString(i + 2, msgHashList.get(i));
}
updatedCount = stmtUpdate.executeUpdate();
if (updatedCount != msgHashList.size()) {
stderr_println(LOG_DEBUG, "[!] Number of updated rows does not match number of selected rows.");
}
} catch (Exception e) {
stderr_println(LOG_ERROR, String.format("[-] Error updating [%s] Data Status: %s",tableName, e.getMessage()));
}
return updatedCount;
}
/**
* 基于 msgDataIndexList 更新 状态
*/
public static synchronized int updateStatusByMsgDataIndexList(String tableName, List msgDataIndexList, String updateStatus) {
int updatedCount = -1;
String updateSQL = ("UPDATE " + tableName + " SET run_status = ? WHERE msg_data_index IN $buildInParamList$;")
.replace("$buildInParamList$", buildSQL.buildInParamList(msgDataIndexList.size()));
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmtUpdate = conn.prepareStatement(updateSQL)) {
stmtUpdate.setString(1, updateStatus);
for (int i = 0; i < msgDataIndexList.size(); i++) {
stmtUpdate.setInt(i + 2, msgDataIndexList.get(i));
}
updatedCount = stmtUpdate.executeUpdate();
if (updatedCount != msgDataIndexList.size()) {
stderr_println(LOG_DEBUG, "[!] Number of updated rows does not match number of selected rows.");
}
} catch (Exception e) {
stderr_println(LOG_ERROR, String.format("[-] Error updating Req Data Status: %s", e.getMessage()));
}
return updatedCount;
}
/**
* 当达到某个状态条件时 更新 msgHash 对应数据 的状态
*/
public static synchronized int updateStatusWhenStatusByMsgHash(String tableName, String msgHash, String updateStatus, String whenStatus) {
int updatedCount = -1;
String updateSQL = "UPDATE " + tableName + " SET run_status = ? WHERE run_status = ? and msg_hash = ?;";
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmtUpdate = conn.prepareStatement(updateSQL)) {
stmtUpdate.setString(1, updateStatus);
stmtUpdate.setString(2, whenStatus);
stmtUpdate.setString(3, msgHash);
updatedCount = stmtUpdate.executeUpdate();
} catch (Exception e) {
stderr_println(LOG_ERROR, String.format("[-] updateStatusWhenStatusByMsgHash: [%s] Error->: %s",tableName, e.getMessage()));
}
return updatedCount;
}
/**
* 当达到某个状态条件时 更新 root_url 对应数据 的状态
*/
public static synchronized int updateStatusWhenStatusByRootUrl(String tableName, String rootUrl, String updateStatus, String whenStatus) {
int updatedCount = -1;
String updateSQL = "UPDATE " + tableName + " SET run_status = ? WHERE run_status = ? and root_url = ?;";
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmtUpdate = conn.prepareStatement(updateSQL)) {
stmtUpdate.setString(1, updateStatus);
stmtUpdate.setString(2, whenStatus);
stmtUpdate.setString(3, rootUrl);
updatedCount = stmtUpdate.executeUpdate();
} catch (Exception e) {
stderr_println(LOG_ERROR, String.format("[-] updateStatusWhenStatusByRootUrl: [%s] Error->: %s",tableName, e.getMessage()));
}
return updatedCount;
}
}
================================================
FILE: src/main/java/sqlUtils/Constants.java
================================================
package sqlUtils;
public class Constants {
public static final String ANALYSE_WAIT = "Waiting"; //等待自动处理
public static final String ANALYSE_ING = "Analysing"; //自动处理中
public static final String ANALYSE_END = "Analysed"; //自动处理完毕
public static final String HANDLE_WAIT = "Pending"; //等待手动处理
public static final String HANDLE_ING = "Handling"; //手动处理中
public static final String HANDLE_END = "Handled"; //手动处理完毕
public static final String SPLIT_SYMBOL = "<->";
public static final String RULE_CONF_PREFIX = "CONF_"; //配置文件中 配置规则的开头
}
================================================
FILE: src/main/java/sqlUtils/UnionTableSql.java
================================================
package sqlUtils;
import database.AnalyseHostResultTable;
import database.DBService;
import database.PathTreeTable;
import model.FindPathModel;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.List;
import static utils.BurpPrintUtils.*;
public class UnionTableSql {
//联合 获取一条需要更新的Path数据
public static synchronized List fetchHostTableNeedUpdatePathDataList(int limit){
List findPathModels = new ArrayList<>();
// 首先选取一条记录的ID 状态是已经分析完毕,并且 当前 PathTree 的 基本路径数量 大于 生成分析数据时的 基本路径数量
String selectSQL = ("SELECT A.id, A.root_url, A.find_path " +
"From $tableName1$ A LEFT JOIN $tableName2$ B ON A.root_url = B.root_url " +
"WHERE B.basic_path_num > A.basic_path_num Limit ?;")
.replace("$tableName1$", AnalyseHostResultTable.tableName)
.replace("$tableName2$", PathTreeTable.tableName);
try (Connection conn = DBService.getInstance().getNewConn(); PreparedStatement stmt = conn.prepareStatement(selectSQL)) {
stmt.setInt(1, limit);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
FindPathModel findPathModel = new FindPathModel(
rs.getInt("id"),
rs.getString("root_url"),
rs.getString("find_path")
);
findPathModels.add(findPathModel);
}
}
} catch (Exception e) {
stderr_println(LOG_ERROR, String.format("[-] Error fetch Need Update Path Data List: %s", e.getMessage()));
}
return findPathModels;
}
}
================================================
FILE: src/main/java/sqlUtils/buildSQL.java
================================================
package sqlUtils;
public class buildSQL {
/**
* 构建一个函数,实现根据参数列表数量自动拼接 IN (?,?,?)语句
* @param size
* @return
*/
public static String buildInParamList(int size) {
StringBuilder inParameterList = new StringBuilder(" (");
for (int i = 0; i < size; i++) {
inParameterList.append("?");
if (i < size - 1) {
inParameterList.append(", ");
}
}
inParameterList.append(") ");
return inParameterList.toString();
}
}
================================================
FILE: src/main/java/ui/BasicHostConfigPanel.java
================================================
package ui;
import burp.BurpExtender;
import burp.IProxyScanner;
import utils.UiUtils;
import javax.swing.*;
import java.awt.*;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.event.MouseAdapter;
import java.awt.event.MouseEvent;
import static utils.BurpPrintUtils.*;
public class BasicHostConfigPanel extends JPanel {
public static JLabel lbRequestCountOnHost; //记录所有加入到URL的请求
public static JLabel lbTaskerCountOnHost; //记录所有加入数据库的请求
public static JLabel lbAnalysisEndCountOnHost; //记录所有已经分析完成的结果数量
private static JComboBox choicesComboBoxOnHost; //数据表显示快速选择框
private static JTextField urlSearchBoxOnHost; //URl搜索框
public static JToggleButton autoRefreshUiButtonOnHost; //自动刷新开关按钮状态
public static int timerDelayOnHost = 15; //定时器刷新间隔,单位秒
//用于两端联动使用
public static JToggleButton proxyListenButtonOnHost;
public static JToggleButton autoRecordPathButtonOnHost; //自动保存响应状态码合适的URL 目前过滤功能不完善,只能手动开启
public static JToggleButton dynamicPathFilterButtonOnHost;
public static JToggleButton autoPathsToUrlsButtonOnHost;
public static JToggleButton autoRefreshUnvisitedButtonOnHost;
public static JToggleButton autoRecursiveButtonOnHost;
public static JToggleButton forceDecodeUnicodeButtonOnHost;
public BasicHostConfigPanel() {
GridBagLayout gridBagLayout = new GridBagLayout();
//GridBagLayout 允许以网格形式布局容器中的组件,同时为每个组件提供独立的定位和大小控制,非常适用于需要复杂布局设计的GUI界面。
// 列数,行数 //表示容器被划分为两列,每一列的初始宽度均为0。
// 这里的0不代表实际宽度为零,而是告诉布局管理器根据组件的实际大小和其他约束(如权重)来计算列宽。
gridBagLayout.columnWidths = new int[] { 0, 0};
gridBagLayout.rowHeights = new int[] {5};
// 各列占宽度比,各行占高度比
gridBagLayout.columnWeights = new double[] { 1.0D, Double.MIN_VALUE };
//设置了两列的扩展权重。第一列的权重为1.0,意味着当容器有多余空间时,这一列会优先扩展以填充可用空间。
// 第二列的权重设为Double.MIN_VALUE,表示这一列不应该扩展,保持最小或固定大小。
setLayout(gridBagLayout);
//创建FilterPanel
JPanel FilterPanel = new JPanel();
GridBagConstraints gbc_panel_1 = new GridBagConstraints();
gbc_panel_1.insets = new Insets(0, 5, 5, 5);
gbc_panel_1.fill = 2;
gbc_panel_1.gridx = 0;
gbc_panel_1.gridy = 2;
add(FilterPanel, gbc_panel_1);
//设置一个名为FilterPanel的面板在父容器中的布局位置
// 布局约束包括:
//insets: 设置了组件边缘的内边距,上5px,左5px,下5px,右5px,为组件提供一定的间距。
//fill: 设置组件在可扩展空间中的填充方式,值为2表示BOTH,即组件可以在水平和垂直方向上填充其显示区域。
//gridx 和 gridy: 分别设置组件在网格布局中的起始列和起始行,这里是第0列第2行。
//为 FilterPanel 设置布局
GridBagLayout gbl_panel_1 = new GridBagLayout();
gbl_panel_1.columnWidths = new int[] { 0, 0, 0, 0, 0 };//设置每列的初始宽度为0 指示布局管理器根据组件实际大小和其他约束来计算宽度。
gbl_panel_1.rowHeights = new int[] { 0, 0 }; //设置每行的初始高度为0,指按需计算行高。
// 指定每列的扩展权重。这里前9列的权重都设为0.0,意味着这些列不会随容器大小变化而扩展,
// 而最后列的权重设为Double.MIN_VALUE,这通常用于指示该列应该尽可能小,不参与额外空间的分配。
//gbl_panel_1.columnWeights = new double[] { 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, 0.0D, Double.MIN_VALUE};
//第一行权重为0.0,不随容器扩展,第二行的权重为Double.MIN_VALUE,表示该行也不扩展。
gbl_panel_1.rowWeights = new double[] { 0.0D, Double.MIN_VALUE };
FilterPanel.setLayout(gbl_panel_1);
// 在添加 "Requests Total" 和 lbRequestCount 之前添加一个占位组件
Component leftStrut = Box.createHorizontalStrut(5); // 你可以根据需要调整这个值
GridBagConstraints gbc_leftStrut = new GridBagConstraints();
gbc_leftStrut.insets = new Insets(0, 0, 0, 5);
gbc_leftStrut.fill = GridBagConstraints.HORIZONTAL;
gbc_leftStrut.weightx = 1.0; // 这个值决定了 leftStrut 占据的空间大小
gbc_leftStrut.gridx = 6;
gbc_leftStrut.gridy = 0;
FilterPanel.add(leftStrut, gbc_leftStrut);
// 转发url总数,默认0
JLabel lbRequest = new JLabel("Requests Total:");
GridBagConstraints gbc_lbRequest = new GridBagConstraints();
gbc_lbRequest.insets = new Insets(0, 0, 0, 5);
gbc_lbRequest.fill = GridBagConstraints.HORIZONTAL;
gbc_lbRequest.weightx = 0.0;
gbc_lbRequest.gridx = 0;
gbc_lbRequest.gridy = 0;
FilterPanel.add(lbRequest, gbc_lbRequest);
lbRequestCountOnHost = new JLabel("0");
lbRequestCountOnHost.setForeground(new Color(0,0,255));
GridBagConstraints gbc_lbRequestCount = new GridBagConstraints();
gbc_lbRequestCount.insets = new Insets(0, 0, 0, 5);
gbc_lbRequestCount.fill = GridBagConstraints.HORIZONTAL;
gbc_lbRequestCount.weightx = 0.0;
gbc_lbRequestCount.gridx = 1;
gbc_lbRequestCount.gridy = 0;
FilterPanel.add(lbRequestCountOnHost, gbc_lbRequestCount);
// 转发成功url数,默认0
JLabel lbTasker = new JLabel("Tasker Total:");
GridBagConstraints gbc_lbTasker = new GridBagConstraints();
gbc_lbTasker.insets = new Insets(0, 0, 0, 5);
gbc_lbTasker.fill = 0;
gbc_lbTasker.gridx = 2;
gbc_lbTasker.gridy = 0;
FilterPanel.add(lbTasker, gbc_lbTasker);
lbTaskerCountOnHost = new JLabel("0");
lbTaskerCountOnHost.setForeground(new Color(0, 255, 0));
GridBagConstraints gbc_lbTaskerCount = new GridBagConstraints();
gbc_lbTaskerCount.insets = new Insets(0, 0, 0, 5);
gbc_lbTaskerCount.fill = 0;
gbc_lbTaskerCount.gridx = 3;
gbc_lbTaskerCount.gridy = 0;
FilterPanel.add(lbTaskerCountOnHost, gbc_lbTaskerCount);
// 分析URL的数量
JLabel lbAnalysisEnd = new JLabel("Analysis End:");
GridBagConstraints gbc_lbAnalysisEnd = new GridBagConstraints();
gbc_lbAnalysisEnd.insets = new Insets(0, 0, 0, 5);
gbc_lbAnalysisEnd.fill = 0;
gbc_lbAnalysisEnd.gridx = 4;
gbc_lbAnalysisEnd.gridy = 0;
FilterPanel.add(lbAnalysisEnd, gbc_lbAnalysisEnd);
lbAnalysisEndCountOnHost = new JLabel("0");
lbAnalysisEndCountOnHost.setForeground(new Color(0, 0, 255)); // 蓝色
GridBagConstraints gbc_lbAnalysisEndCount = new GridBagConstraints();
gbc_lbAnalysisEndCount.insets = new Insets(0, 0, 0, 5);
gbc_lbAnalysisEndCount.fill = 0;
gbc_lbAnalysisEndCount.gridx = 5;
gbc_lbAnalysisEndCount.gridy = 0;
FilterPanel.add(lbAnalysisEndCountOnHost, gbc_lbAnalysisEndCount);
// 添加填充以在左侧占位
Component horizontalBlank = Box.createHorizontalGlue(); //创建一个水平组件
GridBagConstraints gbc_leftFiller = new GridBagConstraints();
gbc_leftFiller.weightx = 1; // 使得这个组件吸收额外的水平空间
gbc_leftFiller.gridx = 6; // 位置设置为第一个单元格
gbc_leftFiller.gridy = 0; // 第一行
gbc_leftFiller.fill = GridBagConstraints.HORIZONTAL; // 水平填充
FilterPanel.add(horizontalBlank, gbc_leftFiller);
// 开关 是否开启代理流量监听 //自动保存响应状态码合适的URL 目前过滤功能不完善,只能手动开启
proxyListenButtonOnHost = UiUtils.getToggleButtonByDefaultValue(BurpExtender.proxyListenIsOpenDefault);
proxyListenButtonOnHost.setToolTipText("Proxy模块流量监听开关");
proxyListenButtonOnHost.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
//默认开启本功能, 点击后应该关闭配置 //默认关闭本功能, 点击后应该开启配置
boolean selected = proxyListenButtonOnHost.isSelected();
IProxyScanner.proxyListenIsOpen = BurpExtender.proxyListenIsOpenDefault ? !selected : selected;
stdout_println(LOG_DEBUG, String.format("proxyListenIsOpen: %s", IProxyScanner.proxyListenIsOpen));
BasicUrlConfigPanel.proxyListenButtonOnUrl.setSelected(selected); //联动更新URL面板的情况
}
});
// 刷新按钮按钮
JToggleButton clickRefreshButtonOnHost = new JToggleButton(UiUtils.getImageIcon("/icon/refreshButton2.png", 24, 24));
clickRefreshButtonOnHost.setPreferredSize(new Dimension(30, 30));
clickRefreshButtonOnHost.setBorder(null); // 设置无边框
clickRefreshButtonOnHost.setFocusPainted(false); // 移除焦点边框
clickRefreshButtonOnHost.setContentAreaFilled(false); // 移除选中状态下的背景填充
clickRefreshButtonOnHost.setToolTipText("点击强制刷新表格");
// 手动刷新按钮监听事件
clickRefreshButtonOnHost.addActionListener(new ActionListener() {
private boolean canClick = true;
@Override
public void actionPerformed(ActionEvent e) {
if (canClick) {
canClick = false;
ImageIcon originalIcon = (ImageIcon) clickRefreshButtonOnHost.getIcon(); // 保存原始图标
String originalTip = clickRefreshButtonOnHost.getToolTipText(); // 保存原始批注
// 更换为新图标
clickRefreshButtonOnHost.setIcon(UiUtils.getImageIcon("/icon/runningButton.png", 24, 24)); // 立即显示新图标
try{
// 调用更新未访问URL列的数据
BasicHostInfoPanel.getInstance().updateUnVisitedUrlsByRootUrls(null);
// 调用刷新表格的方法
BasicHostInfoPanel.getInstance().refreshBasicHostTableModel();
//建议JVM清理内存
System.gc();
} catch (Exception ep){
stderr_println(LOG_ERROR, String.format("[!] 刷新表格发生错误:%s", ep.getMessage()) );
}
// 设置定时器,5秒后允许再次点击并恢复图标
Timer timer = new Timer(3000, new ActionListener() {
@Override
public void actionPerformed(ActionEvent ae) {
canClick = true;
clickRefreshButtonOnHost.setIcon(originalIcon); // 恢复原始图标
clickRefreshButtonOnHost.setToolTipText(originalTip); // 恢复原始批注
}
});
timer.setRepeats(false);
timer.start();
}
}
});
// 开关 是否开启自动记录PATH
autoRecordPathButtonOnHost = UiUtils.getToggleButtonByDefaultValue(BurpExtender.autoRecordPathIsOpenDefault);
autoRecordPathButtonOnHost.setToolTipText("自动保存有效请求PATH");
autoRecordPathButtonOnHost.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
//默认开启本功能, 点击后应该关闭配置 //默认关闭本功能, 点击后应该开启配置
boolean selected = autoRecordPathButtonOnHost.isSelected();
IProxyScanner.autoRecordPathIsOpen = BurpExtender.autoRecordPathIsOpenDefault ? !selected : selected;
stdout_println(LOG_DEBUG, String.format("autoRecordPathIsOpen: %s", IProxyScanner.autoRecordPathIsOpen));
BasicUrlConfigPanel.autoRecordPathButtonOnUrl.setSelected(selected); //联动更新URL面板的情况
}
});
// 开关 是否开启复杂的动态PATH过滤
dynamicPathFilterButtonOnHost = UiUtils.getToggleButtonByDefaultValue(BurpExtender.dynamicPathFilterIsOpenDefault);
dynamicPathFilterButtonOnHost.setToolTipText("开启智能响应过滤(访问随机URL获取目标的404页面的条件)");
dynamicPathFilterButtonOnHost.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
//默认开启本功能, 点击后应该关闭配置 //默认关闭本功能, 点击后应该开启配置
boolean selected = dynamicPathFilterButtonOnHost.isSelected();
IProxyScanner.dynamicPathFilterIsOpen = BurpExtender.dynamicPathFilterIsOpenDefault ? !selected : selected;
stdout_println(LOG_DEBUG, String.format("dynamicPathFilterIsOpen: %s", IProxyScanner.dynamicPathFilterIsOpen));
BasicUrlConfigPanel.dynamicPathFilterButtonOnUrl.setSelected(selected); //联动更新URL面板的情况
}
});
autoPathsToUrlsButtonOnHost = UiUtils.getToggleButtonByDefaultValue(BurpExtender.autoPathsToUrlsIsOpenDefault);
autoPathsToUrlsButtonOnHost.setToolTipText("自动基于PathTree结合FindPath生成URL");
autoPathsToUrlsButtonOnHost.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
//默认开启本功能, 点击后应该关闭配置 //默认关闭本功能, 点击后应该开启配置
boolean selected = autoPathsToUrlsButtonOnHost.isSelected();
IProxyScanner.autoPathsToUrlsIsOpen = BurpExtender.autoPathsToUrlsIsOpenDefault ? !selected : selected;
stdout_println(LOG_DEBUG, String.format("autoPathsToUrlsIsOpen: %s", IProxyScanner.autoPathsToUrlsIsOpen));
BasicUrlConfigPanel.autoPathsToUrlsButtonOnUrl.setSelected(selected); //联动更新URL面板的情况
}
});
// 开关 是否开启自动刷新未访问URL
autoRefreshUnvisitedButtonOnHost = UiUtils.getToggleButtonByDefaultValue(BurpExtender.autoRefreshUnvisitedIsOpenDefault);
autoRefreshUnvisitedButtonOnHost.setToolTipText("自动刷新未访问URL");
autoRefreshUnvisitedButtonOnHost.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
//默认开启本功能, 点击后应该关闭配置 //默认关闭本功能, 点击后应该开启配置
boolean selected = autoRefreshUnvisitedButtonOnHost.isSelected();
//默认开启本功能, 点击后应该关闭配置 //默认关闭本功能, 点击后应该开启配置
IProxyScanner.autoRefreshUnvisitedIsOpen = BurpExtender.autoRefreshUnvisitedIsOpenDefault ? !selected : selected;
stdout_println(LOG_DEBUG, String.format("autoRefreshUnvisitedIsOpen: %s", IProxyScanner.autoRefreshUnvisitedIsOpen));
BasicUrlConfigPanel.autoRefreshUnvisitedButtonOnUrl.setSelected(selected); //联动更新URL面板的情况
}
});
// 开关 是否开启对提取URL进行发起请求
autoRecursiveButtonOnHost = UiUtils.getToggleButtonByDefaultValue(BurpExtender.autoRecursiveIsOpenDefault);
autoRecursiveButtonOnHost.setToolTipText("自动测试未访问URL");
autoRecursiveButtonOnHost.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
//默认开启本功能, 点击后应该关闭配置 //默认关闭本功能, 点击后应该开启配置
boolean selected = autoRecursiveButtonOnHost.isSelected();
IProxyScanner.autoRecursiveIsOpen = BurpExtender.autoRecursiveIsOpenDefault ? !selected : selected;
stdout_println(LOG_DEBUG, String.format("autoRecursiveIsOpen: %s", IProxyScanner.autoRecursiveIsOpen));
BasicUrlConfigPanel.autoRecursiveButtonOnUrl.setSelected(selected); //联动更新URL面板的情况
}
});
//刷新按钮
autoRefreshUiButtonOnHost = UiUtils.getToggleButtonByDefaultValue(BurpExtender.autoRefreshUiIsOpenDefault);
autoRefreshUiButtonOnHost.setToolTipText(String.format("每[%s]秒刷新表格", timerDelayOnHost));
// 自动刷新按钮监听事件
autoRefreshUiButtonOnHost.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
//默认开启本功能, 点击后应该关闭配置 //默认关闭本功能, 点击后应该开启配置
boolean selected = autoRefreshUiButtonOnHost.isSelected();
IProxyScanner.autoRefreshUiIsOpen = BurpExtender.autoRefreshUiIsOpenDefault ? !selected : selected;
stdout_println(LOG_DEBUG, String.format("autoRefreshUiIsOpen: %s", IProxyScanner.autoRefreshUiIsOpen));
BasicUrlConfigPanel.autoRefreshUiButtonOnUrl.setSelected(selected); //联动更新URL面板的情况
//根据当前刷新开关状态配置定时器暂停或者重启
UiUtils.setAutoRefreshUiByButton(IProxyScanner.autoRefreshUiIsOpen);
}
});
// 开关 是否开启对提取URL进行发起请求
forceDecodeUnicodeButtonOnHost = UiUtils.getToggleButtonByDefaultValue(BurpExtender.forceDecodeUnicodeDefault);
forceDecodeUnicodeButtonOnHost.setToolTipText("强制解码响应Unicode 默认只对Json响应解码");
forceDecodeUnicodeButtonOnHost.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
//默认开启本功能, 点击后应该关闭配置 //默认关闭本功能, 点击后应该开启配置
boolean selected = forceDecodeUnicodeButtonOnHost.isSelected();
IProxyScanner.forceDecodeUnicode = BurpExtender.forceDecodeUnicodeDefault ? !selected : selected;
stdout_println(LOG_DEBUG, String.format("forceDecodeUnicode: %s", IProxyScanner.forceDecodeUnicode));
BasicUrlConfigPanel.forceDecodeUnicodeButtonOnUrl.setSelected(selected); //联动更新URL面板的情况
}
});
// 设置按钮的 GridBagConstraints
GridBagConstraints gbc_buttons = new GridBagConstraints();
gbc_buttons.insets = new Insets(0, 5, 0, 5);
gbc_buttons.gridy = 0; // 设置按钮的纵坐标位置
gbc_buttons.fill = GridBagConstraints.NONE; // 不填充
// 点击按钮 点击后刷新数据 含未访问数据
gbc_buttons.gridx = 7; // 设置按钮的横坐标位置
FilterPanel.add(proxyListenButtonOnHost, gbc_buttons);
gbc_buttons.gridx = 8; // 设置按钮的横坐标位置
FilterPanel.add(forceDecodeUnicodeButtonOnHost, gbc_buttons);
// 自动记录有效的PATH到path表中 功能开关
gbc_buttons.gridx = 9; // 设置按钮的横坐标位置
FilterPanel.add(autoRecordPathButtonOnHost, gbc_buttons);
// 高级动态有效路径过滤 功能开关
gbc_buttons.gridx = 10;
FilterPanel.add(dynamicPathFilterButtonOnHost, gbc_buttons);
// 高级动态有效路径过滤 功能开关
gbc_buttons.gridx = 11;
FilterPanel.add(autoPathsToUrlsButtonOnHost, gbc_buttons);
// 自动刷新 未访问URL列表
gbc_buttons.gridx = 12; // 设置按钮的横坐标位置
FilterPanel.add(autoRefreshUnvisitedButtonOnHost, gbc_buttons);
// 自动递归 开关
gbc_buttons.gridx = 13; // 设置按钮的横坐标位置
FilterPanel.add(autoRecursiveButtonOnHost, gbc_buttons);
// 定时刷新按钮
gbc_buttons.gridx = 14; // 将横坐标位置移动到下一个单元格
FilterPanel.add(autoRefreshUiButtonOnHost, gbc_buttons);
// 点击按钮 点击后刷新数据 含未访问数据
gbc_buttons.gridx = 15; // 设置按钮的横坐标位置
FilterPanel.add(clickRefreshButtonOnHost, gbc_buttons);
// 添加填充以在右侧占位
GridBagConstraints gbc_rightFiller = new GridBagConstraints();
gbc_rightFiller.weightx = 1; // 使得这个组件吸收额外的水平空间
gbc_rightFiller.gridx = 16; // 位置设置为最后一个单元格
gbc_rightFiller.gridy = 0; // 第一行
gbc_rightFiller.fill = GridBagConstraints.HORIZONTAL; // 水平填充
FilterPanel.add(horizontalBlank, gbc_rightFiller);
// 全部按钮
choicesComboBoxOnHost = new JComboBox<>(new String[]{
"显示有效内容",
"待处理有效内容",
"显示敏感内容",
"待处理敏感内容",
"显示未访问路径",
"显示全部内容",
"显示无效内容",
});
GridBagConstraints gbc_btnall = new GridBagConstraints();
gbc_btnall.insets = new Insets(0, 0, 0, 5);
gbc_btnall.fill = 0;
gbc_btnall.gridx = 17; // 根据该值来确定是确定从左到右的顺序
gbc_btnall.gridy = 0;
FilterPanel.add(choicesComboBoxOnHost, gbc_btnall);
// 检索框
urlSearchBoxOnHost = new JTextField(15);
GridBagConstraints gbc_btnSearchField = new GridBagConstraints();
gbc_btnSearchField.insets = new Insets(0, 0, 0, 5);
gbc_btnSearchField.fill = 0;
gbc_btnSearchField.gridx = 18; // 根据该值来确定是确定从左到右的顺序
gbc_btnSearchField.gridy = 0;
urlSearchBoxOnHost.setToolTipText("搜索URL关键字");
FilterPanel.add(urlSearchBoxOnHost, gbc_btnSearchField);
// 检索按钮
JButton searchButton = new JButton();
searchButton.setIcon(UiUtils.getImageIcon("/icon/searchButton.png"));
searchButton.setToolTipText("点击搜索");
GridBagConstraints gbc_btnSearch = new GridBagConstraints();
gbc_btnSearch.insets = new Insets(0, 0, 0, 5);
gbc_btnSearch.fill = 0;
gbc_btnSearch.gridx = 19; // 根据该值来确定是确定从左到右的顺序
gbc_btnSearch.gridy = 0;
FilterPanel.add(searchButton, gbc_btnSearch);
// 功能按钮
JButton moreButton = new JButton();
moreButton.setToolTipText("更多功能 ");
moreButton.setIcon(UiUtils.getImageIcon("/icon/moreButton.png", 17, 17));
GridBagConstraints gbc_btnMore = new GridBagConstraints();
gbc_btnMore.insets = new Insets(0, 0, 0, 5);
gbc_btnMore.fill = 0;
gbc_btnMore.gridx = 20; // 根据该值来确定是确定从左到右的顺序
gbc_btnMore.gridy = 0;
FilterPanel.add(moreButton, gbc_btnMore);
// 快速选择框的监听事件
choicesComboBoxOnHost.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
try{
// 触发显示所有行事件
String searchText = urlSearchBoxOnHost.getText();
if(searchText.isEmpty()){
searchText = "";
}
String selectedOption = (String) choicesComboBoxOnHost.getSelectedItem();
BasicHostInfoPanel.showDataHostTableByFilter(selectedOption, searchText);
} catch (Exception ex) {
stderr_println(String.format("[!] choicesComboBoxOnHost: %s", ex.getMessage()));
}
}
});
// 检索按钮事件监听器
searchButton.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
String searchText = urlSearchBoxOnHost.getText();
String selectedOption = (String) BasicHostConfigPanel.choicesComboBoxOnHost.getSelectedItem();
BasicHostInfoPanel.showDataHostTableByFilter(selectedOption, searchText);
}
});
//搜索框的回车事件
urlSearchBoxOnHost.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
String searchText = urlSearchBoxOnHost.getText();
String selectedOption = (String) BasicHostConfigPanel.choicesComboBoxOnHost.getSelectedItem();
BasicHostInfoPanel.showDataHostTableByFilter(selectedOption, searchText);
}
});
// 功能按钮 弹出选项
JPopupMenu moreMenu = UiUtils.createMoreMenuWithAction();
// 点击”功能“的监听事件
moreButton.addMouseListener(new MouseAdapter() {
public void mouseClicked(MouseEvent e) {
moreMenu.show(e.getComponent(), e.getX(), e.getY());
}
});
}
public static String getUrlSearchBoxTextOnHost() {
return urlSearchBoxOnHost.getText();
}
public static void setUrlSearchBoxTextOnHost(String string) {
urlSearchBoxOnHost.setText(string);
}
public static String getComboBoxSelectedOptionOnHost() {
return (String) BasicHostConfigPanel.choicesComboBoxOnHost.getSelectedItem();
}
}
================================================
FILE: src/main/java/ui/BasicHostInfoPanel.java
================================================
package ui;
import burp.*;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONObject;
import com.alibaba.fastjson2.JSONWriter;
import database.*;
import model.*;
import sqlUtils.CommonDeleteLine;
import sqlUtils.CommonFetchData;
import sqlUtils.CommonUpdateStatus;
import sqlUtils.Constants;
import ui.MainTabRender.RunStatusCellRenderer;
import ui.MainTabRender.TableHeaderWithTips;
import ui.MainTabRender.HasImportantCellRenderer;
import utils.CastUtils;
import utils.PathTreeUtils;
import utils.RespHashUtils;
import utils.UiUtils;
import javax.swing.Timer;
import javax.swing.*;
import javax.swing.border.EmptyBorder;
import javax.swing.table.DefaultTableModel;
import java.awt.*;
import java.awt.event.*;
import java.util.List;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import static utils.BurpPrintUtils.*;
import static utils.CastUtils.isEmptyObj;
import static utils.CastUtils.isNotEmptyObj;
public class BasicHostInfoPanel extends JPanel {
private static volatile BasicHostInfoPanel instance; //实现单例模式
private static JTable basicHostMsgTableUI; //表格UI
private static DefaultTableModel basicHostMsgTableModel; // 存储表格数据
private static JEditorPane basicHostFindInfoTextPane; //显示 响应中直接提取的敏感信息【支持颜色】
private static ITextEditor basicHostRespFindUrlTEditor; //显示 响应中直接提取的URL
private static ITextEditor basicHostRespFindPathTEditor; //显示 响应中直接提取的PATH
private static ITextEditor basicHostDirectPath2UrlTEditor; //显示 基于PATH直接拼接计算出的URL
private static ITextEditor basicHostSmartPath2UrlTEditor; //显示 基于树算法+PATH提取结果计算出的URL
private static ITextEditor basicHostUnvisitedUrlTEditor; //显示 目前提取URL结果中未访问过的URL
private static JEditorPane basicHostAllUrlStatusTEditor; //显示 所有当前RootUrls的URL访问记录
private static ITextEditor basicHostPathTreeTEditor; //当前目标的路径树信息
private static Timer basicHostTimer; //定时器 为线程调度提供了一个简单的时间触发机制,广泛应用于需要定时执行某些操作的场景,
public static BasicHostInfoPanel getInstance() {
if (instance == null) {
synchronized (BasicHostInfoPanel.class) {
if (instance == null) {
instance = new BasicHostInfoPanel();
}
}
}
return instance;
}
public BasicHostInfoPanel() {
// EmptyBorder 四周各有了5像素的空白边距
setBorder(new EmptyBorder(5, 5, 5, 5));
////BorderLayout 将容器分为五个区域:北 南 东 西 中 每个区域可以放置一个组件,
setLayout(new BorderLayout(0, 0));
// 主分隔面板
// JSplitPane可以包含两个(或更多)子组件,允许用户通过拖动分隔条来改变两个子组件的相对大小。
JSplitPane basicHostMainSplitPane = new JSplitPane(JSplitPane.VERTICAL_SPLIT);
// 首行配置面板
BasicHostConfigPanel basicHostConfigPanel = new BasicHostConfigPanel();
// 数据表格
initBasicHostDataTableUI();
//将包含table的滚动面板的upScrollPane 设置为另一个组件mainSplitPane的上半部分。
basicHostMainSplitPane.setTopComponent(new JScrollPane(basicHostMsgTableUI));
//获取下方的消息面板
JTabbedPane basicHostMsgTabs = getBasicHostMsgTabs();
basicHostMainSplitPane.setBottomComponent(basicHostMsgTabs);
//组合最终的内容面板
add(basicHostConfigPanel, BorderLayout.NORTH);
add(basicHostMainSplitPane, BorderLayout.CENTER);
//初始化表格数据
initBasicHostDataTableUIData(basicHostMsgTableModel);
// 初始化定时刷新页面函数 单位是毫秒
stopTimerBasicHost();
startTimerBasicHost();
}
/**
* 查询 TableLineDataModelBasicHostSQL 初始化 table 数据
*/
private void initBasicHostDataTableUIData(DefaultTableModel tableModel) {
SwingUtilities.invokeLater(new Runnable() {
public void run() {
//获取所有数据 查询 HOST信息表
ArrayList allReqAnalyseData = TableLineDataModelBasicHostSQL.fetchHostTableLineAll();
//将数据赋值给表模型
basicHostPopulateModelFromList(tableModel, allReqAnalyseData);
}
});
}
/**
* 把 jsonArray 赋值到 model 中
* @param model
* @param arrayList
*/
private void basicHostPopulateModelFromList(DefaultTableModel model, ArrayList arrayList) {
if (isEmptyObj(arrayList)) return;
Iterator iterator = arrayList.iterator();
while (iterator.hasNext()) {
BasicHostTableLineDataModel apiDataModel = iterator.next();
Object[] rowData = apiDataModel.toRowDataArray();
model.addRow(rowData);
}
//刷新表数据模型
model.fireTableDataChanged();
}
/**
* 初始化Table
*/
private void initBasicHostDataTableUI() {
// 数据展示面板
basicHostMsgTableModel = new DefaultTableModel(new Object[]{
"id",
"root_url",
"host",
"domain",
"important",
"find_info",
"find_url",
"find_path",
"find_api",
"path_url",
"unvisited",
"all_url",
"basic_num",
"run_status"
}, 0) {
@Override
public boolean isCellEditable(int row, int column) {
//在数据模型层面禁止编辑行数据
return false;
}
};
basicHostMsgTableUI = UiUtils.creatTableUiWithTips(basicHostMsgTableModel);
// 设置列选中模式
int listSelectionModel = ListSelectionModel.MULTIPLE_INTERVAL_SELECTION;
basicHostMsgTableUI.setSelectionMode(listSelectionModel);
//自己实现TableHeader 支持请求头提示
String[] basicHostColHeaderTooltips = new String[]{
"【请求ID】",
"【请求目标】",
"【HOST信息】",
"【根域名信息】",
"【是否重要信息】",
"【敏感信息数量】 == 当前网站响应中的敏感信息",
"【直接URL数量】 == 当前网站响应中提取的URL",
"【网站PATH数量】 == 当前网站响应中提取的PATH",
"【拼接URL数量】 == 当前请求目录 直接组合 已提取PATH(已过滤)",
"【动态URL数量】 == 网站有效目录 智能组合 已提取PATH(已过滤|只能计算带目录的PATH|跟随网站有效目录新增而变动)",
"【未访问URL数量】 == 当前直接URL数量+拼接URL数量+动态URL数量-全局已访问URL",
"【所有提取URL数量】 == 当前直接URL+PATH直接组合URL+PATH动态组合URL",
"【动态URL计算基准】(表明动态URL基于多少个网站路径计算|跟随网站有效目录新增而变动)",
"【请求上下文分析状态】(不为 Waiting 表示已提取[敏感信息|URL信息|PATH信息])"
};
TableHeaderWithTips basicHostTableHeader = new TableHeaderWithTips(basicHostMsgTableUI.getColumnModel(), basicHostColHeaderTooltips);
basicHostMsgTableUI.setTableHeader(basicHostTableHeader);
//添加表头排序功能
UiUtils.tableAddActionSortByHeader(basicHostMsgTableUI, basicHostMsgTableModel);
//设置数据表的宽度
UiUtils.tableSetColumnMaxWidth(basicHostMsgTableUI, 0, 50);
UiUtils.tableSetColumnMinWidth(basicHostMsgTableUI, 1, 200);
UiUtils.tableSetColumnMinWidth(basicHostMsgTableUI, 2, 100);
//设置表格每列的对齐设置
List leftColumns = Arrays.asList(1);
UiUtils.tableSetColumnsAlignRender(basicHostMsgTableUI, leftColumns);
//为重要信息列添加额外的渲染
HasImportantCellRenderer havingImportantRenderer = new HasImportantCellRenderer();
int ImportantColumnIndex = 3; //重要信息列所在的列号减1
basicHostMsgTableUI.getColumnModel().getColumn(ImportantColumnIndex).setCellRenderer(havingImportantRenderer);
//为状态信息列添加额外的渲染 在最后一列,可以设置为动态值
RunStatusCellRenderer runStatusCellRenderer = new RunStatusCellRenderer();
int runStatusColumnIndex = basicHostMsgTableUI.getColumnCount() - 1;
basicHostMsgTableUI.getColumnModel().getColumn(runStatusColumnIndex).setCellRenderer(runStatusCellRenderer);
//为表格添加点击显示下方的消息动作
basicHostTableAddActionSetMsgTabData();
//为表的每一行添加右键菜单
basicHostTableAddRightClickMenu(basicHostMsgTableUI, listSelectionModel);
}
/**
* 初始化任务定时器 定时刷新UI内容
* @return
*/
public static void initTimerBasicHost() {
// 确保在重新初始化之前停止旧的定时器
int delay = BasicHostConfigPanel.timerDelayOnHost * 1000;
basicHostTimer = new Timer(delay, new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
//当定时自动刷新URL功能是开启时进行操作
if (IProxyScanner.autoRefreshUiIsOpen && basicHostTimer.isRunning()) {
// 调用刷新表格的方法
try{
stdout_println(LOG_DEBUG, String.format("[*] Timer Refresh UI Basic Host On [%s]", delay));
//仅当开启了 自动刷新未访问URL 时调用 更新未访问URL列的数据
if (IProxyScanner.autoRefreshUnvisitedIsOpen){
BasicHostInfoPanel.getInstance().updateUnVisitedUrlsByRootUrls(null);
}
// 调用刷新表格的方法
BasicHostInfoPanel.getInstance().refreshBasicHostTableModel();
//建议JVM清理内存
System.gc();
//提示自动刷新表格完成
} catch (Exception exception){
stderr_println(LOG_ERROR, String.format("[!] Timer Refresh UI Basic Host Error: %s", exception.getMessage()) );
}
}
}
});
stdout_println(LOG_DEBUG, "[*] Init Timer Basic Host");
}
// 启动定时器
public static void startTimerBasicHost() {
if (basicHostTimer != null) {
if (!basicHostTimer.isRunning()){
basicHostTimer.start();
stdout_println(LOG_DEBUG, "[*] Start Timer Basic Host");
}
} else {
initTimerBasicHost();
}
}
// 定义一个方法来停止定时器
public static void stopTimerBasicHost() {
if (basicHostTimer != null && basicHostTimer.isRunning()) {
basicHostTimer.stop();
stdout_println(LOG_DEBUG, "[*] Stop Timer Basic Host");
}
}
/**
* 初始化创建表格下方的消息内容面板
*/
private JTabbedPane getBasicHostMsgTabs() {
IBurpExtenderCallbacks callbacks = BurpExtender.getCallbacks();
// 将 结果消息面板 添加到窗口下方
JTabbedPane tabs = new JTabbedPane();
//敏感信息结果面板 使用 "text/html" 可用于 html 渲染颜色
basicHostFindInfoTextPane = new JEditorPane("text/html", "");
JScrollPane basicHostFindInfoTextScrollPane = new JScrollPane(basicHostFindInfoTextPane);
basicHostFindInfoTextScrollPane.setVerticalScrollBarPolicy(JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED);
// 提取到URL的面板
basicHostRespFindUrlTEditor = callbacks.createTextEditor();
basicHostRespFindPathTEditor = callbacks.createTextEditor();
basicHostDirectPath2UrlTEditor = callbacks.createTextEditor();
basicHostSmartPath2UrlTEditor = callbacks.createTextEditor();
basicHostUnvisitedUrlTEditor = callbacks.createTextEditor();
basicHostPathTreeTEditor = callbacks.createTextEditor();
//响应状态结果面板 使用 "text/html" 可用于 html 渲染
basicHostAllUrlStatusTEditor = new JEditorPane("text/html", "");
JScrollPane basicHostAllUrlStatusScrollPane = new JScrollPane(basicHostAllUrlStatusTEditor);
basicHostAllUrlStatusScrollPane.setVerticalScrollBarPolicy(JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED);
tabs.addTab("RespFindInfo",null, basicHostFindInfoTextScrollPane, "基于当前响应体提取的敏感信息"); //显示提取的信息
tabs.addTab("RespFindUrl",null, basicHostRespFindUrlTEditor.getComponent(), "基于当前响应体提取的URL"); //显示在这个URL中找到的PATH
tabs.addTab("RespFindPath",null, basicHostRespFindPathTEditor.getComponent(), "基于当前响应体提取的PATH"); //显示在这个URL中找到的PATH
tabs.addTab("DirectPath2Url",null, basicHostDirectPath2UrlTEditor.getComponent(), "基于当前请求URL目录 拼接 提取的PATH"); //显示在这个URL中找到的PATH
tabs.addTab("SmartPath2Url",null, basicHostSmartPath2UrlTEditor.getComponent(), "基于当前网站有效目录 和 提取的PATH 动态计算出的URL"); //显示在这个URL中找到的PATH
tabs.addTab("UnvisitedUrl",null, basicHostUnvisitedUrlTEditor.getComponent(), "当前所有提取URL中的未访问过的URl"); //显示在这个URL中找到的Path 且还没有访问过的URL
tabs.addTab("PathTreeInfo",null, basicHostPathTreeTEditor.getComponent(), "当前网站的路径树信息");
tabs.addTab("AllUrlStatus",null, basicHostAllUrlStatusScrollPane, "当前网站所有提取URL的响应状态聚合");
return tabs;
}
/**
* 清空当前Msg tabs中显示的数据
*/
private static void clearBasicHostMsgTabsShowData() {
basicHostFindInfoTextPane.setText("");
basicHostRespFindUrlTEditor.setText(new byte[0]);
basicHostRespFindPathTEditor.setText(new byte[0]);
basicHostDirectPath2UrlTEditor.setText(new byte[0]);
basicHostSmartPath2UrlTEditor.setText(new byte[0]);
basicHostUnvisitedUrlTEditor.setText(new byte[0]);
basicHostPathTreeTEditor.setText(new byte[0]);
basicHostAllUrlStatusTEditor.setText("");
}
/**
* 鼠标点击或键盘移动到行时,自动更新下方的msgTab
*/
private void basicHostTableAddActionSetMsgTabData() {
//为表格 添加 鼠标监听器
//获取点击事件发生时鼠标所在行的索引 根据选中行的索引来更新其他组件的状态或内容。
basicHostMsgTableUI.addMouseListener(new MouseAdapter() {
@Override
public void mouseClicked(MouseEvent e) {
// 只有在双击时才执行
//if (e.getClickCount() == 2) {
SwingUtilities.invokeLater(new Runnable() {
public void run() {
try {
int row = basicHostMsgTableUI.rowAtPoint(e.getPoint());
if (row >= 0) {
updateComponentsBasedOnSelectedRow(row);
}
}catch (Exception ef) {
BurpExtender.getStderr().println("[-] Error click table: " + basicHostMsgTableUI.rowAtPoint(e.getPoint()));
ef.printStackTrace(BurpExtender.getStderr());
}
}
});
}
});
//为表格 添加 键盘按键释放事件监听器
//获取按键事件发生时鼠标所在行的索引 根据选中行的索引来更新其他组件的状态或内容。
basicHostMsgTableUI.addKeyListener(new KeyAdapter() {
@Override
public void keyReleased(KeyEvent e) {
//关注向上 和向下 的按键事件
if (e.getKeyCode() == KeyEvent.VK_UP || e.getKeyCode() == KeyEvent.VK_DOWN) {
SwingUtilities.invokeLater(new Runnable() {
public void run() {
try {
int row = basicHostMsgTableUI.getSelectedRow();
if (row >= 0) {
updateComponentsBasedOnSelectedRow(row);
}
}catch (Exception ef) {
BurpExtender.getStderr().println("[-] Error KeyEvent.VK_UP OR KeyEvent.VK_DOWN: ");
ef.printStackTrace(BurpExtender.getStderr());
}
}
});
}
}
});
}
private String recordRootUrl;
/**
* 更新表格行对应的下方数据信息
* @param row
*/
private void updateComponentsBasedOnSelectedRow(int row) {
//清理下方数据内容
clearBasicHostMsgTabsShowData();
//1、获取当前行的 rootUrl
String currentRootUrl = null;
try {
//实现排序后 视图行 数据的正确获取
currentRootUrl = UiUtils.getStringAtActualRow(basicHostMsgTableUI, row, 1);
} catch (Exception e) {
stderr_println(LOG_ERROR, String.format("[!] Table get Value At Row [%s] Error:%s", row, e.getMessage()));
}
//更新之前 msgHash Date为 处理中 注意 要修改 ReqDataTable
if (isNotEmptyObj(recordRootUrl)){
//当原来的状态是手动处理中时,就修改状态为处理完成
CommonUpdateStatus.updateStatusWhenStatusByRootUrl(AnalyseHostResultTable.tableName, recordRootUrl, Constants.HANDLE_END, Constants.HANDLE_ING);
}
//更新当前 msgHash Date为 处理中
if (isNotEmptyObj(currentRootUrl) ){
//当原来的状态是自动分析完成时,就修改请求状态为手工处理中
CommonUpdateStatus.updateStatusWhenStatusByRootUrl(AnalyseHostResultTable.tableName, currentRootUrl, Constants.HANDLE_ING, Constants.HANDLE_WAIT);
recordRootUrl = currentRootUrl;
} else {
return;
}
//点击时就调用更新数据
updateAllExtractUrRespStatus(Collections.singletonList(currentRootUrl), false);
//查询路径树信息 并美化输出
PathTreeModel pathTreeModel = PathTreeTable.fetchPathTreeByRootUrl(currentRootUrl);
if (pathTreeModel!=null){
JSONObject pathTree = pathTreeModel.getPathTree();
String prettyJson = JSON.toJSONString(pathTree, JSONWriter.Feature.PrettyFormat);
prettyJson = CastUtils.removeJsonForMat(prettyJson);
basicHostPathTreeTEditor.setText(prettyJson.getBytes());
}
//查询详细数据
BasicHostTableTabDataModel tabDataModel = AnalyseHostResultTable.fetchHostResultByRootUrl(currentRootUrl);
if (tabDataModel != null) {
//格式化为可输出的类型
String findInfo = CastUtils.urlInfoJsonArrayMapFormatHtml(tabDataModel.getFindInfo());
String findUrl = CastUtils.stringJsonArrayFormat(tabDataModel.getFindUrl());
String findPath = CastUtils.stringJsonArrayFormat(tabDataModel.getFindPath());
String findApi = CastUtils.stringJsonArrayFormat(tabDataModel.getFindApi());
String pathToUrl = CastUtils.stringJsonArrayFormat(tabDataModel.getPathToUrl());
String unvisitedUrl = CastUtils.stringJsonArrayFormat(tabDataModel.getUnvisitedUrl());
String allUrlStatus = CastUtils.stringUrlStatusMapFormatHtml(tabDataModel.getAllUrlStatus());
basicHostFindInfoTextPane.setText(findInfo);
basicHostRespFindUrlTEditor.setText(findUrl.getBytes());
basicHostRespFindPathTEditor.setText(findPath.getBytes());
basicHostDirectPath2UrlTEditor.setText(findApi.getBytes());
basicHostSmartPath2UrlTEditor.setText(pathToUrl.getBytes());
basicHostUnvisitedUrlTEditor.setText(unvisitedUrl.getBytes());
basicHostAllUrlStatusTEditor.setText(allUrlStatus);
}
}
/**
* 定时刷新表数据
*/
public void refreshBasicHostTableModel() {
//设置已加入数据库的数量
BasicHostConfigPanel.lbTaskerCountOnHost.setText(String.valueOf(CommonFetchData.fetchTableCounts(ReqDataTable.tableName)));
//设置成功分析的数量
BasicHostConfigPanel.lbAnalysisEndCountOnHost.setText(String.valueOf(CommonFetchData.fetchTableCountsByStatus(Constants.ANALYSE_END)));
// 获取搜索框和搜索选项
final String searchText = BasicHostConfigPanel.getUrlSearchBoxTextOnHost();
final String selectedOption = BasicHostConfigPanel.getComboBoxSelectedOptionOnHost();
// 使用SwingWorker来处理数据更新,避免阻塞EDT
SwingWorker worker = new SwingWorker() {
@Override
protected Void doInBackground() throws Exception {
try {
// 执行耗时的数据操作
BasicHostInfoPanel.showDataHostTableByFilter(selectedOption, searchText.isEmpty() ? "" : searchText);
} catch (Exception e) {
// 处理数据操作中可能出现的异常
System.err.println("Error while updating data: " + e.getMessage());
e.printStackTrace();
}
return null;
}
@Override
protected void done() {
// 更新UI组件
try {
// 更新UI组件
SwingUtilities.invokeLater(() -> {
try {
basicHostMsgTableModel.fireTableDataChanged(); // 通知模型数据发生了变化
} catch (Exception e) {
// 处理更新UI组件时可能出现的异常
System.err.println("Error while updating UI: " + e.getMessage());
e.printStackTrace();
}
});
} catch (Exception e) {
// 处理在done()方法中可能出现的异常,例如InterruptedException或ExecutionException
System.err.println("Error in done method: " + e.getMessage());
e.printStackTrace();
}
}
};
worker.execute();
}
/**
* 基于过滤选项 和 搜索框内容 显示结果
* @param selectOption
* @param searchText
*/
public static void showDataHostTableByFilter(String selectOption, String searchText) {
// 在后台线程获取数据,避免冻结UI
new SwingWorker() {
@Override
protected Void doInBackground() throws Exception {
// 构建一个新的表格模型
basicHostMsgTableModel.setRowCount(0);
// 获取数据库中的所有ApiDataModels
ArrayList apiDataModels;
switch (selectOption) {
case "显示有效内容":
apiDataModels = TableLineDataModelBasicHostSQL.fetchHostTableLineHasInfoOrUri();
break;
case "待处理有效内容":
apiDataModels = TableLineDataModelBasicHostSQL.fetchHostTableLineHasInfoOrUriNotHandle();
break;
case "显示敏感内容":
apiDataModels = TableLineDataModelBasicHostSQL.fetchHostTableLineHasInfo();
break;
case "待处理敏感内容":
apiDataModels = TableLineDataModelBasicHostSQL.fetchHostTableLineHasInfoNotHandle();
break;
case "显示未访问路径":
apiDataModels = TableLineDataModelBasicHostSQL.fetchHostTableLineHasUnVisitedUrls();
break;
case "显示无效内容":
apiDataModels = TableLineDataModelBasicHostSQL.fetchHostTableLineAnyIsNull();
break;
case "显示全部内容":
default:
apiDataModels = TableLineDataModelBasicHostSQL.fetchHostTableLineAll();
break;
}
// 遍历apiDataModelMap
for (BasicHostTableLineDataModel apiDataModel : apiDataModels) {
String url = apiDataModel.getRootUrl();
//是否包含关键字,当输入了关键字时,使用本函数再次进行过滤
if (url.toLowerCase().contains(searchText.toLowerCase())) {
Object[] rowData = apiDataModel.toRowDataArray();
//model.insertRow(0, rowData); //插入到首行
basicHostMsgTableModel.insertRow(basicHostMsgTableModel.getRowCount(), rowData); //插入到最后一行
}
}
return null;
}
@Override
protected void done() {
try {
get();
} catch (InterruptedException | ExecutionException e) {
stderr_println(String.format("[!] showFilter error: %s", e.getMessage()));
//e.printStackTrace(BurpExtender.getStderr());
}
}
}.execute();
}
/**
* 查询所有 UnVisitedUrls 并逐个进行过滤
* @param rootUrls rootUrls目标列表, 为空 为Null时更新全部
*/
public void updateUnVisitedUrlsByRootUrls(List rootUrls) {
// 使用SwingWorker来处理数据更新,避免阻塞EDT
new SwingWorker() {
@Override
protected Void doInBackground() throws Exception {
// 获取所有未访问URl 注意需要大于0
List unVisitedUrlsModels;
if (rootUrls == null || rootUrls.isEmpty()) {
//更新所有的结果
unVisitedUrlsModels = AnalyseHostResultTable.fetchAllUnVisitedUrlsWithLimit(99);
} else {
//仅更新指定 rootUrls 对应的未访问URL
unVisitedUrlsModels = AnalyseHostResultTable.fetchUnVisitedUrlsByRootUrls(rootUrls);
}
//忽略没有内容的情况
if (unVisitedUrlsModels.isEmpty()) {
stderr_println(LOG_ERROR, String.format("[!] 获取对应的未访问URL为空: %s", rootUrls));
return null;
} else {
stdout_println(LOG_DEBUG, String.format("[*] 刷新未访问URL开始...Size: %s", unVisitedUrlsModels.size()));
}
//代码执行模式, eachMode 减少内存占用,但是查询次数更多
boolean eachMode = true;
if (eachMode){
//1、查询所有的非RootUrl的对应的Hash
List inRootUrls = new ArrayList<>();
if (isNotEmptyObj(rootUrls)){
inRootUrls = rootUrls;
} else {
for (UnVisitedUrlsModel urlsModel : unVisitedUrlsModels) {
inRootUrls.add(urlsModel.getRootUrl());
}
}
//2、获取当前非预期RootUrl的对应访问记录用于过滤
String accessedUrlHashesNotInRootUrls = CommonFetchData.fetchColumnGroupConcatStringNotInRootUrls(
RecordUrlTable.tableName,
RecordUrlTable.urlHashName,
inRootUrls
);
System.out.println(String.format("accessedUrlHashesNotInRootUrls:%s", accessedUrlHashesNotInRootUrls));
//3、多次循环查询每个RootUrl对应的访问记录
for (UnVisitedUrlsModel urlsModel : unVisitedUrlsModels) {
String currentRootUrl = urlsModel.getRootUrl();
List rawUnVisitedUrls = urlsModel.getUnvisitedUrls();
if (rawUnVisitedUrls.isEmpty()) continue;
//过滤黑名单中的URL 因为黑名单是不定时更新的
List newUnVisitedUrls = AnalyseInfo.filterFindUrls(currentRootUrl, rawUnVisitedUrls, BurpExtender.onlyScopeDomain);
//过滤 rootUrl无关的访问记录
if (newUnVisitedUrls.size() > 0 && isNotEmptyObj(accessedUrlHashesNotInRootUrls)){
List tmpUnVisitedUrls = new ArrayList<>();
for (String url : newUnVisitedUrls) {
String urlHash = RespHashUtils.calcCRC32(url);
if (!accessedUrlHashesNotInRootUrls.contains(urlHash)) {
tmpUnVisitedUrls.add(url);
}
}
newUnVisitedUrls = tmpUnVisitedUrls;
}
//过滤 rootUrl相关的访问记录
if (newUnVisitedUrls.size() > 0){
String accessedUrlHashesInRootUrl = CommonFetchData.fetchColumnGroupConcatStringInRootUrls(
RecordUrlTable.tableName,
RecordUrlTable.urlHashName,
Collections.singletonList(currentRootUrl)
);
System.out.println(String.format("accessedUrlHashesInRootUrl:%s", accessedUrlHashesInRootUrl));
if (isNotEmptyObj(accessedUrlHashesInRootUrl)){
List tmpUnVisitedUrls = new ArrayList<>();
for (String url : newUnVisitedUrls) {
String urlHash = RespHashUtils.calcCRC32(url);
if (!accessedUrlHashesInRootUrl.contains(urlHash)) {
tmpUnVisitedUrls.add(url);
}
}
newUnVisitedUrls = tmpUnVisitedUrls;
}
}
//更新记录并保存
urlsModel.setUnvisitedUrls(newUnVisitedUrls);
try {
AnalyseHostResultTable.updateUnVisitedUrlsByModel(urlsModel);
} catch (Exception ex) {
stderr_println(String.format("[!] Updating unvisited URL Error:%s", ex.getMessage()));
}
}
}
else {
// // 一次性获取所有 已经被访问过得URL列表【URL HASH】
// String accessedUrlHashes = CommonFetchData.fetchColumnGroupConcatString(RecordUrlTable.tableName, RecordUrlTable.urlHashName);
//
// //遍历 unVisitedUrlsModels 进行更新
// for (UnVisitedUrlsModel urlsModel : unVisitedUrlsModels) {
// //更新 unVisitedUrls 对象
// List rawUnVisitedUrls = urlsModel.getUnvisitedUrls();
// List newUnVisitedUrls = new ArrayList<>();
// for (String url : rawUnVisitedUrls) {
// String urlHash = CastUtils.calcCRC32(url);
// if (!accessedUrlHashes.contains(urlHash)) {
// newUnVisitedUrls.add(url);
// }
// }
//
// //过滤黑名单中的URL 因为黑名单是不定时更新的
// newUnVisitedUrls = AnalyseInfo.filterFindUrls(urlsModel.getRootUrl(), newUnVisitedUrls, BurpExtender.onlyScopeDomain);
// urlsModel.setUnvisitedUrls(newUnVisitedUrls);
//
// // 执行更新插入数据操作
// try {
// AnalyseHostUnVisitedUrls.updateUnVisitedUrlsByModel(urlsModel);
// } catch (Exception ex) {
// stderr_println(String.format("[!] Updating unvisited URL Error:%s", ex.getMessage()));
// }
// }
// 一次性获取所有 已经被访问过得URL列表【URL HASH】
String accessedUrlHashes = CommonFetchData.fetchColumnGroupConcatString(RecordUrlTable.tableName, RecordUrlTable.urlHashName);
//遍历 unVisitedUrlsModels 进行更新
unVisitedUrlsModels.parallelStream()
.forEach( urlsModel -> {
List rawUnVisitedUrls = urlsModel.getUnvisitedUrls();
//过滤黑名单中的URL 因为黑名单是不定时更新的
rawUnVisitedUrls = AnalyseInfo.filterFindUrls(urlsModel.getRootUrl(), rawUnVisitedUrls, BurpExtender.onlyScopeDomain);
//过滤访问记录
List