![](\"./datasets/overview.png\")
\n\n`Input`: A URL and its screenshot `Output`: Phish/Benign, Phishing target\n\n- Step 1: Enter Deep Object Detection Model, get predicted logos and inputs (inputs are not used for later prediction, just for explanation)\n\n- Step 2: Enter Deep Siamese Model\n - If Siamese report no target, `Return Benign, None`\n - Else Siamese report a target, `Return Phish, Phishing target`\n\n\n## Setup\n\nPrerequisite: [Pixi installed](https://pixi.sh/latest/)\n\nFor Linux/Mac,\n\n ```bash\n export KMP_DUPLICATE_LIB_OK=TRUE\n git clone https://github.com/lindsey98/Phishpedia.git\n cd Phishpedia\n pixi install\n chmod +x setup.sh\n ./setup.sh\n ```\n\nFor Windows, in PowerShell,\n\n ```bash\n git clone https://github.com/lindsey98/Phishpedia.git\n cd Phishpedia\n pixi install\n setup.bat\n ```\n\n## Running Phishpedia from Command Line\n\n```bash\npixi run python phishpedia.py --folder This website has been analyzed and determined to be ${labelElement.textContent.toLowerCase()}. \n Because we have matched a brand ${data.matched_brand} with confidence ${Math.round(data.confidence * 100, 3)}, \n and the domain extracted from url is within the domain list under the brand (which is [${data.correct_domain}]). \n Enjoy your surfing!
\n `;\n } else if (data.result === 'Phishing') {\n labelElement.className = 'phishing';\n labelElement.textContent = 'Phishing';\n matched_brand_element.textContent = data.matched_brand;\n siamese_conf_element.textContent = data.confidence;\n correct_domain_element.textContent = data.correct_domain;\n explanationElement.innerHTML = `\nThis website has been analyzed and determined to be ${labelElement.textContent.toLowerCase()}. \n Because we have matched a brand ${data.matched_brand} with confidence ${Math.round(data.confidence * 100, 3)}%, \n but the domain extracted from url is NOT within the domain list under the brand (which is [${data.correct_domain}]). \n Please proceed with caution!
\n `;\n } else {\n labelElement.className = 'unknown';\n labelElement.textContent = 'Unknown';\n matched_brand_element.textContent = \"unknown\";\n siamese_conf_element.textContent = \"0.00\";\n correct_domain_element.textContent = \"unknown\";\n explanationElement.innerHTML = `\nSorry, we don't find any matched brand in database so this website is determined to be ${labelElement.textContent.toLowerCase()}.
\nIt is still possible that this is a phishing site. Please proceed with caution!
\n `;\n }\n })\n .catch(error => {\n console.error('Error:', error);\n alert('检测失败，请稍后重试。');\n });\n },\n handleImageUpload(event) { // 处理图片上传事件\n const file = event.target.files[0];\n if (file) {\n this.uploadedImage = file;\n this.uploadImage();\n }\n },\n uploadImage() { // 上传图片到服务器\n const formData = new FormData();\n formData.append('image', this.uploadedImage);\n\n fetch('/upload', { // 假设上传图片的路由是 /upload\n method: 'POST',\n body: formData\n })\n .then(response => response.json())\n .then(data => {\n if (data.success) {\n this.imageUrl = data.imageUrl; // 更新图片URL\n this.uploadSuccess = true; // 标记上传成功\n } else {\n alert('上传图片失败: ' + data.error);\n }\n })\n .catch(error => {\n console.error('Error:', error);\n alert('上传图片失败，请稍后重试。');\n });\n },\n clearUpload() { // 清除上传的图像\n fetch('/clear_upload', { // 假设删除图片的路由是 /delete-image\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json'\n },\n body: JSON.stringify({ imageUrl: this.imageUrl })\n })\n .then(response => response.json())\n .then(data => {\n if (data.success) {\n this.imageUrl = '';\n this.uploadSuccess = false; // 重置上传状态\n } else {\n alert('删除图片失败: ' + data.error);\n }\n })\n .catch(error => {\n console.error('Error:', error);\n alert('删除图片失败，请稍后重试。');\n });\n }\n }\n});\n" }, { "path": "WEBtool/static/js/sidebar.js", "content": "// sidebar.js\nnew Vue({\n el: '#sidebar',\n data() {\n return {\n selectedDirectory: null, // 记录当前选中的目录\n selectedFile: null, // 记录当前选中的文件\n selectedDirectoryName: '',\n selectedFileName: '',\n showAddBrandForm: false, // 控制表单显示与隐藏\n brandName: '', // 品牌名称\n brandDomain: '', // 品牌域名\n }\n },\n mounted() {\n // 网页加载时调用 fetchFileTree 函数\n this.fetchFileTree();\n document.getElementById('logo-file-input').addEventListener('change', this.handleLogoFileSelect);\n\n const sidebar = document.getElementById(\"sidebar\");\n const sidebarToggle = document.getElementById(\"sidebar-toggle\");\n const closeSidebar = document.getElementById(\"close-sidebar\");\n\n // 点击打开侧边栏\n sidebarToggle.addEventListener(\"click\", () => {\n sidebar.classList.add(\"open\");\n });\n\n // 点击关闭侧边栏\n closeSidebar.addEventListener(\"click\", () => {\n sidebar.classList.remove(\"open\");\n this.clearSelected();\n });\n\n // 点击侧边栏外部关闭\n document.addEventListener(\"click\", (event) => {\n if (!sidebar.contains(event.target) && !sidebarToggle.contains(event.target)) {\n sidebar.classList.remove(\"open\");\n this.clearSelected();\n }\n });\n },\n methods: {\n // 递归渲染文件树\n renderFileTree(directory, parentPath = '') {\n // 获取文件树容器\n const fileTreeRoot = document.getElementById('file-tree-root');\n fileTreeRoot.innerHTML = ''; // 清空现有内容\n\n // 递归生成文件树节点\n const createFileTreeNode = (item, parentPath) => {\n const li = document.createElement('li');\n li.classList.add('file-item');\n\n const currentPath = parentPath ? `${parentPath}/${item.name}` : item.name;\n\n if (item.type === 'directory') {\n li.classList.add('file-folder');\n\n const folderNameContainer = document.createElement('div');\n folderNameContainer.classList.add('folder-name');\n folderNameContainer.innerHTML = `📁${item.name}`;\n li.appendChild(folderNameContainer);\n\n if (item.children) {\n const ul = document.createElement('ul');\n ul.classList.add('hidden'); // 默认隐藏子目录\n item.children.forEach((child) => {\n ul.appendChild(createFileTreeNode(child, currentPath)); // 传递当前目录的路径\n });\n li.appendChild(ul);\n\n // 单击选中目录\n folderNameContainer.addEventListener('click', (e) => {\n e.stopPropagation();\n this.selectDirectory(e, item.name);\n });\n\n // 双击展开/隐藏目录\n folderNameContainer.addEventListener('dblclick', (e) => {\n e.stopPropagation();\n ul.classList.toggle('hidden');\n });\n }\n } else {\n li.classList.add('file-file');\n li.innerHTML = `📄${item.name}`;\n\n // 单击选中文件\n li.addEventListener('click', (event) => {\n this.selectFile(event, item.name, parentPath);\n });\n }\n\n return li;\n };\n\n // 遍历顶层文件和目录\n directory.forEach((item) => {\n fileTreeRoot.appendChild(createFileTreeNode(item, parentPath));\n });\n },\n // 获取文件树数据\n fetchFileTree() {\n // 发送请求获取文件树数据\n fetch('/get-directory') // 后端文件树接口\n .then((response) => response.json())\n .then((data) => {\n if (data.file_tree) {\n this.fileTree = data.file_tree; // 存储文件树数据\n this.renderFileTree(this.fileTree); // 渲染文件树\n } else {\n console.error('Invalid file tree data');\n alert('文件树加载失败');\n }\n })\n .catch((error) => {\n console.error('Error fetching file tree:', error);\n alert('无法加载文件树，请稍后重试。');\n });\n },\n\n // 选中目录\n selectDirectory(event, directoryName) {\n const folderNameContainer = event.currentTarget;\n\n if (this.selectedDirectory) {\n this.selectedDirectory.classList.remove('selected');\n }\n if (this.selectedFile) {\n this.selectedFile.classList.remove('selected');\n }\n\n // 设置当前选中的目录\n this.selectedDirectory = folderNameContainer;\n this.selectedDirectoryName = directoryName;\n folderNameContainer.classList.add('selected');\n this.selectedFile = null;\n this.selectedFileName = '';\n },\n\n // 选中文件\n selectFile(event, fileName, parentPath) {\n const fileElement = event.currentTarget;\n\n if (this.selectedDirectory) {\n this.selectedDirectory.classList.remove('selected');\n }\n if (this.selectedFile) {\n this.selectedFile.classList.remove('selected');\n }\n\n // 设置当前选中的文件\n this.selectedFile = fileElement;\n this.selectedFileName = fileName;\n fileElement.classList.add('selected');\n this.selectedDirectory = null;\n this.selectedDirectoryName = parentPath;\n },\n\n // 增加品牌\n addBrand() {\n this.showAddBrandForm = true;\n },\n\n // 关闭添加品牌的表单\n closeAddBrandForm() {\n this.showAddBrandForm = false;\n this.brandName = '';\n this.brandDomain = '';\n },\n\n // 提交添加品牌的表单\n submitAddBrandForm() {\n if (!this.brandName || !this.brandDomain) {\n alert('Please fill in all fields.');\n closeAddBrandForm()\n return;\n }\n\n const formData = new FormData();\n formData.append('brandName', this.brandName);\n formData.append('brandDomain', this.brandDomain);\n\n fetch('/add-brand', {\n method: 'POST',\n body: formData\n })\n .then(response => response.json())\n .then(data => {\n if (data.success) {\n alert('Brand added successfully.');\n this.fetchFileTree();\n this.closeAddBrandForm();\n } else {\n alert('Failed to add brand: ' + data.error);\n }\n })\n .catch(error => {\n console.error('Error:', error);\n alert('Failed to add brand, please try again.');\n });\n },\n\n // 删除品牌\n delBrand() {\n if (this.selectedDirectory == null) {\n alert('Please select a brand first.');\n return;\n }\n const formData = new FormData();\n formData.append('directory', this.selectedDirectoryName);\n\n fetch('/del-brand', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json'\n },\n body: JSON.stringify({\n directory: this.selectedDirectoryName\n })\n })\n .then(response => response.json())\n .then(data => {\n if (data.success) {\n alert('Brand deletedsuccessfully.');\n this.fetchFileTree();\n }\n })\n },\n\n // 增加logo\n addLogo() {\n console.log('addLogo');\n if (this.selectedDirectory == null) {\n alert('Please select a brand first.');\n return;\n }\n document.getElementById('logo-file-input').click();\n },\n\n handleLogoFileSelect(event) {\n const file = event.target.files[0];\n if (file) {\n const formData = new FormData();\n formData.append('logo', file);\n formData.append('directory', this.selectedDirectoryName);\n\n fetch('/add-logo', {\n method: 'POST',\n body: formData\n })\n .then(response => response.json())\n .then(data => {\n if (data.success) {\n this.fetchFileTree();\n } else {\n alert('Failed to add logo: ' + data.error);\n }\n })\n .catch(error => {\n console.error('Error:', error);\n alert('Failed to add logo, please try again.');\n });\n }\n },\n\n // 删除logo\n delLogo() {\n if (this.selectedFile == null) {\n alert('Please select a logo first.');\n return;\n }\n\n const formData = new FormData();\n formData.append('directory', this.selectedDirectoryName);\n formData.append('filename', this.selectedFileName);\n\n fetch('/del-logo', {\n method: 'POST',\n body: formData\n })\n .then(response => response.json())\n .then(data => {\n if (data.success) {\n this.fetchFileTree();\n } else {\n alert('Failed to delete logo: ' + data.error);\n }\n })\n .catch(error => {\n console.error('Error:', error);\n alert('Failed to delete logo, please try again.');\n });\n },\n\n async reloadModel() {\n const overlay = document.getElementById('overlay');\n\n overlay.style.display = 'flex';\n\n try {\n const response = await fetch('/reload-model', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json'\n }\n });\n const data = await response.json();\n } catch (error) {\n alert('Failed to reload model.');\n } finally {\n overlay.style.display = 'none';\n }\n },\n\n clearSelected() {\n if (this.selectedDirectory) {\n this.selectedDirectory.classList.remove('selected');\n this.selectDirectory = null;\n }\n if (this.selectedFile) {\n this.selectedFile.classList.remove('selected');\n this.selectFile = null;\n }\n this.selectedDirectoryName = '';\n this.selectedFileName = '';\n },\n }\n});" }, { "path": "WEBtool/templates/index.html", "content": "\n\n\n\n \n \n\n ![\"Logo\"](\"{{)
\n PhishPedia\n \n
\n\n \n
\n\n \n \n\n \n \n \n
\n Reloading model, this may take some time...
\n\n
\n\n \n \n \n \n\n\n"
},
{
"path": "WEBtool/utils_web.py",
"content": "# help function for phishpedia web app\nimport os\nimport pickle\nimport shutil\nimport socket\nimport base64\nimport io\nfrom PIL import Image\nimport cv2\n\n\ndef check_port_inuse(port, host):\n try:\n s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n s.settimeout(1)\n s.connect((host, port))\n return True\n except socket.error:\n return False\n finally:\n if s:\n s.close()\n\n\ndef allowed_file(filename):\n return '.' in filename and \\\n filename.rsplit('.', 1)[1].lower() in {'png', 'jpg', 'jpeg'}\n\n\ndef initial_upload_folder(upload_folder):\n try:\n shutil.rmtree(upload_folder)\n except FileNotFoundError:\n pass\n os.makedirs(upload_folder, exist_ok=True)\n \n \ndef convert_to_base64(image_array):\n if image_array is None:\n return None\n \n image_array_rgb = cv2.cvtColor(image_array, cv2.COLOR_BGR2RGB)\n img = Image.fromarray(image_array_rgb)\n buffered = io.BytesIO()\n img.save(buffered, format=\"PNG\")\n plotvis_base64 = base64.b64encode(buffered.getvalue()).decode('utf-8')\n return plotvis_base64\n\n\ndef domain_map_add(brand_name, domains_str, domain_map_path):\n domains = [domain.strip() for domain in domains_str.split(',') if domain.strip()]\n \n # Load existing domain mapping\n with open(domain_map_path, 'rb') as f:\n domain_map = pickle.load(f)\n \n # Add new brand and domains\n if brand_name in domain_map:\n if isinstance(domain_map[brand_name], list):\n # Add new domains, avoid duplicates\n existing_domains = set(domain_map[brand_name])\n for domain in domains:\n if domain not in existing_domains:\n domain_map[brand_name].append(domain)\n else:\n # If current value is not a list, convert to list\n old_domain = domain_map[brand_name]\n domain_map[brand_name] = [old_domain] + [d for d in domains if d != old_domain]\n else:\n domain_map[brand_name] = domains\n \n # Save updated mapping\n with open(domain_map_path, 'wb') as f:\n pickle.dump(domain_map, f)\n\n \ndef domain_map_delete(brand_name, domain_map_path):\n # Load existing domain mapping\n with open(domain_map_path, 'rb') as f:\n domain_map = pickle.load(f)\n \n print(\"before deleting\", len(domain_map))\n \n # Delete brand and its domains\n if brand_name in domain_map:\n del domain_map[brand_name]\n \n print(\"after deleting\", len(domain_map))\n \n # Save updated mapping\n with open(domain_map_path, 'wb') as f:\n pickle.dump(domain_map, f)\n"
},
{
"path": "configs.py",
"content": "# Global configuration\nimport yaml\nfrom logo_matching import cache_reference_list, load_model_weights\nfrom logo_recog import config_rcnn\nimport os\nimport numpy as np\n\n\ndef get_absolute_path(relative_path):\n base_path = os.path.dirname(__file__)\n return os.path.abspath(os.path.join(base_path, relative_path))\n\n\ndef load_config(reload_targetlist=False):\n with open(os.path.join(os.path.dirname(__file__), 'configs.yaml')) as file:\n configs = yaml.load(file, Loader=yaml.FullLoader)\n\n # Iterate through the configuration and update paths\n for section, settings in configs.items():\n for key, value in settings.items():\n if 'PATH' in key and isinstance(value, str): # Check if the key indicates a path\n absolute_path = get_absolute_path(value)\n configs[section][key] = absolute_path\n\n ELE_CFG_PATH = configs['ELE_MODEL']['CFG_PATH']\n ELE_WEIGHTS_PATH = configs['ELE_MODEL']['WEIGHTS_PATH']\n ELE_CONFIG_THRE = configs['ELE_MODEL']['DETECT_THRE']\n ELE_MODEL = config_rcnn(ELE_CFG_PATH,\n ELE_WEIGHTS_PATH,\n conf_threshold=ELE_CONFIG_THRE)\n\n # siamese model\n SIAMESE_THRE = configs['SIAMESE_MODEL']['MATCH_THRE']\n\n print('Load protected logo list')\n targetlist_zip_path = configs['SIAMESE_MODEL']['TARGETLIST_PATH']\n targetlist_dir = os.path.dirname(targetlist_zip_path)\n zip_file_name = os.path.basename(targetlist_zip_path)\n targetlist_folder = zip_file_name.split('.zip')[0]\n full_targetlist_folder_dir = os.path.join(targetlist_dir, targetlist_folder)\n\n # if reload_targetlist or targetlist_zip_path.endswith('.zip') and not os.path.isdir(full_targetlist_folder_dir):\n # os.makedirs(full_targetlist_folder_dir, exist_ok=True)\n # subprocess.run(f'unzip -o \"{targetlist_zip_path}\" -d \"{full_targetlist_folder_dir}\"', shell=True)\n\n SIAMESE_MODEL = load_model_weights(num_classes=configs['SIAMESE_MODEL']['NUM_CLASSES'],\n weights_path=configs['SIAMESE_MODEL']['WEIGHTS_PATH'])\n\n LOGO_FEATS_NAME = 'LOGO_FEATS.npy'\n LOGO_FILES_NAME = 'LOGO_FILES.npy'\n\n if reload_targetlist or (not os.path.exists(os.path.join(os.path.dirname(__file__), LOGO_FEATS_NAME))):\n LOGO_FEATS, LOGO_FILES = cache_reference_list(model=SIAMESE_MODEL,\n targetlist_path=full_targetlist_folder_dir)\n print('Finish loading protected logo list')\n np.save(os.path.join(os.path.dirname(__file__), LOGO_FEATS_NAME), LOGO_FEATS)\n np.save(os.path.join(os.path.dirname(__file__), LOGO_FILES_NAME), LOGO_FILES)\n\n else:\n LOGO_FEATS, LOGO_FILES = np.load(os.path.join(os.path.dirname(__file__), LOGO_FEATS_NAME)), \\\n np.load(os.path.join(os.path.dirname(__file__), LOGO_FILES_NAME))\n\n DOMAIN_MAP_PATH = configs['SIAMESE_MODEL']['DOMAIN_MAP_PATH']\n\n return ELE_MODEL, SIAMESE_THRE, SIAMESE_MODEL, LOGO_FEATS, LOGO_FILES, DOMAIN_MAP_PATH\n"
},
{
"path": "configs.yaml",
"content": "ELE_MODEL: # element recognition model -- logo only\n CFG_PATH: models/faster_rcnn.yaml # os.path.join(os.path.dirname(__file__), xxx)\n WEIGHTS_PATH: models/rcnn_bet365.pth\n DETECT_THRE: 0.05\n\nSIAMESE_MODEL:\n NUM_CLASSES: 277 # number of brands, users don't need to modify this even the targetlist is expanded\n MATCH_THRE: 0.87 # FIXME: threshold is 0.87 in phish-discovery?\n WEIGHTS_PATH: models/resnetv2_rgb_new.pth.tar\n TARGETLIST_PATH: models/expand_targetlist.zip\n DOMAIN_MAP_PATH: models/domain_map.pkl"
},
{
"path": "datasets/test_sites/accounts.g.cdcde.com/html.txt",
"content": ""
},
{
"path": "datasets/test_sites/accounts.g.cdcde.com/info.txt",
"content": ""
},
{
"path": "logo_matching.py",
"content": "from PIL import Image, ImageOps\nfrom torchvision import transforms\nfrom utils import brand_converter, resolution_alignment, l2_norm\nfrom models import KNOWN_MODELS\nimport torch\nimport os\nimport numpy as np\nfrom collections import OrderedDict\nfrom tqdm import tqdm\nfrom tldextract import tldextract\nimport pickle\n\nCOUNTRY_TLDs = [\n \".af\",\n \".ax\",\n \".al\",\n \".dz\",\n \".as\",\n \".ad\",\n \".ao\",\n \".ai\",\n \".aq\",\n \".ag\",\n \".ar\",\n \".am\",\n \".aw\",\n \".ac\",\n \".au\",\n \".at\",\n \".az\",\n \".bs\",\n \".bh\",\n \".bd\",\n \".bb\",\n \".eus\",\n \".by\",\n \".be\",\n \".bz\",\n \".bj\",\n \".bm\",\n \".bt\",\n \".bo\",\n \".bq\",\".an\",\".nl\",\n \".ba\",\n \".bw\",\n \".bv\",\n \".br\",\n \".io\",\n \".vg\",\n \".bn\",\n \".bg\",\n \".bf\",\n \".mm\",\n \".bi\",\n \".kh\",\n \".cm\",\n \".ca\",\n \".cv\",\n \".cat\",\n \".ky\",\n \".cf\",\n \".td\",\n \".cl\",\n \".cn\",\n \".cx\",\n \".cc\",\n \".co\",\n \".km\",\n \".cd\",\n \".cg\",\n \".ck\",\n \".cr\",\n \".ci\",\n \".hr\",\n \".cu\",\n \".cw\",\n \".cy\",\n \".cz\",\n \".dk\",\n \".dj\",\n \".dm\",\n \".do\",\n \".tl\",\".tp\",\n \".ec\",\n \".eg\",\n \".sv\",\n \".gq\",\n \".er\",\n \".ee\",\n \".et\",\n \".eu\",\n \".fk\",\n \".fo\",\n \".fm\",\n \".fj\",\n \".fi\",\n \".fr\",\n \".gf\",\n \".pf\",\n \".tf\",\n \".ga\",\n \".gal\",\n \".gm\",\n \".ps\",\n \".ge\",\n \".de\",\n \".gh\",\n \".gi\",\n \".gr\",\n \".gl\",\n \".gd\",\n \".gp\",\n \".gu\",\n \".gt\",\n \".gg\",\n \".gn\",\n \".gw\",\n \".gy\",\n \".ht\",\n \".hm\",\n \".hn\",\n \".hk\",\n \".hu\",\n \".is\",\n \".in\",\n \".id\",\n \".ir\",\n \".iq\",\n \".ie\",\n \".im\",\n \".il\",\n \".it\",\n \".jm\",\n \".jp\",\n \".je\",\n \".jo\",\n \".kz\",\n \".ke\",\n \".ki\",\n \".kw\",\n \".kg\",\n \".la\",\n \".lv\",\n \".lb\",\n \".ls\",\n \".lr\",\n \".ly\",\n \".li\",\n \".lt\",\n \".lu\",\n \".mo\",\n \".mk\",\n \".mg\",\n \".mw\",\n \".my\",\n \".mv\",\n \".ml\",\n \".mt\",\n \".mh\",\n \".mq\",\n \".mr\",\n \".mu\",\n \".yt\",\n \".mx\",\n \".md\",\n \".mc\",\n \".mn\",\n \".me\",\n \".ms\",\n \".ma\",\n \".mz\",\n \".mm\",\n \".na\",\n \".nr\",\n \".np\",\n \".nl\",\n \".nc\",\n \".nz\",\n \".ni\",\n \".ne\",\n \".ng\",\n \".nu\",\n \".nf\",\n \".nc\",\".tr\",\n \".kp\",\n \".mp\",\n \".no\",\n \".om\",\n \".pk\",\n \".pw\",\n \".ps\",\n \".pa\",\n \".pg\",\n \".py\",\n \".pe\",\n \".ph\",\n \".pn\",\n \".pl\",\n \".pt\",\n \".pr\",\n \".qa\",\n \".ro\",\n \".ru\",\n \".rw\",\n \".re\",\n \".bq\",\".an\",\n \".bl\",\".gp\",\".fr\",\n \".sh\",\n \".kn\",\n \".lc\",\n \".mf\",\".gp\",\".fr\",\n \".pm\",\n \".vc\",\n \".ws\",\n \".sm\",\n \".st\",\n \".sa\",\n \".sn\",\n \".rs\",\n \".sc\",\n \".sl\",\n \".sg\",\n \".bq\",\".an\",\".nl\",\n \".sx\",\".an\",\n \".sk\",\n \".si\",\n \".sb\",\n \".so\",\n \".so\",\n \".za\",\n \".gs\",\n \".kr\",\n \".ss\",\n \".es\",\n \".lk\",\n \".sd\",\n \".sr\",\n \".sj\",\n \".sz\",\n \".se\",\n \".ch\",\n \".sy\",\n \".tw\",\n \".tj\",\n \".tz\",\n \".th\",\n \".tg\",\n \".tk\",\n \".to\",\n \".tt\",\n \".tn\",\n \".tr\",\n \".tm\",\n \".tc\",\n \".tv\",\n \".ug\",\n \".ua\",\n \".ae\",\n \".uk\",\n \".us\",\n \".vi\",\n \".uy\",\n \".uz\",\n \".vu\",\n \".va\",\n \".ve\",\n \".vn\",\n \".wf\",\n \".eh\",\n \".ma\",\n \".ye\",\n \".zm\",\n \".zw\"\n]\n\ndef check_domain_brand_inconsistency(logo_boxes,\n domain_map_path: str,\n model, logo_feat_list,\n file_name_list, shot_path: str,\n url: str, similarity_threshold: float,\n topk: float = 3):\n # targetlist domain list\n with open(domain_map_path, 'rb') as handle:\n domain_map = pickle.load(handle)\n\n print('Number of logo boxes:', len(logo_boxes))\n suffix_part = '.'+ tldextract.extract(url).suffix\n domain_part = tldextract.extract(url).domain\n extracted_domain = domain_part + suffix_part\n matched_target, matched_domain, matched_coord, this_conf = None, None, None, None\n\n if len(logo_boxes) > 0:\n # siamese prediction for logo box\n for i, coord in enumerate(logo_boxes):\n\n if i == topk:\n break\n\n min_x, min_y, max_x, max_y = coord\n bbox = [float(min_x), float(min_y), float(max_x), float(max_y)]\n matched_target, matched_domain, this_conf = pred_brand(model, domain_map,\n logo_feat_list, file_name_list,\n shot_path, bbox,\n similarity_threshold=similarity_threshold,\n grayscale=False,\n do_aspect_ratio_check=False,\n do_resolution_alignment=False)\n\n # print(target_this, domain_this, this_conf)\n # domain matcher to avoid FP\n if matched_target and matched_domain:\n matched_coord = coord\n matched_domain_parts = [tldextract.extract(x).domain for x in matched_domain]\n matched_suffix_parts = [tldextract.extract(x).suffix for x in matched_domain]\n \n # If the webpage domain exactly aligns with the target website's domain => Benign\n if extracted_domain in matched_domain:\n matched_target, matched_domain = None, None # Clear if domains are consistent\n elif domain_part in matched_domain_parts: # # elIf only the 2nd-level-domains align, and the tld is regional => Benign\n if \".\" + suffix_part.split('.')[-1] in COUNTRY_TLDs:\n matched_target, matched_domain = None, None\n else:\n break # Inconsistent domain found, break the loop\n else:\n break # Inconsistent domain found, break the loop\n\n return brand_converter(matched_target), matched_domain, matched_coord, this_conf\n\n\ndef load_model_weights(num_classes: int, weights_path: str):\n '''\n :param num_classes: number of protected brands\n :param weights_path: siamese weights\n :return model: siamese model\n '''\n # Initialize model\n device = 'cuda' if torch.cuda.is_available() else 'cpu'\n model = KNOWN_MODELS[\"BiT-M-R50x1\"](head_size=num_classes, zero_head=True)\n\n # Load weights\n weights = torch.load(weights_path, map_location='cpu')\n weights = weights['model'] if 'model' in weights.keys() else weights\n new_state_dict = OrderedDict()\n for k, v in weights.items():\n if 'module.' in k:\n name = k.split('module.')[1]\n else:\n name = k\n new_state_dict[name] = v\n\n model.load_state_dict(new_state_dict)\n model.to(device)\n model.eval()\n return model\n\n\ndef cache_reference_list(model, targetlist_path: str, grayscale=False):\n '''\n cache the embeddings of the reference list\n :param targetlist_path: targetlist folder\n :param grayscale: convert logo to grayscale or not, default is RGB\n :return logo_feat_list: targetlist embeddings\n :return file_name_list: targetlist paths\n '''\n\n # Prediction for targetlists\n logo_feat_list = []\n file_name_list = []\n\n target_list = os.listdir(targetlist_path)\n for target in tqdm(target_list):\n if target.startswith('.'): # skip hidden files\n continue\n logo_list = os.listdir(os.path.join(targetlist_path, target))\n for logo_path in logo_list:\n # List of valid image extensions\n valid_extensions = ['.png', 'PNG', '.jpeg', '.jpg', '.JPG', '.JPEG']\n if any(logo_path.endswith(ext) for ext in valid_extensions):\n skip_prefixes = ['loginpage', 'homepage']\n if any(logo_path.startswith(prefix) for prefix in skip_prefixes): # skip homepage/loginpage\n continue\n try:\n logo_feat_list.append(get_embedding(img=os.path.join(targetlist_path, target, logo_path),\n model=model, grayscale=grayscale))\n file_name_list.append(str(os.path.join(targetlist_path, target, logo_path)))\n except OSError:\n print(f\"Error opening image: {os.path.join(targetlist_path, target, logo_path)}\")\n continue\n\n return logo_feat_list, file_name_list\n\n\n@torch.no_grad()\ndef get_embedding(img, model, grayscale=False):\n '''\n Inference for a single image\n :param img: image path in str or image in PIL.Image\n :param model: model to make inference\n :param grayscale: convert image to grayscale or not\n :return feature embedding of shape (2048,)\n '''\n # img_size = 224\n img_size = 128\n mean = [0.5, 0.5, 0.5]\n std = [0.5, 0.5, 0.5]\n device = 'cuda' if torch.cuda.is_available() else 'cpu'\n\n img_transforms = transforms.Compose(\n [transforms.ToTensor(),\n transforms.Normalize(mean=mean, std=std),\n ])\n\n img = Image.open(img) if isinstance(img, str) else img\n img = img.convert(\"L\").convert(\"RGB\") if grayscale else img.convert(\"RGB\")\n\n ## Resize the image while keeping the original aspect ratio\n pad_color = 255 if grayscale else (255, 255, 255)\n img = ImageOps.expand(\n img,\n (\n (max(img.size) - img.size[0]) // 2,\n (max(img.size) - img.size[1]) // 2,\n (max(img.size) - img.size[0]) // 2,\n (max(img.size) - img.size[1]) // 2\n ),\n fill=pad_color\n )\n\n img = img.resize((img_size, img_size))\n\n # Predict the embedding\n img = img_transforms(img)\n img = img[None, ...].to(device)\n logo_feat = model.features(img)\n logo_feat = l2_norm(logo_feat).squeeze(0).cpu().numpy() # L2-normalization final shape is (2048,)\n\n return logo_feat\n\ndef chunked_dot(logo_feat_list, img_feat, chunk_size=128):\n sim_list = []\n\n for start in range(0, logo_feat_list.shape[0], chunk_size):\n end = start + chunk_size\n chunk = logo_feat_list[start:end]\n sim_chunk = np.dot(chunk, img_feat.T) # shape: (chunk_size, M)\n sim_list.extend(sim_chunk)\n\n return sim_list\n\ndef pred_brand(model, domain_map, logo_feat_list, file_name_list, shot_path: str, gt_bbox, similarity_threshold,\n grayscale=False,\n do_resolution_alignment=True,\n do_aspect_ratio_check=True):\n '''\n Return predicted brand for one cropped image\n :param model: model to use\n :param domain_map: brand-domain dictionary\n :param logo_feat_list: reference logo feature embeddings\n :param file_name_list: reference logo paths\n :param shot_path: path to the screenshot\n :param gt_bbox: 1x4 np.ndarray/list/tensor bounding box coords\n :param similarity_threshold: similarity threshold for siamese\n :param do_resolution_alignment: if the similarity does not exceed the threshold, do we align their resolutions to have a retry\n :param do_aspect_ratio_check: once two logos are similar, whether we want to a further check on their aspect ratios\n :param grayscale: convert image(cropped) to grayscale or not\n :return: predicted target, predicted target's domain\n '''\n\n try:\n img = Image.open(shot_path)\n except OSError: # if the image cannot be identified, return nothing\n print('Screenshot cannot be open')\n return None, None, None\n\n # get predicted box --> crop from screenshot\n cropped = img.crop((gt_bbox[0], gt_bbox[1], gt_bbox[2], gt_bbox[3]))\n img_feat = get_embedding(cropped, model, grayscale=grayscale)\n\n # get cosine similarity with every protected logo\n sim_list = chunked_dot(logo_feat_list, img_feat) # take dot product for every pair of embeddings (Cosine Similarity)\n pred_brand_list = file_name_list\n\n assert len(sim_list) == len(pred_brand_list)\n\n # get top 3 brands\n idx = np.argsort(sim_list)[::-1][:3]\n pred_brand_list = np.array(pred_brand_list)[idx]\n sim_list = np.array(sim_list)[idx]\n\n # top1,2,3 candidate logos\n top3_brandlist = [brand_converter(os.path.basename(os.path.dirname(x))) for x in pred_brand_list]\n top3_domainlist = [domain_map[x] for x in top3_brandlist]\n top3_simlist = sim_list\n\n for j in range(3):\n predicted_brand, predicted_domain = None, None\n\n # If we are trying those lower rank logo, the predicted brand of them should be the same as top1 logo, otherwise might be false positive\n if top3_brandlist[j] != top3_brandlist[0]:\n continue\n\n # If the largest similarity exceeds threshold\n if top3_simlist[j] >= similarity_threshold:\n predicted_brand = top3_brandlist[j]\n predicted_domain = top3_domainlist[j]\n final_sim = top3_simlist[j]\n\n # Else if not exceed, try resolution alignment, see if can improve\n elif do_resolution_alignment:\n orig_candidate_logo = Image.open(pred_brand_list[j])\n cropped, candidate_logo = resolution_alignment(cropped, orig_candidate_logo)\n img_feat = get_embedding(cropped, model, grayscale=grayscale)\n logo_feat = get_embedding(candidate_logo, model, grayscale=grayscale)\n final_sim = logo_feat.dot(img_feat)\n if final_sim >= similarity_threshold:\n predicted_brand = top3_brandlist[j]\n predicted_domain = top3_domainlist[j]\n else:\n break # no hope, do not try other lower rank logos\n\n ## If there is a prediction, do aspect ratio check\n if predicted_brand is not None:\n if do_aspect_ratio_check:\n orig_candidate_logo = Image.open(pred_brand_list[j])\n ratio_crop = cropped.size[0] / cropped.size[1]\n ratio_logo = orig_candidate_logo.size[0] / orig_candidate_logo.size[1]\n # aspect ratios of matched pair must not deviate by more than factor of 2.5\n if max(ratio_crop, ratio_logo) / min(ratio_crop, ratio_logo) > 2.5:\n continue # did not pass aspect ratio check, try other\n return predicted_brand, predicted_domain, final_sim\n\n return None, None, top3_simlist[0]\n"
},
{
"path": "logo_recog.py",
"content": "from detectron2.config import get_cfg\nfrom detectron2.engine import DefaultPredictor\nimport cv2\nimport numpy as np\nimport torch\n\n\ndef pred_rcnn(im, predictor):\n '''\n Perform inference for RCNN\n :param im:\n :param predictor:\n :return:\n '''\n im = cv2.imread(im)\n\n if im is not None:\n if im.shape[-1] == 4:\n im = cv2.cvtColor(im, cv2.COLOR_BGRA2BGR)\n else:\n print(f\"Image at path {im} is None\")\n return None\n\n outputs = predictor(im)\n\n instances = outputs['instances']\n pred_classes = instances.pred_classes # tensor\n pred_boxes = instances.pred_boxes # Boxes object\n\n logo_boxes = pred_boxes[pred_classes == 1].tensor\n\n return logo_boxes\n\n\ndef config_rcnn(cfg_path, weights_path, conf_threshold):\n '''\n Configure weights and confidence threshold\n :param cfg_path:\n :param weights_path:\n :param conf_threshold:\n :return:\n '''\n cfg = get_cfg()\n cfg.merge_from_file(cfg_path)\n cfg.MODEL.WEIGHTS = weights_path\n cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = conf_threshold\n # uncomment if you installed detectron2 cpu version\n if not torch.cuda.is_available():\n cfg.MODEL.DEVICE = 'cpu'\n\n # Initialize model\n predictor = DefaultPredictor(cfg)\n return predictor\n\n\nCOLORS = {\n 0: (255, 255, 0), # logo\n 1: (36, 255, 12), # input\n 2: (0, 255, 255), # button\n 3: (0, 0, 255), # label\n 4: (255, 0, 0) # block\n}\n\n\ndef vis(img_path, pred_boxes):\n '''\n Visualize rcnn predictions\n :param img_path: str\n :param pred_boxes: torch.Tensor of shape Nx4, bounding box coordinates in (x1, y1, x2, y2)\n :param pred_classes: torch.Tensor of shape Nx1 0 for logo, 1 for input, 2 for button, 3 for label(text near input), 4 for block\n :return None\n '''\n\n check = cv2.imread(img_path)\n if pred_boxes is None or len(pred_boxes) == 0:\n print(\"Pred_boxes is None or the length of pred_boxes is 0\")\n return check\n pred_boxes = pred_boxes.numpy() if not isinstance(pred_boxes, np.ndarray) else pred_boxes\n\n # draw rectangle\n for j, box in enumerate(pred_boxes):\n if j == 0:\n cv2.rectangle(check, (int(box[0]), int(box[1])), (int(box[2]), int(box[3])), COLORS[0], 2)\n else:\n cv2.rectangle(check, (int(box[0]), int(box[1])), (int(box[2]), int(box[3])), COLORS[1], 2)\n\n return check\n"
},
{
"path": "models.py",
"content": "# Copyright 2020 Google LLC\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Lint as: python3\n\"\"\"Bottleneck ResNet v2 with GroupNorm and Weight Standardization.\"\"\"\n\nfrom collections import OrderedDict # pylint: disable=g-importing-member\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\n\nclass StdConv2d(nn.Conv2d):\n\n def forward(self, x):\n w = self.weight\n v, m = torch.var_mean(w, dim=[1, 2, 3], keepdim=True, unbiased=False)\n w = (w - m) / torch.sqrt(v + 1e-10)\n return F.conv2d(x, w, self.bias, self.stride, self.padding,\n self.dilation, self.groups)\n\n\ndef conv3x3(cin, cout, stride=1, groups=1, bias=False):\n return StdConv2d(cin, cout, kernel_size=3, stride=stride,\n padding=1, bias=bias, groups=groups)\n\n\ndef conv1x1(cin, cout, stride=1, bias=False):\n return StdConv2d(cin, cout, kernel_size=1, stride=stride,\n padding=0, bias=bias)\n\n\ndef tf2th(conv_weights):\n \"\"\"Possibly convert HWIO to OIHW.\"\"\"\n if conv_weights.ndim == 4:\n conv_weights = conv_weights.transpose([3, 2, 0, 1])\n return torch.from_numpy(conv_weights)\n\n\nclass PreActBottleneck(nn.Module):\n \"\"\"Pre-activation (v2) bottleneck block.\n\n Follows the implementation of \"Identity Mappings in Deep Residual Networks\":\n https://github.com/KaimingHe/resnet-1k-layers/blob/master/resnet-pre-act.lua\n\n Except it puts the stride on 3x3 conv when available.\n \"\"\"\n\n def __init__(self, cin, cout=None, cmid=None, stride=1):\n super().__init__()\n cout = cout or cin\n cmid = cmid or cout // 4\n\n self.gn1 = nn.GroupNorm(32, cin)\n self.conv1 = conv1x1(cin, cmid)\n self.gn2 = nn.GroupNorm(32, cmid)\n self.conv2 = conv3x3(cmid, cmid, stride) # Original code has it on conv1!!\n self.gn3 = nn.GroupNorm(32, cmid)\n self.conv3 = conv1x1(cmid, cout)\n self.relu = nn.ReLU(inplace=True)\n\n if (stride != 1 or cin != cout):\n # Projection also with pre-activation according to paper.\n self.downsample = conv1x1(cin, cout, stride)\n\n def forward(self, x):\n out = self.relu(self.gn1(x))\n\n # Residual branch\n residual = x\n if hasattr(self, 'downsample'):\n residual = self.downsample(out)\n\n # Unit's branch\n out = self.conv1(out)\n out = self.conv2(self.relu(self.gn2(out)))\n out = self.conv3(self.relu(self.gn3(out)))\n\n return out + residual\n\n def load_from(self, weights, prefix=''):\n convname = 'standardized_conv2d'\n with torch.no_grad():\n self.conv1.weight.copy_(tf2th(weights[f'{prefix}a/{convname}/kernel']))\n self.conv2.weight.copy_(tf2th(weights[f'{prefix}b/{convname}/kernel']))\n self.conv3.weight.copy_(tf2th(weights[f'{prefix}c/{convname}/kernel']))\n self.gn1.weight.copy_(tf2th(weights[f'{prefix}a/group_norm/gamma']))\n self.gn2.weight.copy_(tf2th(weights[f'{prefix}b/group_norm/gamma']))\n self.gn3.weight.copy_(tf2th(weights[f'{prefix}c/group_norm/gamma']))\n self.gn1.bias.copy_(tf2th(weights[f'{prefix}a/group_norm/beta']))\n self.gn2.bias.copy_(tf2th(weights[f'{prefix}b/group_norm/beta']))\n self.gn3.bias.copy_(tf2th(weights[f'{prefix}c/group_norm/beta']))\n if hasattr(self, 'downsample'):\n w = weights[f'{prefix}a/proj/{convname}/kernel']\n self.downsample.weight.copy_(tf2th(w))\n\n\nclass ResNetV2(nn.Module):\n \"\"\"Implementation of Pre-activation (v2) ResNet mode.\"\"\"\n\n def __init__(self, block_units, width_factor, head_size=21843, zero_head=False):\n super().__init__()\n wf = width_factor # shortcut 'cause we'll use it a lot.\n\n # The following will be unreadable if we split lines.\n # pylint: disable=line-too-long\n self.root = nn.Sequential(OrderedDict([\n ('conv', StdConv2d(3, 64 * wf, kernel_size=7, stride=2, padding=3, bias=False)),\n ('pad', nn.ConstantPad2d(1, 0)),\n ('pool', nn.MaxPool2d(kernel_size=3, stride=2, padding=0)),\n # The following is subtly not the same!\n # ('pool', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)),\n ]))\n\n self.body = nn.Sequential(OrderedDict([\n ('block1', nn.Sequential(OrderedDict(\n [('unit01', PreActBottleneck(cin=64 * wf, cout=256 * wf, cmid=64 * wf))] +\n [(f'unit{i:02d}', PreActBottleneck(cin=256 * wf, cout=256 * wf, cmid=64 * wf)) for i in\n range(2, block_units[0] + 1)],\n ))),\n ('block2', nn.Sequential(OrderedDict(\n [('unit01', PreActBottleneck(cin=256 * wf, cout=512 * wf, cmid=128 * wf, stride=2))] +\n [(f'unit{i:02d}', PreActBottleneck(cin=512 * wf, cout=512 * wf, cmid=128 * wf)) for i in\n range(2, block_units[1] + 1)],\n ))),\n ('block3', nn.Sequential(OrderedDict(\n [('unit01', PreActBottleneck(cin=512 * wf, cout=1024 * wf, cmid=256 * wf, stride=2))] +\n [(f'unit{i:02d}', PreActBottleneck(cin=1024 * wf, cout=1024 * wf, cmid=256 * wf)) for i in\n range(2, block_units[2] + 1)],\n ))),\n ('block4', nn.Sequential(OrderedDict(\n [('unit01', PreActBottleneck(cin=1024 * wf, cout=2048 * wf, cmid=512 * wf, stride=2))] +\n [(f'unit{i:02d}', PreActBottleneck(cin=2048 * wf, cout=2048 * wf, cmid=512 * wf)) for i in\n range(2, block_units[3] + 1)],\n ))),\n ]))\n # pylint: enable=line-too-long\n\n self.zero_head = zero_head\n self.head = nn.Sequential(OrderedDict([\n ('gn', nn.GroupNorm(32, 2048 * wf)),\n ('relu', nn.ReLU(inplace=True)),\n ('avg', nn.AdaptiveAvgPool2d(output_size=1)),\n ('conv', nn.Conv2d(2048 * wf, head_size, kernel_size=1, bias=True)),\n ]))\n\n def features(self, x):\n x = self.head[:-1](self.body(self.root(x)))\n\n return x.squeeze(-1).squeeze(-1)\n\n def forward(self, x):\n x = self.head(self.body(self.root(x)))\n assert x.shape[-2:] == (1, 1) # We should have no spatial shape left.\n return x[..., 0, 0]\n\n def load_from(self, weights, prefix='resnet/'):\n with torch.no_grad():\n self.root.conv.weight.copy_(\n tf2th(weights[f'{prefix}root_block/standardized_conv2d/kernel'])) # pylint: disable=line-too-long\n self.head.gn.weight.copy_(tf2th(weights[f'{prefix}group_norm/gamma']))\n self.head.gn.bias.copy_(tf2th(weights[f'{prefix}group_norm/beta']))\n if self.zero_head:\n nn.init.zeros_(self.head.conv.weight)\n nn.init.zeros_(self.head.conv.bias)\n else:\n self.head.conv.weight.copy_(\n tf2th(weights[f'{prefix}head/conv2d/kernel'])) # pylint: disable=line-too-long\n self.head.conv.bias.copy_(tf2th(weights[f'{prefix}head/conv2d/bias']))\n\n for bname, block in self.body.named_children():\n for uname, unit in block.named_children():\n unit.load_from(weights, prefix=f'{prefix}{bname}/{uname}/')\n\n\nKNOWN_MODELS = OrderedDict([\n ('BiT-M-R50x1', lambda *a, **kw: ResNetV2([3, 4, 6, 3], 1, *a, **kw)),\n ('BiT-M-R50x3', lambda *a, **kw: ResNetV2([3, 4, 6, 3], 3, *a, **kw)),\n ('BiT-M-R101x1', lambda *a, **kw: ResNetV2([3, 4, 23, 3], 1, *a, **kw)),\n ('BiT-M-R101x3', lambda *a, **kw: ResNetV2([3, 4, 23, 3], 3, *a, **kw)),\n ('BiT-M-R152x2', lambda *a, **kw: ResNetV2([3, 8, 36, 3], 2, *a, **kw)),\n ('BiT-M-R152x4', lambda *a, **kw: ResNetV2([3, 8, 36, 3], 4, *a, **kw)),\n ('BiT-S-R50x1', lambda *a, **kw: ResNetV2([3, 4, 6, 3], 1, *a, **kw)),\n ('BiT-S-R50x3', lambda *a, **kw: ResNetV2([3, 4, 6, 3], 3, *a, **kw)),\n ('BiT-S-R101x1', lambda *a, **kw: ResNetV2([3, 4, 23, 3], 1, *a, **kw)),\n ('BiT-S-R101x3', lambda *a, **kw: ResNetV2([3, 4, 23, 3], 3, *a, **kw)),\n ('BiT-S-R152x2', lambda *a, **kw: ResNetV2([3, 8, 36, 3], 2, *a, **kw)),\n ('BiT-S-R152x4', lambda *a, **kw: ResNetV2([3, 8, 36, 3], 4, *a, **kw)),\n])\n"
},
{
"path": "phishpedia.py",
"content": "import time\nfrom datetime import datetime\nimport argparse\nimport os\nimport torch\nimport cv2\nfrom configs import load_config\nfrom logo_recog import pred_rcnn, vis\nfrom logo_matching import check_domain_brand_inconsistency\nfrom tqdm import tqdm\n\nimport re\n\nos.environ['KMP_DUPLICATE_LIB_OK'] = 'True'\n\n\ndef result_file_write(f, folder, url, phish_category, pred_target, matched_domain, siamese_conf, logo_recog_time,\n logo_match_time):\n f.write(folder + \"\\t\")\n f.write(url + \"\\t\")\n f.write(str(phish_category) + \"\\t\")\n f.write(str(pred_target) + \"\\t\") # write top1 prediction only\n f.write(str(matched_domain) + \"\\t\")\n f.write(str(siamese_conf) + \"\\t\")\n f.write(str(round(logo_recog_time, 4)) + \"\\t\")\n f.write(str(round(logo_match_time, 4)) + \"\\n\")\n\n\nclass PhishpediaWrapper:\n _caller_prefix = \"PhishpediaWrapper\"\n _DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n\n def __init__(self):\n self._load_config()\n\n def _load_config(self):\n self.ELE_MODEL, self.SIAMESE_THRE, self.SIAMESE_MODEL, \\\n self.LOGO_FEATS, self.LOGO_FILES, \\\n self.DOMAIN_MAP_PATH = load_config()\n print(f'Length of reference list = {len(self.LOGO_FEATS)}')\n\n def test_orig_phishpedia(self, url, screenshot_path, html_path):\n # 0 for benign, 1 for phish, default is benign\n phish_category = 0\n pred_target = None\n matched_domain = None\n siamese_conf = None\n plotvis = None\n logo_match_time = 0\n print(\"Entering phishpedia\")\n\n ####################### Step1: Logo detector ##############################################\n start_time = time.time()\n pred_boxes = pred_rcnn(im=screenshot_path, predictor=self.ELE_MODEL)\n logo_recog_time = time.time() - start_time\n\n if pred_boxes is not None:\n pred_boxes = pred_boxes.detach().cpu().numpy()\n plotvis = vis(screenshot_path, pred_boxes)\n\n # If no element is reported\n if pred_boxes is None or len(pred_boxes) == 0:\n print('No logo is detected')\n return phish_category, pred_target, matched_domain, plotvis, siamese_conf, pred_boxes, logo_recog_time, logo_match_time\n\n ######################## Step2: Siamese (Logo matcher) ########################################\n start_time = time.time()\n pred_target, matched_domain, matched_coord, siamese_conf = check_domain_brand_inconsistency(\n logo_boxes=pred_boxes,\n domain_map_path=self.DOMAIN_MAP_PATH,\n model=self.SIAMESE_MODEL,\n logo_feat_list=self.LOGO_FEATS,\n file_name_list=self.LOGO_FILES,\n url=url,\n shot_path=screenshot_path,\n similarity_threshold=self.SIAMESE_THRE,\n topk=1)\n logo_match_time = time.time() - start_time\n\n if pred_target is None:\n print('Did not match to any brand, report as benign')\n return phish_category, pred_target, matched_domain, plotvis, siamese_conf, pred_boxes, logo_recog_time, logo_match_time\n\n print('Match to Target: {} with confidence {:.4f}'.format(pred_target, siamese_conf))\n phish_category = 1\n # Visualize, add annotations\n cv2.putText(plotvis, \"Target: {} with confidence {:.4f}\".format(pred_target, siamese_conf),\n (int(matched_coord[0] + 20), int(matched_coord[1] + 20)),\n cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 0, 0), 2)\n\n return phish_category, pred_target, matched_domain, plotvis, siamese_conf, pred_boxes, logo_recog_time, logo_match_time\n\n\nif __name__ == '__main__':\n\n '''run'''\n today = datetime.now().strftime('%Y%m%d')\n\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--folder\", required=True, type=str)\n parser.add_argument(\"--output_txt\", default=f'{today}_results.txt', help=\"Output txt path\")\n args = parser.parse_args()\n\n request_dir = args.folder\n phishpedia_cls = PhishpediaWrapper()\n result_txt = args.output_txt\n\n os.makedirs(request_dir, exist_ok=True)\n\n for folder in tqdm(os.listdir(request_dir)):\n html_path = os.path.join(request_dir, folder, \"html.txt\")\n screenshot_path = os.path.join(request_dir, folder, \"shot.png\")\n info_path = os.path.join(request_dir, folder, 'info.txt')\n\n if not os.path.exists(screenshot_path):\n continue\n if not os.path.exists(html_path):\n html_path = os.path.join(request_dir, folder, \"index.html\")\n\n with open(info_path, 'r') as file:\n url = file.read()\n \n if os.path.exists(result_txt):\n with open(result_txt, 'r', encoding='ISO-8859-1') as file:\n if url in file.read():\n continue\n\n _forbidden_suffixes = r\"\\.(mp3|wav|wma|ogg|mkv|zip|tar|xz|rar|z|deb|bin|iso|csv|tsv|dat|txt|css|log|xml|sql|mdb|apk|bat|exe|jar|wsf|fnt|fon|otf|ttf|ai|bmp|gif|ico|jp(e)?g|png|ps|psd|svg|tif|tiff|cer|rss|key|odp|pps|ppt|pptx|c|class|cpp|cs|h|java|sh|swift|vb|odf|xlr|xls|xlsx|bak|cab|cfg|cpl|cur|dll|dmp|drv|icns|ini|lnk|msi|sys|tmp|3g2|3gp|avi|flv|h264|m4v|mov|mp4|mp(e)?g|rm|swf|vob|wmv|doc(x)?|odt|rtf|tex|wks|wps|wpd)$\"\n if re.search(_forbidden_suffixes, url, re.IGNORECASE):\n continue\n\n phish_category, pred_target, matched_domain, \\\n plotvis, siamese_conf, pred_boxes, \\\n logo_recog_time, logo_match_time = phishpedia_cls.test_orig_phishpedia(url, screenshot_path, html_path)\n\n try:\n with open(result_txt, \"a+\", encoding='ISO-8859-1') as f:\n result_file_write(f, folder, url, phish_category, pred_target, matched_domain, siamese_conf,\n logo_recog_time, logo_match_time)\n except UnicodeError:\n with open(result_txt, \"a+\", encoding='utf-8') as f:\n result_file_write(f, folder, url, phish_category, pred_target, matched_domain, siamese_conf,\n logo_recog_time, logo_match_time)\n if phish_category:\n os.makedirs(os.path.join(request_dir, folder), exist_ok=True)\n cv2.imwrite(os.path.join(request_dir, folder, \"predict.png\"), plotvis)\n \n"
},
{
"path": "pixi.toml",
"content": "[project]\nname = \"phishpedia\"\nchannels = [\"conda-forge\"]\nplatforms = [\"osx-arm64\", \"linux-64\", \"win-64\"]\n\n[dependencies]\npython = \">=3.8\"\npip = \"*\"\nsetuptools = \"*\"\nwheel = \"*\"\nnumpy = \"1.23.0\"\nrequests = \"*\"\nscikit-learn = \"*\"\nspacy = \"*\"\nbeautifulsoup4 = \"*\"\nmatplotlib = \"*\"\npandas = \"*\"\nnltk = \"*\"\ntqdm = \"*\"\nunidecode = \"*\"\ngdown = \"*\"\ntldextract = \"*\"\nscipy = \"*\"\npathlib = \"*\"\nfvcore = \"*\"\nlxml = \"*\"\npsutil = \"*\"\nPillow = \"8.4.0\"\n\n\n[pypi-dependencies]\n\"flask\" = \"*\"\n\"flask-cors\" = \"*\"\n\"pycocotools\" = \"*\"\n\"opencv-python\"= \"*\"\n\"opencv-contrib-python\"= \"*\"\ntorch = { version = \">=1.9.0\", index = \"https://download.pytorch.org/whl/cpu\" }\ntorchvision = { version = \">=0.10.0\", index = \"https://download.pytorch.org/whl/cpu\" }\n\n"
},
{
"path": "setup.bat",
"content": "@echo off\nsetlocal enabledelayedexpansion\n\n:: ------------------------------------------------------------------------------\n:: Initialization and Logging\n:: ------------------------------------------------------------------------------\necho [%DATE% %TIME%] Starting setup...\n\n:: ------------------------------------------------------------------------------\n:: Tool Checks\n:: ------------------------------------------------------------------------------\nwhere pixi >nul 2>nul || (\n echo [ERROR] pixi not found. Please install Pixi.\n exit /b 1\n)\nwhere gdown >nul 2>nul || (\n echo [ERROR] gdown not found. Please install gdown (via pixi).\n exit /b 1\n)\nwhere unzip >nul 2>nul || (\n echo [ERROR] unzip not found. Please install unzip utility.\n exit /b 1\n)\n\n:: ------------------------------------------------------------------------------\n:: Setup Directories\n:: ------------------------------------------------------------------------------\nset \"FILEDIR=%cd%\"\nset \"MODELS_DIR=%FILEDIR%\\models\"\nif not exist \"%MODELS_DIR%\" mkdir \"%MODELS_DIR%\"\ncd /d \"%MODELS_DIR%\"\n\n:: ------------------------------------------------------------------------------\n:: Install Detectron2\n:: ------------------------------------------------------------------------------\necho [%DATE% %TIME%] Installing detectron2...\npixi run pip install --no-build-isolation git+https://github.com/facebookresearch/detectron2.git || (\n echo [ERROR] Failed to install detectron2.\n exit /b 1\n)\n\n:: ------------------------------------------------------------------------------\n:: File Metadata\n:: ------------------------------------------------------------------------------\nset RETRY_COUNT=3\n\n:: Model files and Google Drive IDs\nset file1=rcnn_bet365.pth\nset id1=1tE2Mu5WC8uqCxei3XqAd7AWaP5JTmVWH\n\nset file2=faster_rcnn.yaml\nset id2=1Q6lqjpl4exW7q_dPbComcj0udBMDl8CW\n\nset file3=resnetv2_rgb_new.pth.tar\nset id3=1H0Q_DbdKPLFcZee8I14K62qV7TTy7xvS\n\nset file4=expand_targetlist.zip\nset id4=1fr5ZxBKyDiNZ_1B6rRAfZbAHBBoUjZ7I\n\nset file5=domain_map.pkl\nset id5=1qSdkSSoCYUkZMKs44Rup_1DPBxHnEKl1\n\n:: ------------------------------------------------------------------------------\n:: Download Loop\n:: ------------------------------------------------------------------------------\nfor /L %%i in (1,1,5) do (\n call set \"FILENAME=%%file%%i%%\"\n call set \"FILEID=%%id%%i%%\"\n\n if exist \"!FILENAME!\" (\n echo [INFO] !FILENAME! already exists. Skipping.\n ) else (\n set /A count=1\n :retry_%%i\n echo [%DATE% %TIME%] Downloading !FILENAME! (Attempt !count!/%RETRY_COUNT%)...\n pixi run gdown --id !FILEID! -O \"!FILENAME!\" && goto downloaded_%%i\n\n set /A count+=1\n if !count! LEQ %RETRY_COUNT% (\n timeout /t 2 >nul\n goto retry_%%i\n ) else (\n echo [ERROR] Failed to download !FILENAME! after %RETRY_COUNT% attempts.\n exit /b 1\n )\n :downloaded_%%i\n )\n)\n\n:: ------------------------------------------------------------------------------\n:: Extraction\n:: ------------------------------------------------------------------------------\necho [%DATE% %TIME%] Extracting expand_targetlist.zip...\nunzip -o expand_targetlist.zip -d expand_targetlist || (\n echo [ERROR] Failed to unzip file.\n exit /b 1\n)\n\n:: Flatten nested folder if necessary\ncd expand_targetlist\nif exist expand_targetlist\\ (\n echo [INFO] Flattening nested expand_targetlist directory...\n move expand_targetlist\\*.* . >nul\n rmdir expand_targetlist\n)\n\n:: ------------------------------------------------------------------------------\n:: Done\n:: ------------------------------------------------------------------------------\necho [%DATE% %TIME%] [SUCCESS] Model setup and extraction complete.\nendlocal\n"
},
{
"path": "setup.sh",
"content": "#!/bin/bash\n\nset -euo pipefail # Safer bash behavior\nIFS=$'\\n\\t'\n\n# Install Detectron2\npixi run pip install --no-build-isolation git+https://github.com/facebookresearch/detectron2.git\n\n# Set up model directory\nFILEDIR=\"$(pwd)\"\nMODELS_DIR=\"$FILEDIR/models\"\nmkdir -p \"$MODELS_DIR\"\ncd \"$MODELS_DIR\"\n\n# Download model files\npixi run gdown --id \"1tE2Mu5WC8uqCxei3XqAd7AWaP5JTmVWH\" -O \"rcnn_bet365.pth\"\npixi run gdown --id \"1Q6lqjpl4exW7q_dPbComcj0udBMDl8CW\" -O \"faster_rcnn.yaml\"\npixi run gdown --id \"1H0Q_DbdKPLFcZee8I14K62qV7TTy7xvS\" -O \"resnetv2_rgb_new.pth.tar\"\npixi run gdown --id \"1fr5ZxBKyDiNZ_1B6rRAfZbAHBBoUjZ7I\" -O \"expand_targetlist.zip\"\npixi run gdown --id \"1qSdkSSoCYUkZMKs44Rup_1DPBxHnEKl1\" -O \"domain_map.pkl\"\n\n# Extract and flatten expand_targetlist\necho \"Extracting expand_targetlist.zip...\"\nunzip -o expand_targetlist.zip -d expand_targetlist\n\ncd expand_targetlist || error_exit \"Extraction directory missing.\"\n\nif [ -d \"expand_targetlist\" ]; then\n echo \"Flattening nested expand_targetlist/ directory...\"\n mv expand_targetlist/* .\n rm -r expand_targetlist\nfi\n\necho \"Model setup and extraction complete.\"\n"
},
{
"path": "utils.py",
"content": "import torch.nn.functional as F\nimport math\n\n\ndef resolution_alignment(img1, img2):\n '''\n Resize two images according to the minimum resolution between the two\n :param img1: first image in PIL.Image\n :param img2: second image in PIL.Image\n :return: resized img1 in PIL.Image, resized img2 in PIL.Image\n '''\n w1, h1 = img1.size\n w2, h2 = img2.size\n w_min, h_min = min(w1, w2), min(h1, h2)\n if w_min == 0 or h_min == 0: # something wrong, stop resizing\n return img1, img2\n if w_min < h_min:\n img1_resize = img1.resize((int(w_min), math.ceil(h1 * (w_min / w1)))) # ceiling to prevent rounding to 0\n img2_resize = img2.resize((int(w_min), math.ceil(h2 * (w_min / w2))))\n else:\n img1_resize = img1.resize((math.ceil(w1 * (h_min / h1)), int(h_min)))\n img2_resize = img2.resize((math.ceil(w2 * (h_min / h2)), int(h_min)))\n return img1_resize, img2_resize\n\n\ndef brand_converter(brand_name):\n '''\n Helper function to deal with inconsistency in brand naming\n '''\n brand_tran_dict = {'Adobe Inc.': 'Adobe', 'Adobe Inc': 'Adobe',\n 'ADP, LLC': 'ADP', 'ADP, LLC.': 'ADP',\n 'Amazon.com Inc.': 'Amazon', 'Amazon.com Inc': 'Amazon',\n 'Americanas.com S,A Comercio Electrnico': 'Americanas.com S',\n 'AOL Inc.': 'AOL', 'AOL Inc': 'AOL',\n 'Apple Inc.': 'Apple', 'Apple Inc': 'Apple',\n 'AT&T Inc.': 'AT&T', 'AT&T Inc': 'AT&T',\n 'Banco do Brasil S.A.': 'Banco do Brasil S.A',\n 'Credit Agricole S.A.': 'Credit Agricole S.A',\n 'DGI (French Tax Authority)': 'DGI French Tax Authority',\n 'DHL Airways, Inc.': 'DHL Airways', 'DHL Airways, Inc': 'DHL Airways', 'DHL': 'DHL Airways',\n 'Dropbox, Inc.': 'Dropbox', 'Dropbox, Inc': 'Dropbox',\n 'eBay Inc.': 'eBay', 'eBay Inc': 'eBay',\n 'Facebook, Inc.': 'Facebook', 'Facebook, Inc': 'Facebook',\n 'Free (ISP)': 'Free ISP',\n 'Google Inc.': 'Google', 'Google Inc': 'Google',\n 'Mastercard International Incorporated': 'Mastercard International',\n 'Netflix Inc.': 'Netflix', 'Netflix Inc': 'Netflix',\n 'PayPal Inc.': 'PayPal', 'PayPal Inc': 'PayPal',\n 'Royal KPN N.V.': 'Royal KPN N.V',\n 'SF Express Co.': 'SF Express Co',\n 'SNS Bank N.V.': 'SNS Bank N.V',\n 'Square, Inc.': 'Square', 'Square, Inc': 'Square',\n 'Webmail Providers': 'Webmail Provider',\n 'Yahoo! Inc': 'Yahoo!', 'Yahoo! Inc.': 'Yahoo!',\n 'Microsoft OneDrive': 'Microsoft', 'Office365': 'Microsoft', 'Outlook': 'Microsoft',\n 'Global Sources (HK)': 'Global Sources HK',\n 'T-Online': 'Deutsche Telekom',\n 'Airbnb, Inc': 'Airbnb, Inc.',\n 'azul': 'Azul',\n 'Raiffeisen Bank S.A': 'Raiffeisen Bank S.A.',\n 'Twitter, Inc': 'Twitter, Inc.', 'Twitter': 'Twitter, Inc.',\n 'capital_one': 'Capital One Financial Corporation',\n 'la_banque_postale': 'La Banque postale',\n 'db': 'Deutsche Bank AG',\n 'Swiss Post': 'PostFinance', 'PostFinance': 'PostFinance',\n 'grupo_bancolombia': 'Bancolombia',\n 'barclays': 'Barclays Bank Plc',\n 'gov_uk': 'Government of the United Kingdom',\n 'Aruba S.p.A': 'Aruba S.p.A.',\n 'TSB Bank Plc': 'TSB Bank Limited',\n 'strato': 'Strato AG',\n 'cogeco': 'Cogeco',\n 'Canada Revenue Agency': 'Government of Canada',\n 'UniCredit Bulbank': 'UniCredit Bank Aktiengesellschaft',\n 'ameli_fr': 'French Health Insurance',\n 'Banco de Credito del Peru': 'bcp'\n }\n # find the value in the dict else return the origin brand name\n tran_brand_name = brand_tran_dict.get(brand_name, None)\n if tran_brand_name:\n return tran_brand_name\n else:\n return brand_name\n\n\ndef l2_norm(x):\n \"\"\"\n l2 normalization\n :param x:\n :return:\n \"\"\"\n if len(x.shape):\n x = x.reshape((x.shape[0], -1))\n return F.normalize(x, p=2, dim=1)\n"
}
]\n
\n \n \n
\n \n \n \n
\n \n \n
\n \n \n \n \n \n \n
\n Or ctrl+v here
\n \n\n ![\"Uploaded]()
\n \n
\n \n \n Uploaded Successfully!\n
\n \n
\n \n
\n \n Logo Extraction\n
\n \n \n
\n \n Detection Result\n
\n \n
\n \n 📊\n Result\n \n
\n \n \n
\n - \n
- \n 🏷️\n Matched Brand\n \n \n
- \n 💬\n Siamese Confidence\n \n \n
- \n 🌐\n Correct Domain\n \n \n
- \n ⏱️\n Detection Time\n \n \n
- \n \n \n