Fresh fruits
Shop Now
#### Happy Coding ♥️
================================================
FILE: autoscraper/__init__.py
================================================
from autoscraper.auto_scraper import AutoScraper
================================================
FILE: autoscraper/auto_scraper.py
================================================
import hashlib
import json
from collections import defaultdict
from html import unescape
from urllib.parse import urljoin, urlparse
import requests
from bs4 import BeautifulSoup
from autoscraper.utils import (
FuzzyText,
ResultItem,
get_non_rec_text,
normalize,
text_match,
unique_hashable,
unique_stack_list,
)
class AutoScraper(object):
"""
AutoScraper : A Smart, Automatic, Fast and Lightweight Web Scraper for Python.
AutoScraper automatically learns a set of rules required to extract the needed content
from a web page. So the programmer doesn't need to explicitly construct the rules.
Attributes
----------
stack_list: list
List of rules learned by AutoScraper
Methods
-------
build() - Learns a set of rules represented as stack_list based on the wanted_list,
which can be reused for scraping similar elements from other web pages in the future.
get_result_similar() - Gets similar results based on the previously learned rules.
get_result_exact() - Gets exact results based on the previously learned rules.
get_results() - Gets exact and similar results based on the previously learned rules.
save() - Serializes the stack_list as JSON and saves it to disk.
load() - De-serializes the JSON representation of the stack_list and loads it back.
remove_rules() - Removes one or more learned rule[s] from the stack_list.
keep_rules() - Keeps only the specified learned rules in the stack_list and removes the others.
"""
request_headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 \
(KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36"
}
def __init__(self, stack_list=None):
self.stack_list = stack_list or []
def save(self, file_path):
"""
Serializes the stack_list as JSON and saves it to the disk.
Parameters
----------
file_path: str
Path of the JSON output
Returns
-------
None
"""
data = dict(stack_list=self.stack_list)
with open(file_path, "w") as f:
json.dump(data, f)
def load(self, file_path):
"""
De-serializes the JSON representation of the stack_list and loads it back.
Parameters
----------
file_path: str
Path of the JSON file to load stack_list from.
Returns
-------
None
"""
with open(file_path, "r") as f:
data = json.load(f)
# for backward compatibility
if isinstance(data, list):
self.stack_list = data
return
self.stack_list = data["stack_list"]
@classmethod
def _fetch_html(cls, url, request_args=None):
request_args = request_args or {}
headers = dict(cls.request_headers)
if url:
headers["Host"] = urlparse(url).netloc
user_headers = request_args.pop("headers", {})
headers.update(user_headers)
res = requests.get(url, headers=headers, **request_args)
if res.encoding == "ISO-8859-1" and not "ISO-8859-1" in res.headers.get(
"Content-Type", ""
):
res.encoding = res.apparent_encoding
html = res.text
return html
@classmethod
def _get_soup(cls, url=None, html=None, request_args=None):
if html:
html = normalize(unescape(html))
return BeautifulSoup(html, "lxml")
html = cls._fetch_html(url, request_args)
html = normalize(unescape(html))
return BeautifulSoup(html, "lxml")
@staticmethod
def _get_valid_attrs(item):
key_attrs = {"class", "style"}
attrs = {
k: v if v != [] else "" for k, v in item.attrs.items() if k in key_attrs
}
for attr in key_attrs:
if attr not in attrs:
attrs[attr] = ""
return attrs
@staticmethod
def _child_has_text(child, text, url, text_fuzz_ratio):
child_text = child.getText().strip()
if text_match(text, child_text, text_fuzz_ratio):
parent_text = child.parent.getText().strip()
if child_text == parent_text and child.parent.parent:
return False
child.wanted_attr = None
return True
if text_match(text, get_non_rec_text(child), text_fuzz_ratio):
child.is_non_rec_text = True
child.wanted_attr = None
return True
for key, value in child.attrs.items():
if not isinstance(value, str):
continue
value = value.strip()
if text_match(text, value, text_fuzz_ratio):
child.wanted_attr = key
return True
if key in {"href", "src"}:
full_url = urljoin(url, value)
if text_match(text, full_url, text_fuzz_ratio):
child.wanted_attr = key
child.is_full_url = True
return True
return False
def _get_children(self, soup, text, url, text_fuzz_ratio):
children = reversed(soup.findChildren())
children = [
x for x in children if self._child_has_text(x, text, url, text_fuzz_ratio)
]
return children
def build(
self,
url=None,
wanted_list=None,
wanted_dict=None,
html=None,
request_args=None,
update=False,
text_fuzz_ratio=1.0,
):
"""
Automatically constructs a set of rules to scrape the specified target[s] from a web page.
The rules are represented as stack_list.
Parameters:
----------
url: str, optional
URL of the target web page. You should either pass url or html or both.
wanted_list: list of strings or compiled regular expressions, optional
A list of needed contents to be scraped.
AutoScraper learns a set of rules to scrape these targets. If specified,
wanted_dict will be ignored.
wanted_dict: dict, optional
A dict of needed contents to be scraped. Keys are aliases and values are list of target texts
or compiled regular expressions.
AutoScraper learns a set of rules to scrape these targets and sets its aliases.
html: str, optional
An HTML string can also be passed instead of URL.
You should either pass url or html or both.
request_args: dict, optional
A dictionary used to specify a set of additional request parameters used by requests
module. You can specify proxy URLs, custom headers etc.
update: bool, optional, defaults to False
If True, new learned rules will be added to the previous ones.
If False, all previously learned rules will be removed.
text_fuzz_ratio: float in range [0, 1], optional, defaults to 1.0
The fuzziness ratio threshold for matching the wanted contents.
Returns:
--------
List of similar results
"""
if not wanted_list and not (wanted_dict and any(wanted_dict.values())):
raise ValueError("No targets were supplied")
soup = self._get_soup(url=url, html=html, request_args=request_args)
result_list = []
if update is False:
self.stack_list = []
if wanted_list:
wanted_dict = {"": wanted_list}
wanted_list = []
for alias, wanted_items in wanted_dict.items():
wanted_items = [normalize(w) for w in wanted_items]
wanted_list += wanted_items
for wanted in wanted_items:
children = self._get_children(soup, wanted, url, text_fuzz_ratio)
for child in children:
result, stack = self._get_result_for_child(child, soup, url)
stack["alias"] = alias
result_list += result
self.stack_list.append(stack)
result_list = [item.text for item in result_list]
result_list = unique_hashable(result_list)
self.stack_list = unique_stack_list(self.stack_list)
return result_list
@classmethod
def _build_stack(cls, child, url):
content = [(child.name, cls._get_valid_attrs(child))]
parent = child
while True:
grand_parent = parent.findParent()
if not grand_parent:
break
children = grand_parent.findAll(
parent.name, cls._get_valid_attrs(parent), recursive=False
)
for i, c in enumerate(children):
if c == parent:
content.insert(
0, (grand_parent.name, cls._get_valid_attrs(grand_parent), i)
)
break
if not grand_parent.parent:
break
parent = grand_parent
wanted_attr = getattr(child, "wanted_attr", None)
is_full_url = getattr(child, "is_full_url", False)
is_non_rec_text = getattr(child, "is_non_rec_text", False)
stack = dict(
content=content,
wanted_attr=wanted_attr,
is_full_url=is_full_url,
is_non_rec_text=is_non_rec_text,
)
stack["url"] = url if is_full_url else ""
stack["hash"] = hashlib.sha256(str(stack).encode("utf-8")).hexdigest()
stack["stack_id"] = "rule_" + stack["hash"][:8]
return stack
def _get_result_for_child(self, child, soup, url):
stack = self._build_stack(child, url)
result = self._get_result_with_stack(stack, soup, url, 1.0)
return result, stack
@staticmethod
def _fetch_result_from_child(child, wanted_attr, is_full_url, url, is_non_rec_text):
if wanted_attr is None:
if is_non_rec_text:
return get_non_rec_text(child)
return child.getText().strip()
if wanted_attr not in child.attrs:
return None
if is_full_url:
return urljoin(url, child.attrs[wanted_attr])
return child.attrs[wanted_attr]
@staticmethod
def _get_fuzzy_attrs(attrs, attr_fuzz_ratio):
attrs = dict(attrs)
for key, val in attrs.items():
if isinstance(val, str) and val:
val = FuzzyText(val, attr_fuzz_ratio)
elif isinstance(val, (list, tuple)):
val = [FuzzyText(x, attr_fuzz_ratio) if x else x for x in val]
attrs[key] = val
return attrs
def _get_result_with_stack(self, stack, soup, url, attr_fuzz_ratio, **kwargs):
parents = [soup]
stack_content = stack["content"]
contain_sibling_leaves = kwargs.get("contain_sibling_leaves", False)
for index, item in enumerate(stack_content):
children = []
if item[0] == "[document]":
continue
for parent in parents:
attrs = item[1]
if attr_fuzz_ratio < 1.0:
attrs = self._get_fuzzy_attrs(attrs, attr_fuzz_ratio)
found = parent.findAll(item[0], attrs, recursive=False)
if not found:
continue
if not contain_sibling_leaves and index == len(stack_content) - 1:
idx = min(len(found) - 1, stack_content[index - 1][2])
found = [found[idx]]
children += found
parents = children
wanted_attr = stack["wanted_attr"]
is_full_url = stack["is_full_url"]
is_non_rec_text = stack.get("is_non_rec_text", False)
result = [
ResultItem(
self._fetch_result_from_child(
i, wanted_attr, is_full_url, url, is_non_rec_text
),
getattr(i, "child_index", 0),
)
for i in parents
]
if not kwargs.get("keep_blank", False):
result = [x for x in result if x.text]
return result
def _get_result_with_stack_index_based(
self, stack, soup, url, attr_fuzz_ratio, **kwargs
):
p = soup.findChildren(recursive=False)[0]
stack_content = stack["content"]
for index, item in enumerate(stack_content[:-1]):
if item[0] == "[document]":
continue
content = stack_content[index + 1]
attrs = content[1]
if attr_fuzz_ratio < 1.0:
attrs = self._get_fuzzy_attrs(attrs, attr_fuzz_ratio)
p = p.findAll(content[0], attrs, recursive=False)
if not p:
return []
idx = min(len(p) - 1, item[2])
p = p[idx]
result = [
ResultItem(
self._fetch_result_from_child(
p,
stack["wanted_attr"],
stack["is_full_url"],
url,
stack["is_non_rec_text"],
),
getattr(p, "child_index", 0),
)
]
if not kwargs.get("keep_blank", False):
result = [x for x in result if x.text]
return result
def _get_result_by_func(
self,
func,
url,
html,
soup,
request_args,
grouped,
group_by_alias,
unique,
attr_fuzz_ratio,
**kwargs
):
if not soup:
soup = self._get_soup(url=url, html=html, request_args=request_args)
keep_order = kwargs.get("keep_order", False)
if group_by_alias or (keep_order and not grouped):
for index, child in enumerate(soup.findChildren()):
setattr(child, "child_index", index)
result_list = []
grouped_result = defaultdict(list)
for stack in self.stack_list:
if not url:
url = stack.get("url", "")
result = func(stack, soup, url, attr_fuzz_ratio, **kwargs)
if not grouped and not group_by_alias:
result_list += result
continue
group_id = stack.get("alias", "") if group_by_alias else stack["stack_id"]
grouped_result[group_id] += result
return self._clean_result(
result_list, grouped_result, grouped, group_by_alias, unique, keep_order
)
@staticmethod
def _clean_result(
result_list, grouped_result, grouped, grouped_by_alias, unique, keep_order
):
if not grouped and not grouped_by_alias:
if unique is None:
unique = True
if keep_order:
result_list = sorted(result_list, key=lambda x: x.index)
result = [x.text for x in result_list]
if unique:
result = unique_hashable(result)
return result
for k, val in grouped_result.items():
if grouped_by_alias:
val = sorted(val, key=lambda x: x.index)
val = [x.text for x in val]
if unique:
val = unique_hashable(val)
grouped_result[k] = val
return dict(grouped_result)
def get_result_similar(
self,
url=None,
html=None,
soup=None,
request_args=None,
grouped=False,
group_by_alias=False,
unique=None,
attr_fuzz_ratio=1.0,
keep_blank=False,
keep_order=False,
contain_sibling_leaves=False,
):
"""
Gets similar results based on the previously learned rules.
Parameters:
----------
url: str, optional
URL of the target web page. You should either pass url or html or both.
html: str, optional
An HTML string can also be passed instead of URL.
You should either pass url or html or both.
request_args: dict, optional
A dictionary used to specify a set of additional request parameters used by requests
module. You can specify proxy URLs, custom headers etc.
grouped: bool, optional, defaults to False
If set to True, the result will be a dictionary with the rule_ids as keys
and a list of scraped data per rule as values.
group_by_alias: bool, optional, defaults to False
If set to True, the result will be a dictionary with the rule alias as keys
and a list of scraped data per alias as values.
unique: bool, optional, defaults to True for non grouped results and
False for grouped results.
If set to True, will remove duplicates from returned result list.
attr_fuzz_ratio: float in range [0, 1], optional, defaults to 1.0
The fuzziness ratio threshold for matching html tag attributes.
keep_blank: bool, optional, defaults to False
If set to True, missing values will be returned as empty strings.
keep_order: bool, optional, defaults to False
If set to True, the results will be ordered as they are present on the web page.
contain_sibling_leaves: bool, optional, defaults to False
If set to True, the results will also contain the sibling leaves of the wanted elements.
Returns:
--------
List of similar results scraped from the web page.
Dictionary if grouped=True or group_by_alias=True.
"""
func = self._get_result_with_stack
return self._get_result_by_func(
func,
url,
html,
soup,
request_args,
grouped,
group_by_alias,
unique,
attr_fuzz_ratio,
keep_blank=keep_blank,
keep_order=keep_order,
contain_sibling_leaves=contain_sibling_leaves,
)
def get_result_exact(
self,
url=None,
html=None,
soup=None,
request_args=None,
grouped=False,
group_by_alias=False,
unique=None,
attr_fuzz_ratio=1.0,
keep_blank=False,
):
"""
Gets exact results based on the previously learned rules.
Parameters:
----------
url: str, optional
URL of the target web page. You should either pass url or html or both.
html: str, optional
An HTML string can also be passed instead of URL.
You should either pass url or html or both.
request_args: dict, optional
A dictionary used to specify a set of additional request parameters used by requests
module. You can specify proxy URLs, custom headers etc.
grouped: bool, optional, defaults to False
If set to True, the result will be a dictionary with the rule_ids as keys
and a list of scraped data per rule as values.
group_by_alias: bool, optional, defaults to False
If set to True, the result will be a dictionary with the rule alias as keys
and a list of scraped data per alias as values.
unique: bool, optional, defaults to True for non grouped results and
False for grouped results.
If set to True, will remove duplicates from returned result list.
attr_fuzz_ratio: float in range [0, 1], optional, defaults to 1.0
The fuzziness ratio threshold for matching html tag attributes.
keep_blank: bool, optional, defaults to False
If set to True, missing values will be returned as empty strings.
Returns:
--------
List of exact results scraped from the web page.
Dictionary if grouped=True or group_by_alias=True.
"""
func = self._get_result_with_stack_index_based
return self._get_result_by_func(
func,
url,
html,
soup,
request_args,
grouped,
group_by_alias,
unique,
attr_fuzz_ratio,
keep_blank=keep_blank,
)
def get_result(
self,
url=None,
html=None,
request_args=None,
grouped=False,
group_by_alias=False,
unique=None,
attr_fuzz_ratio=1.0,
):
"""
Gets similar and exact results based on the previously learned rules.
Parameters:
----------
url: str, optional
URL of the target web page. You should either pass url or html or both.
html: str, optional
An HTML string can also be passed instead of URL.
You should either pass url or html or both.
request_args: dict, optional
A dictionary used to specify a set of additional request parameters used by requests
module. You can specify proxy URLs, custom headers etc.
grouped: bool, optional, defaults to False
If set to True, the result will be dictionaries with the rule_ids as keys
and a list of scraped data per rule as values.
group_by_alias: bool, optional, defaults to False
If set to True, the result will be a dictionary with the rule alias as keys
and a list of scraped data per alias as values.
unique: bool, optional, defaults to True for non grouped results and
False for grouped results.
If set to True, will remove duplicates from returned result list.
attr_fuzz_ratio: float in range [0, 1], optional, defaults to 1.0
The fuzziness ratio threshold for matching html tag attributes.
Returns:
--------
Pair of (similar, exact) results.
See get_result_similar and get_result_exact methods.
"""
soup = self._get_soup(url=url, html=html, request_args=request_args)
args = dict(
url=url,
soup=soup,
grouped=grouped,
group_by_alias=group_by_alias,
unique=unique,
attr_fuzz_ratio=attr_fuzz_ratio,
)
similar = self.get_result_similar(**args)
exact = self.get_result_exact(**args)
return similar, exact
def remove_rules(self, rules):
"""
Removes a list of learned rules from stack_list.
Parameters:
----------
rules : list
A list of rules to be removed
Returns:
--------
None
"""
self.stack_list = [x for x in self.stack_list if x["stack_id"] not in rules]
def keep_rules(self, rules):
"""
Removes all other rules except the specified ones.
Parameters:
----------
rules : list
A list of rules to keep in stack_list and removing the rest.
Returns:
--------
None
"""
self.stack_list = [x for x in self.stack_list if x["stack_id"] in rules]
def set_rule_aliases(self, rule_aliases):
"""
Sets the specified alias for each rule
Parameters:
----------
rule_aliases : dict
A dictionary with keys of rule_id and values of alias
Returns:
--------
None
"""
id_to_stack = {stack["stack_id"]: stack for stack in self.stack_list}
for rule_id, alias in rule_aliases.items():
id_to_stack[rule_id]["alias"] = alias
def generate_python_code(self):
# deprecated
print("This function is deprecated. Please use save() and load() instead.")
================================================
FILE: autoscraper/utils.py
================================================
from collections import OrderedDict
import unicodedata
from difflib import SequenceMatcher
def unique_stack_list(stack_list):
seen = set()
unique_list = []
for stack in stack_list:
stack_hash = stack['hash']
if stack_hash in seen:
continue
unique_list.append(stack)
seen.add(stack_hash)
return unique_list
def unique_hashable(hashable_items):
"""Removes duplicates from the list. Must preserve the orders."""
return list(OrderedDict.fromkeys(hashable_items))
def get_non_rec_text(element):
return ''.join(element.find_all(text=True, recursive=False)).strip()
def normalize(item):
if not isinstance(item, str):
return item
return unicodedata.normalize("NFKD", item.strip())
def text_match(t1, t2, ratio_limit):
if hasattr(t1, 'fullmatch'):
return bool(t1.fullmatch(t2))
if ratio_limit >= 1:
return t1 == t2
return SequenceMatcher(None, t1, t2).ratio() >= ratio_limit
class ResultItem():
def __init__(self, text, index):
self.text = text
self.index = index
def __str__(self):
return self.text
class FuzzyText(object):
def __init__(self, text, ratio_limit):
self.text = text
self.ratio_limit = ratio_limit
self.match = None
def search(self, text):
return SequenceMatcher(None, self.text, text).ratio() >= self.ratio_limit
================================================
FILE: setup.py
================================================
from codecs import open
from os import path
from setuptools import find_packages, setup
here = path.abspath(path.dirname(__file__))
with open(path.join(here, "README.md"), encoding="utf-8") as f:
long_description = f.read()
setup(
name="autoscraper",
version="1.1.14",
description="A Smart, Automatic, Fast and Lightweight Web Scraper for Python",
long_description_content_type="text/markdown",
long_description=long_description,
url="https://github.com/alirezamika/autoscraper",
author="Alireza Mika",
author_email="alirezamika@gmail.com",
license="MIT",
classifiers=[
"Development Status :: 4 - Beta",
"License :: OSI Approved :: MIT License",
"Programming Language :: Python :: 3",
],
keywords="scraping - scraper",
packages=find_packages(exclude=["contrib", "docs", "tests"]),
python_requires=">=3.6",
install_requires=["requests", "bs4", "lxml"],
)
================================================
FILE: tests/__init__.py
================================================
================================================
FILE: tests/conftest.py
================================================
import sys
from types import ModuleType
from html.parser import HTMLParser
class _Node:
def __init__(self, name, attrs, parent=None):
self.name = name
self.attrs = dict(attrs)
self.parent = parent
self.children = []
self.text = ""
def append_child(self, child):
self.children.append(child)
child.parent = self
def getText(self):
return self.text + "".join(c.getText() for c in self.children)
def findChildren(self, recursive=True):
result = []
for child in self.children:
result.append(child)
if recursive:
result.extend(child.findChildren(recursive))
return result
def findParent(self):
return self.parent
def _attr_match(self, child, attrs):
from autoscraper.utils import FuzzyText
for key, val in (attrs or {}).items():
actual = child.attrs.get(key, "")
if isinstance(actual, list):
actual = " ".join(actual)
if isinstance(val, FuzzyText):
if not val.search(actual):
return False
elif actual != val:
return False
return True
def findAll(self, name=None, attrs=None, recursive=True):
result = []
for child in self.children:
if (name is None or child.name == name) and self._attr_match(child, attrs):
result.append(child)
if recursive:
result.extend(child.findAll(name, attrs, recursive))
return result
def find_all(self, name=None, attrs=None, text=None, recursive=True):
if text:
res = []
if self.text.strip():
res.append(self.text)
for child in self.children:
if recursive:
res.extend(child.find_all(text=True, recursive=True))
elif child.text.strip():
res.append(child.text)
return res
return self.findAll(name, attrs, recursive)
class _Parser(HTMLParser):
def __init__(self):
super().__init__()
self.root = _Node("[document]", {})
self.current = self.root
def handle_starttag(self, tag, attrs):
node = _Node(tag, attrs)
self.current.append_child(node)
self.current = node
def handle_endtag(self, tag):
if self.current.parent:
self.current = self.current.parent
def handle_data(self, data):
self.current.text += data
class BeautifulSoup(_Node):
def __init__(self, html, parser):
p = _Parser()
p.feed(html)
super().__init__(p.root.name, p.root.attrs)
self.children = p.root.children
for c in self.children:
c.parent = self
bs4_mod = ModuleType("bs4")
bs4_mod.BeautifulSoup = BeautifulSoup
sys.modules.setdefault("bs4", bs4_mod)
class _Response:
def __init__(self, text=""):
self.encoding = "utf-8"
self.headers = {"Content-Type": "text/html"}
self.text = text
requests_mod = ModuleType("requests")
requests_mod.get = lambda url, headers=None, **kw: _Response()
sys.modules.setdefault("requests", requests_mod)
================================================
FILE: tests/integration/__init__.py
================================================
================================================
FILE: tests/integration/test_complex_features.py
================================================
import pytest
import re
from autoscraper import AutoScraper
HTML_COMPLEX = """
"""
def test_extract_relative_link():
scraper = AutoScraper()
url = "https://example.com/index.html"
result = scraper.build(url=url, html=HTML_COMPLEX, wanted_list=["https://example.com/apple"])
assert "https://example.com/apple" in result
similar = scraper.get_result_similar(
url=url, html=HTML_COMPLEX, contain_sibling_leaves=True, unique=True
)
assert set(similar) == {
"https://example.com/banana",
"https://example.com/apple",
"https://example.com/orange",
}
exact = scraper.get_result_exact(url=url, html=HTML_COMPLEX)
assert exact == ["https://example.com/apple"]
def test_build_with_regex():
scraper = AutoScraper()
scraper.build(html=HTML_COMPLEX, wanted_list=[re.compile("Ban.*")])
result = scraper.get_result_exact(html=HTML_COMPLEX)
assert "Banana" in result[0]
def test_update_appends_rules():
scraper = AutoScraper()
scraper.build(html=HTML_COMPLEX, wanted_list=["Banana"])
count = len(scraper.stack_list)
scraper.build(html=HTML_COMPLEX, wanted_list=["Apple"], update=True)
assert len(scraper.stack_list) == count + 1
def test_remove_rules():
scraper = AutoScraper()
scraper.build(html=HTML_COMPLEX, wanted_list=["Banana"])
scraper.build(html=HTML_COMPLEX, wanted_list=["Apple"], update=True)
rule_ids = [s["stack_id"] for s in scraper.stack_list]
to_remove = rule_ids[0]
scraper.remove_rules([to_remove])
remaining = [s["stack_id"] for s in scraper.stack_list]
assert to_remove not in remaining
assert len(remaining) == len(rule_ids) - 1
def test_keep_blank_returns_empty():
scraper = AutoScraper()
scraper.build(html=HTML_COMPLEX, wanted_list=["/shop"])
html_blank = HTML_COMPLEX.replace('href="/shop"', 'href=""')
result = scraper.get_result_exact(html=html_blank, keep_blank=True)
assert result == [""]
def test_attr_fuzz_ratio():
html_base = ''
html_variant = ''
scraper = AutoScraper()
scraper.build(html=html_base, wanted_list=["Buy"])
res = scraper.get_result_exact(html=html_variant, attr_fuzz_ratio=0.8)
assert res == ["Buy"]
================================================
FILE: tests/integration/test_real_world.py
================================================
import re
from autoscraper import AutoScraper
HTML_PAGE_1 = """
$1
$2