\n \n
![](\"assets/header.svg\")
\n \n \n
\")\n\n\ndef append_bookmark(doc: BookmarkDocument, bookmark: Bookmark):\n url = bookmark.url\n title = html.escape(bookmark.resolved_title or \"\")\n desc = html.escape(bookmark.resolved_description or \"\")\n if bookmark.notes:\n desc += f\"[linkding-notes]{html.escape(bookmark.notes)}[/linkding-notes]\"\n tag_names = bookmark.tag_names\n if bookmark.is_archived:\n tag_names.append(\"linkding:bookmarks.archived\")\n tags = \",\".join(tag_names)\n toread = \"1\" if bookmark.unread else \"0\"\n private = \"0\" if bookmark.shared else \"1\"\n added = int(bookmark.date_added.timestamp())\n modified = int(bookmark.date_modified.timestamp())\n\n doc.append(\n f'
\")\n"
},
{
"path": "bookmarks/services/favicon_loader.py",
"content": "import logging\nimport mimetypes\nimport os.path\nimport re\nimport time\nfrom pathlib import Path\nfrom urllib.parse import urlparse\n\nimport requests\nfrom django.conf import settings\n\nmax_file_age = 60 * 60 * 24 # 1 day\n\nlogger = logging.getLogger(__name__)\n\n# register mime type for .ico files, which is not included in the default\n# mimetypes of the Docker image\nmimetypes.add_type(\"image/x-icon\", \".ico\")\n\n\ndef _ensure_favicon_folder():\n Path(settings.LD_FAVICON_FOLDER).mkdir(parents=True, exist_ok=True)\n\n\ndef _url_to_filename(url: str) -> str:\n return re.sub(r\"\\W+\", \"_\", url)\n\n\ndef _get_url_parameters(url: str) -> dict:\n parsed_uri = urlparse(url)\n return {\n # https://example.com/foo?bar -> https://example.com\n \"url\": f\"{parsed_uri.scheme}://{parsed_uri.hostname}\",\n # https://example.com/foo?bar -> example.com\n \"domain\": parsed_uri.hostname,\n }\n\n\ndef _get_favicon_path(favicon_file: str) -> Path:\n return Path(os.path.join(settings.LD_FAVICON_FOLDER, favicon_file))\n\n\ndef _check_existing_favicon(favicon_name: str):\n # return existing file if a file with the same name, ignoring extension,\n # exists and is not stale\n for filename in os.listdir(settings.LD_FAVICON_FOLDER):\n file_base_name, _ = os.path.splitext(filename)\n if file_base_name == favicon_name:\n favicon_path = _get_favicon_path(filename)\n return filename if not _is_stale(favicon_path) else None\n return None\n\n\ndef _is_stale(path: Path) -> bool:\n stat = path.stat()\n file_age = time.time() - stat.st_mtime\n return file_age >= max_file_age\n\n\ndef load_favicon(url: str) -> str:\n url_parameters = _get_url_parameters(url)\n\n # Create favicon folder if not exists\n _ensure_favicon_folder()\n # Use scheme+hostname as favicon filename to reuse icon for all pages on the same domain\n favicon_name = _url_to_filename(url_parameters[\"url\"])\n favicon_file = _check_existing_favicon(favicon_name)\n\n if not favicon_file:\n # Load favicon from provider, save to file\n favicon_url = settings.LD_FAVICON_PROVIDER.format(**url_parameters)\n logger.debug(f\"Loading favicon from: {favicon_url}\")\n with requests.get(favicon_url, stream=True) as response:\n content_type = response.headers[\"Content-Type\"]\n file_extension = mimetypes.guess_extension(content_type)\n favicon_file = f\"{favicon_name}{file_extension}\"\n favicon_path = _get_favicon_path(favicon_file)\n with open(favicon_path, \"wb\") as file:\n for chunk in response.iter_content(chunk_size=8192):\n file.write(chunk)\n logger.debug(f\"Saved favicon as: {favicon_path}\")\n\n return favicon_file\n"
},
{
"path": "bookmarks/services/importer.py",
"content": "import logging\nfrom dataclasses import dataclass\n\nfrom django.contrib.auth.models import User\nfrom django.utils import timezone\n\nfrom bookmarks.models import Bookmark, Tag\nfrom bookmarks.services import tasks\nfrom bookmarks.services.parser import NetscapeBookmark, parse\nfrom bookmarks.utils import normalize_url, parse_timestamp\n\nlogger = logging.getLogger(__name__)\n\n\n@dataclass\nclass ImportResult:\n total: int = 0\n success: int = 0\n failed: int = 0\n\n\n@dataclass\nclass ImportOptions:\n map_private_flag: bool = False\n\n\nclass TagCache:\n def __init__(self, user: User):\n self.user = user\n self.cache = dict()\n # Init cache with all existing tags for that user\n tags = Tag.objects.filter(owner=user)\n for tag in tags:\n self.put(tag)\n\n def get(self, tag_name: str):\n tag_name_lowercase = tag_name.lower()\n if tag_name_lowercase in self.cache:\n return self.cache[tag_name_lowercase]\n else:\n return None\n\n def get_all(self, tag_names: list[str]):\n result = []\n for tag_name in tag_names:\n tag = self.get(tag_name)\n # Tag may not have been created if tag name exceeded maximum length\n # Prevent returning duplicates\n if tag and tag not in result:\n result.append(tag)\n\n return result\n\n def put(self, tag: Tag):\n self.cache[tag.name.lower()] = tag\n\n\ndef import_netscape_html(\n html: str, user: User, options: ImportOptions | None = None\n) -> ImportResult:\n if options is None:\n options = ImportOptions()\n result = ImportResult()\n import_start = timezone.now()\n\n try:\n netscape_bookmarks = parse(html)\n except Exception:\n logging.exception(\"Could not read bookmarks file.\")\n raise\n\n parse_end = timezone.now()\n logger.debug(f\"Parse duration: {parse_end - import_start}\")\n\n # Create and cache all tags beforehand\n _create_missing_tags(netscape_bookmarks, user)\n tag_cache = TagCache(user)\n\n # Split bookmarks to import into batches, to keep memory usage for bulk operations manageable\n batches = _get_batches(netscape_bookmarks, 200)\n for batch in batches:\n _import_batch(batch, user, options, tag_cache, result)\n\n # Load favicons for newly imported bookmarks\n tasks.schedule_bookmarks_without_favicons(user)\n # Load previews for newly imported bookmarks\n tasks.schedule_bookmarks_without_previews(user)\n\n end = timezone.now()\n logger.debug(f\"Import duration: {end - import_start}\")\n\n return result\n\n\ndef _create_missing_tags(netscape_bookmarks: list[NetscapeBookmark], user: User):\n tag_cache = TagCache(user)\n tags_to_create = []\n\n for netscape_bookmark in netscape_bookmarks:\n for tag_name in netscape_bookmark.tag_names:\n # Skip tag names that exceed the maximum allowed length\n if len(tag_name) > 64:\n logger.warning(\n f\"Ignoring tag '{tag_name}' (length {len(tag_name)}) as it exceeds maximum length of 64 characters\"\n )\n continue\n\n tag = tag_cache.get(tag_name)\n if not tag:\n tag = Tag(name=tag_name, owner=user)\n tag.date_added = timezone.now()\n tags_to_create.append(tag)\n tag_cache.put(tag)\n\n Tag.objects.bulk_create(tags_to_create)\n\n\ndef _get_batches(items: list, batch_size: int):\n batches = []\n offset = 0\n num_items = len(items)\n\n while offset < num_items:\n batch = items[offset : min(offset + batch_size, num_items)]\n if len(batch) > 0:\n batches.append(batch)\n offset = offset + batch_size\n\n return batches\n\n\ndef _import_batch(\n netscape_bookmarks: list[NetscapeBookmark],\n user: User,\n options: ImportOptions,\n tag_cache: TagCache,\n result: ImportResult,\n):\n # Query existing bookmarks\n batch_urls = [bookmark.href for bookmark in netscape_bookmarks]\n existing_bookmarks = Bookmark.objects.filter(owner=user, url__in=batch_urls)\n\n # Create or update bookmarks from parsed Netscape bookmarks\n bookmarks_to_create = []\n bookmarks_to_update = []\n\n for netscape_bookmark in netscape_bookmarks:\n result.total = result.total + 1\n try:\n # Lookup existing bookmark by URL, or create new bookmark if there is no bookmark for that URL yet\n bookmark = next(\n (\n bookmark\n for bookmark in existing_bookmarks\n if bookmark.url == netscape_bookmark.href\n ),\n None,\n )\n if not bookmark:\n bookmark = Bookmark(owner=user)\n is_update = False\n else:\n is_update = True\n # Copy data from parsed bookmark\n _copy_bookmark_data(netscape_bookmark, bookmark, options)\n # Validate bookmark fields, exclude owner to prevent n+1 database query,\n # also there is no specific validation on owner\n bookmark.clean_fields(exclude=[\"owner\"])\n # Schedule for update or insert\n if is_update:\n bookmarks_to_update.append(bookmark)\n else:\n bookmarks_to_create.append(bookmark)\n\n result.success = result.success + 1\n except Exception:\n shortened_bookmark_tag_str = str(netscape_bookmark)[:100] + \"...\"\n logging.exception(\"Error importing bookmark: \" + shortened_bookmark_tag_str)\n result.failed = result.failed + 1\n\n # Bulk update bookmarks in DB\n Bookmark.objects.bulk_update(\n bookmarks_to_update,\n [\n \"url\",\n \"url_normalized\",\n \"date_added\",\n \"date_modified\",\n \"unread\",\n \"shared\",\n \"title\",\n \"description\",\n \"notes\",\n \"owner\",\n ],\n )\n # Bulk insert new bookmarks into DB\n Bookmark.objects.bulk_create(bookmarks_to_create)\n\n # Bulk assign tags\n # In Django 3, bulk_create does not return the auto-generated IDs when bulk inserting,\n # so we have to reload the inserted bookmarks, and match them to the parsed bookmarks by URL\n existing_bookmarks = Bookmark.objects.filter(owner=user, url__in=batch_urls)\n\n BookmarkToTagRelationShip = Bookmark.tags.through\n relationships = []\n\n for netscape_bookmark in netscape_bookmarks:\n # Lookup bookmark by URL again\n bookmark = next(\n (\n bookmark\n for bookmark in existing_bookmarks\n if bookmark.url == netscape_bookmark.href\n ),\n None,\n )\n\n if not bookmark:\n # Something is wrong, we should have just created this bookmark\n shortened_bookmark_tag_str = str(netscape_bookmark)[:100] + \"...\"\n logging.warning(\n f\"Failed to assign tags to the bookmark: {shortened_bookmark_tag_str}. Could not find bookmark by URL.\"\n )\n continue\n\n # Get tag models by string, schedule inserts for bookmark -> tag associations\n tags = tag_cache.get_all(netscape_bookmark.tag_names)\n for tag in tags:\n relationships.append(BookmarkToTagRelationShip(bookmark=bookmark, tag=tag))\n\n # Insert all bookmark -> tag associations at once, should ignore errors if association already exists\n BookmarkToTagRelationShip.objects.bulk_create(relationships, ignore_conflicts=True)\n\n\ndef _copy_bookmark_data(\n netscape_bookmark: NetscapeBookmark, bookmark: Bookmark, options: ImportOptions\n):\n bookmark.url = netscape_bookmark.href\n bookmark.url_normalized = normalize_url(bookmark.url)\n if netscape_bookmark.date_added:\n bookmark.date_added = parse_timestamp(netscape_bookmark.date_added)\n else:\n bookmark.date_added = timezone.now()\n if netscape_bookmark.date_modified:\n bookmark.date_modified = parse_timestamp(netscape_bookmark.date_modified)\n else:\n bookmark.date_modified = bookmark.date_added\n bookmark.unread = netscape_bookmark.to_read\n if netscape_bookmark.title:\n bookmark.title = netscape_bookmark.title\n if netscape_bookmark.description:\n bookmark.description = netscape_bookmark.description\n if netscape_bookmark.notes:\n bookmark.notes = netscape_bookmark.notes\n if options.map_private_flag and not netscape_bookmark.private:\n bookmark.shared = True\n if netscape_bookmark.archived:\n bookmark.is_archived = True\n"
},
{
"path": "bookmarks/services/monolith.py",
"content": "import gzip\nimport os\nimport shutil\nimport subprocess\n\nfrom django.conf import settings\n\n\nclass MonolithError(Exception):\n pass\n\n\n# Monolith isn't used at the moment, as the local snapshot implementation\n# switched to single-file after the prototype. Keeping this around in case\n# it turns out to be useful in the future.\ndef create_snapshot(url: str, filepath: str):\n monolith_path = settings.LD_MONOLITH_PATH\n monolith_options = settings.LD_MONOLITH_OPTIONS\n temp_filepath = filepath + \".tmp\"\n\n try:\n command = f\"{monolith_path} '{url}' {monolith_options} -o {temp_filepath}\"\n subprocess.run(command, check=True, shell=True)\n\n with (\n open(temp_filepath, \"rb\") as raw_file,\n gzip.open(filepath, \"wb\") as gz_file,\n ):\n shutil.copyfileobj(raw_file, gz_file)\n\n os.remove(temp_filepath)\n except subprocess.CalledProcessError as error:\n raise MonolithError(f\"Failed to create snapshot: {error.stderr}\") from error\n"
},
{
"path": "bookmarks/services/parser.py",
"content": "import contextlib\nfrom dataclasses import dataclass\nfrom html.parser import HTMLParser\n\nfrom bookmarks.models import parse_tag_string\n\n\n@dataclass\nclass NetscapeBookmark:\n href: str\n title: str\n description: str\n notes: str\n date_added: str\n date_modified: str\n tag_names: list[str]\n to_read: bool\n private: bool\n archived: bool\n\n\nclass BookmarkParser(HTMLParser):\n def __init__(self):\n super().__init__()\n self.bookmarks = []\n\n self.current_tag = None\n self.bookmark = None\n self.href = \"\"\n self.add_date = \"\"\n self.last_modified = \"\"\n self.tags = \"\"\n self.title = \"\"\n self.description = \"\"\n self.notes = \"\"\n self.toread = \"\"\n self.private = \"\"\n\n def handle_starttag(self, tag: str, attrs: list):\n name = \"handle_start_\" + tag.lower()\n if name in dir(self):\n getattr(self, name)({k.lower(): v for k, v in attrs})\n self.current_tag = tag\n\n def handle_endtag(self, tag: str):\n name = \"handle_end_\" + tag.lower()\n if name in dir(self):\n getattr(self, name)()\n self.current_tag = None\n\n def handle_data(self, data):\n name = f\"handle_{self.current_tag}_data\"\n if name in dir(self):\n getattr(self, name)(data)\n\n def handle_end_dl(self):\n self.add_bookmark()\n\n def handle_start_dt(self, attrs: dict[str, str]):\n self.add_bookmark()\n\n def handle_start_a(self, attrs: dict[str, str]):\n vars(self).update(attrs)\n tag_names = parse_tag_string(self.tags)\n archived = \"linkding:bookmarks.archived\" in self.tags\n with contextlib.suppress(ValueError):\n tag_names.remove(\"linkding:bookmarks.archived\")\n\n self.bookmark = NetscapeBookmark(\n href=self.href,\n title=\"\",\n description=\"\",\n notes=\"\",\n date_added=self.add_date,\n date_modified=self.last_modified,\n tag_names=tag_names,\n to_read=self.toread == \"1\",\n # Mark as private by default, also when attribute is not specified\n private=self.private != \"0\",\n archived=archived,\n )\n\n def handle_a_data(self, data):\n self.title = data.strip()\n\n def handle_dd_data(self, data):\n desc = data.strip()\n if \"[linkding-notes]\" in desc:\n self.notes = desc.split(\"[linkding-notes]\")[1].split(\"[/linkding-notes]\")[0]\n self.description = desc.split(\"[linkding-notes]\")[0]\n\n def add_bookmark(self):\n if self.bookmark:\n self.bookmark.title = self.title\n self.bookmark.description = self.description\n self.bookmark.notes = self.notes\n self.bookmarks.append(self.bookmark)\n self.bookmark = None\n self.href = \"\"\n self.add_date = \"\"\n self.last_modified = \"\"\n self.tags = \"\"\n self.title = \"\"\n self.description = \"\"\n self.notes = \"\"\n self.toread = \"\"\n self.private = \"\"\n\n\ndef parse(html: str) -> list[NetscapeBookmark]:\n parser = BookmarkParser()\n parser.feed(html)\n return parser.bookmarks\n"
},
{
"path": "bookmarks/services/preview_image_loader.py",
"content": "import hashlib\nimport logging\nimport mimetypes\nimport os.path\nfrom pathlib import Path\n\nimport requests\nfrom django.conf import settings\n\nfrom bookmarks.services import website_loader\n\nlogger = logging.getLogger(__name__)\n\n\ndef _ensure_preview_folder():\n Path(settings.LD_PREVIEW_FOLDER).mkdir(parents=True, exist_ok=True)\n\n\ndef _url_to_filename(preview_image: str) -> str:\n return hashlib.md5(preview_image.encode()).hexdigest()\n\n\ndef _get_image_path(preview_image_file: str) -> Path:\n return Path(os.path.join(settings.LD_PREVIEW_FOLDER, preview_image_file))\n\n\ndef load_preview_image(url: str) -> str | None:\n _ensure_preview_folder()\n\n metadata = website_loader.load_website_metadata(url)\n if not metadata.preview_image:\n logger.debug(f\"Could not find preview image in metadata: {url}\")\n return None\n\n image_url = metadata.preview_image\n\n logger.debug(f\"Loading preview image: {image_url}\")\n with requests.get(image_url, stream=True) as response:\n if response.status_code < 200 or response.status_code >= 300:\n logger.debug(\n f\"Bad response status code for preview image: {image_url} status_code={response.status_code}\"\n )\n return None\n\n if \"Content-Length\" not in response.headers:\n logger.debug(f\"Empty Content-Length for preview image: {image_url}\")\n return None\n\n content_length = int(response.headers[\"Content-Length\"])\n if content_length > settings.LD_PREVIEW_MAX_SIZE:\n logger.debug(\n f\"Content-Length exceeds LD_PREVIEW_MAX_SIZE: {image_url} length={content_length}\"\n )\n return None\n\n if \"Content-Type\" not in response.headers:\n logger.debug(f\"Empty Content-Type for preview image: {image_url}\")\n return None\n\n content_type = response.headers[\"Content-Type\"].split(\";\", 1)[0]\n file_extension = mimetypes.guess_extension(content_type)\n\n if file_extension not in settings.LD_PREVIEW_ALLOWED_EXTENSIONS:\n logger.debug(\n f\"Unsupported Content-Type for preview image: {image_url} content_type={content_type}\"\n )\n return None\n\n preview_image_hash = _url_to_filename(url)\n preview_image_file = f\"{preview_image_hash}{file_extension}\"\n preview_image_path = _get_image_path(preview_image_file)\n\n with open(preview_image_path, \"wb\") as file:\n downloaded = 0\n for chunk in response.iter_content(chunk_size=8192):\n downloaded += len(chunk)\n if downloaded > content_length:\n logger.debug(\n f\"Content-Length mismatch for preview image: {image_url} length={content_length} downloaded={downloaded}\"\n )\n file.close()\n preview_image_path.unlink()\n return None\n\n file.write(chunk)\n\n logger.debug(f\"Saved preview image as: {preview_image_path}\")\n\n return preview_image_file\n"
},
{
"path": "bookmarks/services/search_query_parser.py",
"content": "from dataclasses import dataclass\nfrom enum import Enum\n\nfrom bookmarks.models import UserProfile\n\n\nclass TokenType(Enum):\n TERM = \"TERM\"\n TAG = \"TAG\"\n SPECIAL_KEYWORD = \"SPECIAL_KEYWORD\"\n AND = \"AND\"\n OR = \"OR\"\n NOT = \"NOT\"\n LPAREN = \"LPAREN\"\n RPAREN = \"RPAREN\"\n EOF = \"EOF\"\n\n\n@dataclass\nclass Token:\n type: TokenType\n value: str\n position: int\n\n\nclass SearchQueryTokenizer:\n def __init__(self, query: str):\n self.query = query.strip()\n self.position = 0\n self.current_char = self.query[0] if self.query else None\n\n def advance(self):\n \"\"\"Move to the next character in the query.\"\"\"\n self.position += 1\n if self.position >= len(self.query):\n self.current_char = None\n else:\n self.current_char = self.query[self.position]\n\n def skip_whitespace(self):\n \"\"\"Skip whitespace characters.\"\"\"\n while self.current_char and self.current_char.isspace():\n self.advance()\n\n def read_term(self) -> str:\n \"\"\"Read a search term (sequence of non-whitespace, non-special characters).\"\"\"\n term = \"\"\n\n while (\n self.current_char\n and not self.current_char.isspace()\n and self.current_char not in \"()\\\"'#!\"\n ):\n term += self.current_char\n self.advance()\n\n return term\n\n def read_quoted_string(self, quote_char: str) -> str:\n \"\"\"Read a quoted string, handling escaped quotes.\"\"\"\n content = \"\"\n self.advance() # skip opening quote\n\n while self.current_char and self.current_char != quote_char:\n if self.current_char == \"\\\\\":\n # Handle escaped characters\n self.advance()\n if self.current_char:\n if self.current_char == \"n\":\n content += \"\\n\"\n elif self.current_char == \"t\":\n content += \"\\t\"\n elif self.current_char == \"r\":\n content += \"\\r\"\n elif self.current_char == \"\\\\\":\n content += \"\\\\\"\n elif self.current_char == quote_char:\n content += quote_char\n else:\n # For any other escaped character, just include it as-is\n content += self.current_char\n self.advance()\n else:\n content += self.current_char\n self.advance()\n\n if self.current_char == quote_char:\n self.advance() # skip closing quote\n else:\n # Unclosed quote - we could raise an error here, but let's be lenient\n # and treat it as if the quote was closed at the end\n pass\n\n return content\n\n def read_tag(self) -> str:\n \"\"\"Read a tag (starts with # and continues until whitespace or special chars).\"\"\"\n tag = \"\"\n self.advance() # skip the # character\n\n while (\n self.current_char\n and not self.current_char.isspace()\n and self.current_char not in \"()\\\"'\"\n ):\n tag += self.current_char\n self.advance()\n\n return tag\n\n def read_special_keyword(self) -> str:\n \"\"\"Read a special keyword (starts with ! and continues until whitespace or special chars).\"\"\"\n keyword = \"\"\n self.advance() # skip the ! character\n\n while (\n self.current_char\n and not self.current_char.isspace()\n and self.current_char not in \"()\\\"'\"\n ):\n keyword += self.current_char\n self.advance()\n\n return keyword\n\n def tokenize(self) -> list[Token]:\n \"\"\"Convert the query string into a list of tokens.\"\"\"\n tokens = []\n\n while self.current_char:\n self.skip_whitespace()\n\n if not self.current_char:\n break\n\n start_pos = self.position\n\n if self.current_char == \"(\":\n tokens.append(Token(TokenType.LPAREN, \"(\", start_pos))\n self.advance()\n elif self.current_char == \")\":\n tokens.append(Token(TokenType.RPAREN, \")\", start_pos))\n self.advance()\n elif self.current_char in \"\\\"'\":\n # Read a quoted string - always treated as a term\n quote_char = self.current_char\n term = self.read_quoted_string(quote_char)\n tokens.append(Token(TokenType.TERM, term, start_pos))\n elif self.current_char == \"#\":\n # Read a tag\n tag = self.read_tag()\n # Only add the tag token if it has content\n if tag:\n tokens.append(Token(TokenType.TAG, tag, start_pos))\n elif self.current_char == \"!\":\n # Read a special keyword\n keyword = self.read_special_keyword()\n # Only add the keyword token if it has content\n if keyword:\n tokens.append(Token(TokenType.SPECIAL_KEYWORD, keyword, start_pos))\n else:\n # Read a term and check if it's an operator\n term = self.read_term()\n term_lower = term.lower()\n\n if term_lower == \"and\":\n tokens.append(Token(TokenType.AND, term, start_pos))\n elif term_lower == \"or\":\n tokens.append(Token(TokenType.OR, term, start_pos))\n elif term_lower == \"not\":\n tokens.append(Token(TokenType.NOT, term, start_pos))\n else:\n tokens.append(Token(TokenType.TERM, term, start_pos))\n\n tokens.append(Token(TokenType.EOF, \"\", len(self.query)))\n return tokens\n\n\nclass SearchExpression:\n pass\n\n\n@dataclass\nclass TermExpression(SearchExpression):\n term: str\n\n\n@dataclass\nclass TagExpression(SearchExpression):\n tag: str\n\n\n@dataclass\nclass SpecialKeywordExpression(SearchExpression):\n keyword: str\n\n\n@dataclass\nclass AndExpression(SearchExpression):\n left: SearchExpression\n right: SearchExpression\n\n\n@dataclass\nclass OrExpression(SearchExpression):\n left: SearchExpression\n right: SearchExpression\n\n\n@dataclass\nclass NotExpression(SearchExpression):\n operand: SearchExpression\n\n\nclass SearchQueryParseError(Exception):\n def __init__(self, message: str, position: int):\n self.message = message\n self.position = position\n super().__init__(f\"{message} at position {position}\")\n\n\nclass SearchQueryParser:\n def __init__(self, tokens: list[Token]):\n self.tokens = tokens\n self.position = 0\n self.current_token = tokens[0] if tokens else Token(TokenType.EOF, \"\", 0)\n\n def advance(self):\n \"\"\"Move to the next token.\"\"\"\n if self.position < len(self.tokens) - 1:\n self.position += 1\n self.current_token = self.tokens[self.position]\n\n def consume(self, expected_type: TokenType) -> Token:\n \"\"\"Consume a token of the expected type or raise an error.\"\"\"\n if self.current_token.type == expected_type:\n token = self.current_token\n self.advance()\n return token\n else:\n raise SearchQueryParseError(\n f\"Expected {expected_type.value}, got {self.current_token.type.value}\",\n self.current_token.position,\n )\n\n def parse(self) -> SearchExpression | None:\n \"\"\"Parse the tokens into an AST.\"\"\"\n if not self.tokens or (\n len(self.tokens) == 1 and self.tokens[0].type == TokenType.EOF\n ):\n return None\n\n expr = self.parse_or_expression()\n\n if self.current_token.type != TokenType.EOF:\n raise SearchQueryParseError(\n f\"Unexpected token {self.current_token.type.value}\",\n self.current_token.position,\n )\n\n return expr\n\n def parse_or_expression(self) -> SearchExpression:\n \"\"\"Parse OR expressions (lowest precedence).\"\"\"\n left = self.parse_and_expression()\n\n while self.current_token.type == TokenType.OR:\n self.advance() # consume OR\n right = self.parse_and_expression()\n left = OrExpression(left, right)\n\n return left\n\n def parse_and_expression(self) -> SearchExpression:\n \"\"\"Parse AND expressions (medium precedence), including implicit AND.\"\"\"\n left = self.parse_not_expression()\n\n while self.current_token.type == TokenType.AND or self.current_token.type in [\n TokenType.TERM,\n TokenType.TAG,\n TokenType.SPECIAL_KEYWORD,\n TokenType.LPAREN,\n TokenType.NOT,\n ]:\n if self.current_token.type == TokenType.AND:\n self.advance() # consume explicit AND\n # else: implicit AND (don't advance token)\n\n right = self.parse_not_expression()\n left = AndExpression(left, right)\n\n return left\n\n def parse_not_expression(self) -> SearchExpression:\n \"\"\"Parse NOT expressions (high precedence).\"\"\"\n if self.current_token.type == TokenType.NOT:\n self.advance() # consume NOT\n operand = self.parse_not_expression() # right associative\n return NotExpression(operand)\n\n return self.parse_primary_expression()\n\n def parse_primary_expression(self) -> SearchExpression:\n \"\"\"Parse primary expressions (terms, tags, special keywords, and parenthesized expressions).\"\"\"\n if self.current_token.type == TokenType.TERM:\n term = self.current_token.value\n self.advance()\n return TermExpression(term)\n elif self.current_token.type == TokenType.TAG:\n tag = self.current_token.value\n self.advance()\n return TagExpression(tag)\n elif self.current_token.type == TokenType.SPECIAL_KEYWORD:\n keyword = self.current_token.value\n self.advance()\n return SpecialKeywordExpression(keyword)\n elif self.current_token.type == TokenType.LPAREN:\n self.advance() # consume (\n expr = self.parse_or_expression()\n self.consume(TokenType.RPAREN) # consume )\n return expr\n else:\n raise SearchQueryParseError(\n f\"Unexpected token {self.current_token.type.value}\",\n self.current_token.position,\n )\n\n\ndef parse_search_query(query: str) -> SearchExpression | None:\n if not query or not query.strip():\n return None\n\n tokenizer = SearchQueryTokenizer(query)\n tokens = tokenizer.tokenize()\n parser = SearchQueryParser(tokens)\n return parser.parse()\n\n\ndef _needs_parentheses(expr: SearchExpression, parent_type: type) -> bool:\n if isinstance(expr, OrExpression) and parent_type == AndExpression:\n return True\n # AndExpression or OrExpression needs parentheses when inside NotExpression\n return (\n isinstance(expr, (AndExpression, OrExpression)) and parent_type == NotExpression\n )\n\n\ndef _is_simple_expression(expr: SearchExpression) -> bool:\n \"\"\"Check if an expression is simple (term, tag, or keyword).\"\"\"\n return isinstance(expr, (TermExpression, TagExpression, SpecialKeywordExpression))\n\n\ndef _expression_to_string(expr: SearchExpression, parent_type: type = None) -> str:\n if isinstance(expr, TermExpression):\n # Quote terms if they contain spaces or special characters\n if \" \" in expr.term or any(c in expr.term for c in [\"(\", \")\", '\"', \"'\"]):\n # Escape any quotes in the term\n escaped = expr.term.replace(\"\\\\\", \"\\\\\\\\\").replace('\"', '\\\\\"')\n return f'\"{escaped}\"'\n return expr.term\n\n elif isinstance(expr, TagExpression):\n return f\"#{expr.tag}\"\n\n elif isinstance(expr, SpecialKeywordExpression):\n return f\"!{expr.keyword}\"\n\n elif isinstance(expr, NotExpression):\n # Don't pass parent type to children\n operand_str = _expression_to_string(expr.operand, None)\n # Add parentheses if the operand is a binary operation\n if isinstance(expr.operand, (AndExpression, OrExpression)):\n return f\"not ({operand_str})\"\n return f\"not {operand_str}\"\n\n elif isinstance(expr, AndExpression):\n # Don't pass parent type to children - they'll add their own parens only if needed\n left_str = _expression_to_string(expr.left, None)\n right_str = _expression_to_string(expr.right, None)\n\n # Add parentheses to children if needed for precedence\n if _needs_parentheses(expr.left, AndExpression):\n left_str = f\"({left_str})\"\n if _needs_parentheses(expr.right, AndExpression):\n right_str = f\"({right_str})\"\n\n result = f\"{left_str} {right_str}\"\n\n # Add outer parentheses if needed based on parent context\n if parent_type and _needs_parentheses(expr, parent_type):\n result = f\"({result})\"\n\n return result\n\n elif isinstance(expr, OrExpression):\n # Don't pass parent type to children\n left_str = _expression_to_string(expr.left, None)\n right_str = _expression_to_string(expr.right, None)\n\n # OrExpression children don't need parentheses unless they're also OR (handled by recursion)\n result = f\"{left_str} or {right_str}\"\n\n # Add outer parentheses if needed based on parent context\n if parent_type and _needs_parentheses(expr, parent_type):\n result = f\"({result})\"\n\n return result\n\n else:\n raise ValueError(f\"Unknown expression type: {type(expr)}\")\n\n\ndef expression_to_string(expr: SearchExpression | None) -> str:\n if expr is None:\n return \"\"\n return _expression_to_string(expr)\n\n\ndef _strip_tag_from_expression(\n expr: SearchExpression | None, tag_name: str, enable_lax_search: bool = False\n) -> SearchExpression | None:\n if expr is None:\n return None\n\n if isinstance(expr, TagExpression):\n # Remove this tag if it matches\n if expr.tag.lower() == tag_name.lower():\n return None\n return expr\n\n elif isinstance(expr, TermExpression):\n # In lax search mode, also remove terms that match the tag name\n if enable_lax_search and expr.term.lower() == tag_name.lower():\n return None\n return expr\n\n elif isinstance(expr, SpecialKeywordExpression):\n # Keep special keywords as-is\n return expr\n\n elif isinstance(expr, NotExpression):\n # Recursively filter the operand\n filtered_operand = _strip_tag_from_expression(\n expr.operand, tag_name, enable_lax_search\n )\n if filtered_operand is None:\n # If the operand is removed, the whole NOT expression should be removed\n return None\n return NotExpression(filtered_operand)\n\n elif isinstance(expr, AndExpression):\n # Recursively filter both sides\n left = _strip_tag_from_expression(expr.left, tag_name, enable_lax_search)\n right = _strip_tag_from_expression(expr.right, tag_name, enable_lax_search)\n\n # If both sides are removed, remove the AND expression\n if left is None and right is None:\n return None\n # If one side is removed, return the other side\n elif left is None:\n return right\n elif right is None:\n return left\n else:\n return AndExpression(left, right)\n\n elif isinstance(expr, OrExpression):\n # Recursively filter both sides\n left = _strip_tag_from_expression(expr.left, tag_name, enable_lax_search)\n right = _strip_tag_from_expression(expr.right, tag_name, enable_lax_search)\n\n # If both sides are removed, remove the OR expression\n if left is None and right is None:\n return None\n # If one side is removed, return the other side\n elif left is None:\n return right\n elif right is None:\n return left\n else:\n return OrExpression(left, right)\n\n else:\n # Unknown expression type, return as-is\n return expr\n\n\ndef strip_tag_from_query(\n query: str, tag_name: str, user_profile: UserProfile | None = None\n) -> str:\n try:\n ast = parse_search_query(query)\n except SearchQueryParseError:\n return query\n\n if ast is None:\n return \"\"\n\n # Determine if lax search is enabled\n enable_lax_search = False\n if user_profile is not None:\n enable_lax_search = user_profile.tag_search == UserProfile.TAG_SEARCH_LAX\n\n # Strip the tag from the AST\n filtered_ast = _strip_tag_from_expression(ast, tag_name, enable_lax_search)\n\n # Convert back to a query string\n return expression_to_string(filtered_ast)\n\n\ndef _extract_tag_names_from_expression(\n expr: SearchExpression | None, enable_lax_search: bool = False\n) -> list[str]:\n if expr is None:\n return []\n\n if isinstance(expr, TagExpression):\n return [expr.tag]\n\n elif isinstance(expr, TermExpression):\n # In lax search mode, terms are also considered tags\n if enable_lax_search:\n return [expr.term]\n return []\n\n elif isinstance(expr, SpecialKeywordExpression):\n # Special keywords are not tags\n return []\n\n elif isinstance(expr, NotExpression):\n # Recursively extract from the operand\n return _extract_tag_names_from_expression(expr.operand, enable_lax_search)\n\n elif isinstance(expr, (AndExpression, OrExpression)):\n # Recursively extract from both sides and combine\n left_tags = _extract_tag_names_from_expression(expr.left, enable_lax_search)\n right_tags = _extract_tag_names_from_expression(expr.right, enable_lax_search)\n return left_tags + right_tags\n\n else:\n # Unknown expression type\n return []\n\n\ndef extract_tag_names_from_query(\n query: str, user_profile: UserProfile | None = None\n) -> list[str]:\n try:\n ast = parse_search_query(query)\n except SearchQueryParseError:\n return []\n\n if ast is None:\n return []\n\n # Determine if lax search is enabled\n enable_lax_search = False\n if user_profile is not None:\n enable_lax_search = user_profile.tag_search == UserProfile.TAG_SEARCH_LAX\n\n # Extract tag names from the AST\n tag_names = _extract_tag_names_from_expression(ast, enable_lax_search)\n\n # Deduplicate (case-insensitive) and sort\n seen = set()\n unique_tags = []\n for tag in tag_names:\n tag_lower = tag.lower()\n if tag_lower not in seen:\n seen.add(tag_lower)\n unique_tags.append(tag_lower)\n\n return sorted(unique_tags)\n"
},
{
"path": "bookmarks/services/singlefile.py",
"content": "import logging\nimport os\nimport shlex\nimport signal\nimport subprocess\n\nfrom django.conf import settings\n\n\nclass SingleFileError(Exception):\n pass\n\n\nlogger = logging.getLogger(__name__)\n\n\ndef create_snapshot(url: str, filepath: str):\n singlefile_path = settings.LD_SINGLEFILE_PATH\n\n # parse options to list of arguments\n ublock_options = shlex.split(settings.LD_SINGLEFILE_UBLOCK_OPTIONS)\n custom_options = shlex.split(settings.LD_SINGLEFILE_OPTIONS)\n # concat lists\n args = [singlefile_path] + ublock_options + custom_options + [url, filepath]\n try:\n # Use start_new_session=True to create a new process group\n process = subprocess.Popen(args, start_new_session=True)\n process.wait(timeout=settings.LD_SINGLEFILE_TIMEOUT_SEC)\n\n # check if the file was created\n if not os.path.exists(filepath):\n raise SingleFileError(\"Failed to create snapshot\")\n except subprocess.TimeoutExpired:\n # First try to terminate properly\n try:\n logger.error(\n \"Timeout expired while creating snapshot. Terminating process...\"\n )\n process.terminate()\n process.wait(timeout=20)\n raise SingleFileError(\"Timeout expired while creating snapshot\") from None\n except subprocess.TimeoutExpired:\n # Kill the whole process group, which should also clean up any chromium\n # processes spawned by single-file\n logger.error(\"Timeout expired while terminating. Killing process...\")\n os.killpg(os.getpgid(process.pid), signal.SIGTERM)\n raise SingleFileError(\"Timeout expired while creating snapshot\") from None\n except subprocess.CalledProcessError as error:\n raise SingleFileError(f\"Failed to create snapshot: {error.stderr}\") from error\n"
},
{
"path": "bookmarks/services/tags.py",
"content": "import logging\nimport operator\n\nfrom django.contrib.auth.models import User\nfrom django.utils import timezone\n\nfrom bookmarks.models import Tag\nfrom bookmarks.utils import unique\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_or_create_tags(tag_names: list[str], user: User):\n tags = [get_or_create_tag(tag_name, user) for tag_name in tag_names]\n return unique(tags, operator.attrgetter(\"id\"))\n\n\ndef get_or_create_tag(name: str, user: User):\n try:\n return Tag.objects.get(name__iexact=name, owner=user)\n except Tag.DoesNotExist:\n tag = Tag(name=name, owner=user)\n tag.date_added = timezone.now()\n tag.save()\n return tag\n except Tag.MultipleObjectsReturned:\n # Legacy databases might contain duplicate tags with different capitalization\n first_tag = Tag.objects.filter(name__iexact=name, owner=user).first()\n message = (\n f\"Found multiple tags for the name '{name}' with different capitalization. \"\n f\"Using the first tag with the name '{first_tag.name}'. \"\n \"Since v.1.2 tags work case-insensitive, which means duplicates of the same name are not allowed anymore. \"\n \"To solve this error remove the duplicate tag in admin.\"\n )\n logger.error(message)\n return first_tag\n"
},
{
"path": "bookmarks/services/tasks.py",
"content": "import functools\nimport logging\n\nimport waybackpy\nfrom django.conf import settings\nfrom django.contrib.auth.models import User\nfrom django.db.models import Q\nfrom django.utils import timezone\nfrom huey import crontab\nfrom huey.contrib.djhuey import HUEY as huey\nfrom huey.exceptions import TaskLockedException\nfrom waybackpy.exceptions import TooManyRequestsError, WaybackError\n\nfrom bookmarks.models import Bookmark, BookmarkAsset, UserProfile\nfrom bookmarks.services import assets, favicon_loader, preview_image_loader\nfrom bookmarks.services.website_loader import DEFAULT_USER_AGENT, load_website_metadata\n\nlogger = logging.getLogger(__name__)\n\n\n# Create custom decorator for Huey tasks that implements exponential backoff\n# Taken from: https://huey.readthedocs.io/en/latest/guide.html#tips-and-tricks\n# Retry 1: 60\n# Retry 2: 240\n# Retry 3: 960\n# Retry 4: 3840\n# Retry 5: 15360\ndef task(retries=5, retry_delay=15, retry_backoff=4):\n def deco(fn):\n @functools.wraps(fn)\n def inner(*args, **kwargs):\n task = kwargs.pop(\"task\")\n try:\n return fn(*args, **kwargs)\n except TaskLockedException as exc:\n # Task locks are currently only used as workaround to enforce\n # running specific types of tasks (e.g. singlefile snapshots)\n # sequentially. In that case don't reduce the number of retries.\n task.retries = retries\n raise exc\n except Exception as exc:\n task.retry_delay *= retry_backoff\n raise exc\n\n return huey.task(retries=retries, retry_delay=retry_delay, context=True)(inner)\n\n return deco\n\n\ndef is_web_archive_integration_active(user: User) -> bool:\n background_tasks_enabled = not settings.LD_DISABLE_BACKGROUND_TASKS\n web_archive_integration_enabled = (\n user.profile.web_archive_integration\n == UserProfile.WEB_ARCHIVE_INTEGRATION_ENABLED\n )\n\n return background_tasks_enabled and web_archive_integration_enabled\n\n\ndef create_web_archive_snapshot(user: User, bookmark: Bookmark, force_update: bool):\n if is_web_archive_integration_active(user):\n _create_web_archive_snapshot_task(bookmark.id, force_update)\n\n\ndef _create_snapshot(bookmark: Bookmark):\n logger.info(f\"Create new snapshot for bookmark. url={bookmark.url}...\")\n archive = waybackpy.WaybackMachineSaveAPI(\n bookmark.url, DEFAULT_USER_AGENT, max_tries=1\n )\n archive.save()\n bookmark.web_archive_snapshot_url = archive.archive_url\n bookmark.save(update_fields=[\"web_archive_snapshot_url\"])\n logger.info(f\"Successfully created new snapshot for bookmark:. url={bookmark.url}\")\n\n\n@task()\ndef _create_web_archive_snapshot_task(bookmark_id: int, force_update: bool):\n try:\n bookmark = Bookmark.objects.get(id=bookmark_id)\n except Bookmark.DoesNotExist:\n return\n\n # Skip if snapshot exists and update is not explicitly requested\n if bookmark.web_archive_snapshot_url and not force_update:\n return\n\n # Create new snapshot\n try:\n _create_snapshot(bookmark)\n return\n except TooManyRequestsError:\n logger.error(\n f\"Failed to create snapshot due to rate limiting. url={bookmark.url}\"\n )\n except WaybackError as error:\n logger.error(\n f\"Failed to create snapshot. url={bookmark.url}\",\n exc_info=error,\n )\n\n\n@task()\ndef _load_web_archive_snapshot_task(bookmark_id: int):\n # Loading snapshots from CDX API has been removed, keeping the task function\n # for now to prevent errors when huey tries to run the task\n pass\n\n\n@task()\ndef _schedule_bookmarks_without_snapshots_task(user_id: int):\n # Loading snapshots from CDX API has been removed, keeping the task function\n # for now to prevent errors when huey tries to run the task\n pass\n\n\ndef is_favicon_feature_active(user: User) -> bool:\n background_tasks_enabled = not settings.LD_DISABLE_BACKGROUND_TASKS\n\n return background_tasks_enabled and user.profile.enable_favicons\n\n\ndef is_preview_feature_active(user: User) -> bool:\n return (\n user.profile.enable_preview_images and not settings.LD_DISABLE_BACKGROUND_TASKS\n )\n\n\ndef load_favicon(user: User, bookmark: Bookmark):\n if is_favicon_feature_active(user):\n _load_favicon_task(bookmark.id)\n\n\n@task()\ndef _load_favicon_task(bookmark_id: int):\n try:\n bookmark = Bookmark.objects.get(id=bookmark_id)\n except Bookmark.DoesNotExist:\n return\n\n logger.info(f\"Load favicon for bookmark. url={bookmark.url}\")\n\n new_favicon_file = favicon_loader.load_favicon(bookmark.url)\n\n if new_favicon_file != bookmark.favicon_file:\n bookmark.favicon_file = new_favicon_file\n bookmark.save(update_fields=[\"favicon_file\"])\n logger.info(\n f\"Successfully updated favicon for bookmark. url={bookmark.url} icon={new_favicon_file}\"\n )\n\n\ndef schedule_bookmarks_without_favicons(user: User):\n if is_favicon_feature_active(user):\n _schedule_bookmarks_without_favicons_task(user.id)\n\n\n@task()\ndef _schedule_bookmarks_without_favicons_task(user_id: int):\n user = User.objects.get(id=user_id)\n bookmarks = Bookmark.objects.filter(favicon_file__exact=\"\", owner=user)\n\n # TODO: Implement bulk task creation\n for bookmark in bookmarks:\n _load_favicon_task(bookmark.id)\n pass\n\n\ndef schedule_refresh_favicons(user: User):\n if is_favicon_feature_active(user) and settings.LD_ENABLE_REFRESH_FAVICONS:\n _schedule_refresh_favicons_task(user.id)\n\n\n@task()\ndef _schedule_refresh_favicons_task(user_id: int):\n user = User.objects.get(id=user_id)\n bookmarks = Bookmark.objects.filter(owner=user)\n\n # TODO: Implement bulk task creation\n for bookmark in bookmarks:\n _load_favicon_task(bookmark.id)\n\n\ndef load_preview_image(user: User, bookmark: Bookmark):\n if is_preview_feature_active(user):\n _load_preview_image_task(bookmark.id)\n\n\n@task()\ndef _load_preview_image_task(bookmark_id: int):\n try:\n bookmark = Bookmark.objects.get(id=bookmark_id)\n except Bookmark.DoesNotExist:\n return\n\n logger.info(f\"Load preview image for bookmark. url={bookmark.url}\")\n\n new_preview_image_file = preview_image_loader.load_preview_image(bookmark.url)\n\n if new_preview_image_file != bookmark.preview_image_file:\n bookmark.preview_image_file = new_preview_image_file or \"\"\n bookmark.save(update_fields=[\"preview_image_file\"])\n logger.info(\n f\"Successfully updated preview image for bookmark. url={bookmark.url} preview_image_file={new_preview_image_file}\"\n )\n\n\ndef schedule_bookmarks_without_previews(user: User):\n if is_preview_feature_active(user):\n _schedule_bookmarks_without_previews_task(user.id)\n\n\n@task()\ndef _schedule_bookmarks_without_previews_task(user_id: int):\n user = User.objects.get(id=user_id)\n bookmarks = Bookmark.objects.filter(\n Q(preview_image_file__exact=\"\"),\n owner=user,\n )\n\n # TODO: Implement bulk task creation\n for bookmark in bookmarks:\n try:\n _load_preview_image_task(bookmark.id)\n except Exception as exc:\n logging.exception(exc)\n\n\ndef refresh_metadata(bookmark: Bookmark):\n if not settings.LD_DISABLE_BACKGROUND_TASKS:\n _refresh_metadata_task(bookmark.id)\n\n\n@task()\ndef _refresh_metadata_task(bookmark_id: int):\n try:\n bookmark = Bookmark.objects.get(id=bookmark_id)\n except Bookmark.DoesNotExist:\n return\n\n logger.info(f\"Refresh metadata for bookmark. url={bookmark.url}\")\n\n metadata = load_website_metadata(bookmark.url)\n if metadata.title:\n bookmark.title = metadata.title\n if metadata.description:\n bookmark.description = metadata.description\n bookmark.date_modified = timezone.now()\n\n bookmark.save()\n logger.info(f\"Successfully refreshed metadata for bookmark. url={bookmark.url}\")\n\n\ndef is_html_snapshot_feature_active() -> bool:\n return settings.LD_ENABLE_SNAPSHOTS and not settings.LD_DISABLE_BACKGROUND_TASKS\n\n\ndef create_html_snapshot(bookmark: Bookmark):\n if not is_html_snapshot_feature_active():\n return\n\n asset = assets.create_snapshot_asset(bookmark)\n asset.save()\n\n\ndef create_html_snapshots(bookmark_list: list[Bookmark]):\n if not is_html_snapshot_feature_active():\n return\n\n assets_to_create = []\n for bookmark in bookmark_list:\n asset = assets.create_snapshot_asset(bookmark)\n assets_to_create.append(asset)\n\n BookmarkAsset.objects.bulk_create(assets_to_create)\n\n\n# singe-file does not support running multiple instances in parallel, so we can\n# not queue up multiple snapshot tasks at once. Instead, schedule a periodic\n# task that grabs a number of pending assets and creates snapshots for them in\n# sequence. The task uses a lock to ensure that a new task isn't scheduled\n# before the previous one has finished.\n@huey.periodic_task(crontab(minute=\"*\"))\n@huey.lock_task(\"schedule-html-snapshots-lock\")\ndef _schedule_html_snapshots_task():\n # Get five pending assets\n assets = BookmarkAsset.objects.filter(status=BookmarkAsset.STATUS_PENDING).order_by(\n \"date_created\"\n )[:5]\n\n for asset in assets:\n _create_html_snapshot_task(asset.id)\n\n\ndef _create_html_snapshot_task(asset_id: int):\n try:\n asset = BookmarkAsset.objects.get(id=asset_id)\n except BookmarkAsset.DoesNotExist:\n return\n\n logger.info(f\"Create HTML snapshot for bookmark. url={asset.bookmark.url}\")\n\n try:\n assets.create_snapshot(asset)\n\n logger.info(\n f\"Successfully created HTML snapshot for bookmark. url={asset.bookmark.url}\"\n )\n except Exception as error:\n logger.error(\n f\"Failed to HTML snapshot for bookmark. url={asset.bookmark.url}\",\n exc_info=error,\n )\n\n\ndef create_missing_html_snapshots(user: User) -> int:\n if not is_html_snapshot_feature_active():\n return 0\n\n bookmarks_without_snapshots = Bookmark.objects.filter(owner=user).exclude(\n bookmarkasset__asset_type=BookmarkAsset.TYPE_SNAPSHOT,\n bookmarkasset__status__in=[\n BookmarkAsset.STATUS_PENDING,\n BookmarkAsset.STATUS_COMPLETE,\n ],\n )\n bookmarks_without_snapshots |= Bookmark.objects.filter(owner=user).exclude(\n bookmarkasset__asset_type=BookmarkAsset.TYPE_SNAPSHOT\n )\n\n create_html_snapshots(list(bookmarks_without_snapshots))\n\n return bookmarks_without_snapshots.count()\n"
},
{
"path": "bookmarks/services/wayback.py",
"content": "import datetime\n\nfrom django.utils import timezone\n\n\ndef generate_fallback_webarchive_url(\n url: str, timestamp: datetime.datetime\n) -> str | None:\n \"\"\"\n Generate a URL to the web archive for the given URL and timestamp.\n A snapshot for the specific timestamp might not exist, in which case the\n web archive will show the closest snapshot to the given timestamp.\n If there is no snapshot at all the URL will be invalid.\n \"\"\"\n if not url:\n return None\n if not timestamp:\n timestamp = timezone.now()\n\n return f\"https://web.archive.org/web/{timestamp.strftime('%Y%m%d%H%M%S')}/{url}\"\n"
},
{
"path": "bookmarks/services/website_loader.py",
"content": "import logging\nfrom dataclasses import dataclass\nfrom functools import lru_cache\nfrom urllib.parse import urljoin\n\nimport requests\nfrom bs4 import BeautifulSoup\nfrom charset_normalizer import from_bytes\nfrom django.utils import timezone\n\nlogger = logging.getLogger(__name__)\n\n\n@dataclass\nclass WebsiteMetadata:\n url: str\n title: str | None\n description: str | None\n preview_image: str | None\n\n def to_dict(self):\n return {\n \"url\": self.url,\n \"title\": self.title,\n \"description\": self.description,\n \"preview_image\": self.preview_image,\n }\n\n\ndef load_website_metadata(url: str, ignore_cache: bool = False):\n if ignore_cache:\n return _load_website_metadata(url)\n return _load_website_metadata_cached(url)\n\n\n# Caching metadata avoids scraping again when saving bookmarks, in case the\n# metadata was already scraped to show preview values in the bookmark form\n@lru_cache(maxsize=10)\ndef _load_website_metadata_cached(url: str):\n return _load_website_metadata(url)\n\n\ndef _load_website_metadata(url: str):\n title = None\n description = None\n preview_image = None\n try:\n start = timezone.now()\n page_text = load_page(url)\n end = timezone.now()\n logger.debug(f\"Load duration: {end - start}\")\n\n start = timezone.now()\n soup = BeautifulSoup(page_text, \"html.parser\")\n\n if soup.title and soup.title.string:\n title = soup.title.string.strip()\n description_tag = soup.find(\"meta\", attrs={\"name\": \"description\"})\n description = (\n description_tag[\"content\"].strip()\n if description_tag and description_tag[\"content\"]\n else None\n )\n\n if not description:\n description_tag = soup.find(\"meta\", attrs={\"property\": \"og:description\"})\n description = (\n description_tag[\"content\"].strip()\n if description_tag and description_tag[\"content\"]\n else None\n )\n\n image_tag = soup.find(\"meta\", attrs={\"property\": \"og:image\"})\n preview_image = image_tag[\"content\"].strip() if image_tag else None\n if (\n preview_image\n and not preview_image.startswith(\"http://\")\n and not preview_image.startswith(\"https://\")\n ):\n preview_image = urljoin(url, preview_image)\n\n end = timezone.now()\n logger.debug(f\"Parsing duration: {end - start}\")\n except Exception:\n pass\n\n return WebsiteMetadata(\n url=url, title=title, description=description, preview_image=preview_image\n )\n\n\nCHUNK_SIZE = 50 * 1024\nMAX_CONTENT_LIMIT = 5000 * 1024\n\n\ndef load_page(url: str):\n headers = fake_request_headers()\n size = 0\n content = None\n iteration = 0\n # Use with to ensure request gets closed even if it's only read partially\n with requests.get(url, timeout=10, headers=headers, stream=True) as r:\n for chunk in r.iter_content(chunk_size=CHUNK_SIZE):\n size += len(chunk)\n iteration = iteration + 1\n content = chunk if content is None else content + chunk\n\n logger.debug(f\"Loaded chunk (iteration={iteration}, total={size / 1024})\")\n\n # Stop reading if we have parsed end of head tag\n end_of_head = b\"\"\n if end_of_head in content:\n logger.debug(f\"Found closing head tag after {size} bytes\")\n content = content.split(end_of_head)[0] + end_of_head\n break\n # Stop reading if we exceed limit\n if size > MAX_CONTENT_LIMIT:\n logger.debug(f\"Cancel reading document after {size} bytes\")\n break\n if hasattr(r, \"_content_consumed\"):\n logger.debug(f\"Request consumed: {r._content_consumed}\")\n\n # Use charset_normalizer to determine encoding that best matches the response content\n # Several sites seem to specify the response encoding incorrectly, so we ignore it and use custom logic instead\n # This is different from Response.text which does respect the encoding specified in the response first,\n # before trying to determine one\n results = from_bytes(content or \"\")\n return str(results.best())\n\n\nDEFAULT_USER_AGENT = \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36\"\n\n\ndef fake_request_headers():\n return {\n \"Accept\": \"text/html,application/xhtml+xml,application/xml\",\n \"Accept-Encoding\": \"gzip, deflate\",\n \"Dnt\": \"1\",\n \"Upgrade-Insecure-Requests\": \"1\",\n \"User-Agent\": DEFAULT_USER_AGENT,\n }\n\n\ndef detect_content_type(url: str, timeout: int = 10) -> str | None:\n \"\"\"Make HEAD request to detect content type of URL. Returns None on failure.\"\"\"\n headers = fake_request_headers()\n\n try:\n response = requests.head(\n url, headers=headers, timeout=timeout, allow_redirects=True\n )\n if response.status_code == 200:\n return (\n response.headers.get(\"Content-Type\", \"\").split(\";\")[0].strip().lower()\n )\n except requests.RequestException:\n pass\n\n try:\n with requests.get(\n url, headers=headers, timeout=timeout, stream=True, allow_redirects=True\n ) as response:\n if response.status_code == 200:\n return (\n response.headers.get(\"Content-Type\", \"\")\n .split(\";\")[0]\n .strip()\n .lower()\n )\n except requests.RequestException:\n pass\n\n return None\n\n\ndef is_pdf_content_type(content_type: str | None) -> bool:\n \"\"\"Check if the content type indicates a PDF.\"\"\"\n if not content_type:\n return False\n return content_type in (\"application/pdf\", \"application/x-pdf\")\n"
},
{
"path": "bookmarks/settings/__init__.py",
"content": "# Use dev settings as default, use production if dev settings do not exist\n# ruff: noqa\ntry:\n from .dev import *\nexcept:\n from .prod import *\n"
},
{
"path": "bookmarks/settings/base.py",
"content": "\"\"\"\nDjango settings for linkding webapp.\n\nGenerated by 'django-admin startproject' using Django 2.2.2.\n\nFor more information on this file, see\nhttps://docs.djangoproject.com/en/2.2/topics/settings/\n\nFor the full list of settings and their values, see\nhttps://docs.djangoproject.com/en/2.2/ref/settings/\n\"\"\"\n\nimport json\nimport os\nimport shlex\n\n# Build paths inside the project like this: os.path.join(BASE_DIR, ...)\nBASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))\n# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\n\n# Quick-start development settings - unsuitable for production\n# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/\n\n# SECURITY WARNING: keep the secret key used in production secret!\nSECRET_KEY = \"kgq$h3@!!vbb6*nzfz(dbze=*)zsroqa8gvc0#1gx$3cd8z99^\"\n\n# SECURITY WARNING: don't run with debug turned on in production!\nDEBUG = False\n\nALLOWED_HOSTS = [\"*\"]\n\nUSE_X_FORWARDED_HOST = os.getenv(\"LD_USE_X_FORWARDED_HOST\", False) in (\n True,\n \"True\",\n \"true\",\n \"1\",\n)\n\n# Application definition\n\nINSTALLED_APPS = [\n \"bookmarks.apps.BookmarksConfig\",\n \"django.contrib.admin\",\n \"django.contrib.auth\",\n \"django.contrib.contenttypes\",\n \"django.contrib.sessions\",\n \"django.contrib.messages\",\n \"django.contrib.staticfiles\",\n \"rest_framework\",\n \"rest_framework.authtoken\",\n \"huey.contrib.djhuey\",\n \"mozilla_django_oidc\",\n]\n\nMIDDLEWARE = [\n \"django.middleware.security.SecurityMiddleware\",\n \"django.contrib.sessions.middleware.SessionMiddleware\",\n \"django.middleware.common.CommonMiddleware\",\n \"django.middleware.csrf.CsrfViewMiddleware\",\n \"django.contrib.auth.middleware.AuthenticationMiddleware\",\n \"bookmarks.middlewares.LinkdingMiddleware\",\n \"django.contrib.messages.middleware.MessageMiddleware\",\n \"django.middleware.clickjacking.XFrameOptionsMiddleware\",\n \"django.middleware.locale.LocaleMiddleware\",\n]\n\nROOT_URLCONF = \"bookmarks.urls\"\n\nTEMPLATES = [\n {\n \"BACKEND\": \"django.template.backends.django.DjangoTemplates\",\n \"DIRS\": [],\n \"APP_DIRS\": True,\n \"OPTIONS\": {\n \"context_processors\": [\n \"django.template.context_processors.debug\",\n \"django.template.context_processors.request\",\n \"django.contrib.auth.context_processors.auth\",\n \"django.contrib.messages.context_processors.messages\",\n \"bookmarks.context_processors.toasts\",\n \"bookmarks.context_processors.app_version\",\n ],\n },\n },\n]\n\nDEFAULT_AUTO_FIELD = \"django.db.models.AutoField\"\n\nWSGI_APPLICATION = \"bookmarks.wsgi.application\"\n\n# Password validation\n# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators\n\nAUTH_PASSWORD_VALIDATORS = [\n {\n \"NAME\": \"django.contrib.auth.password_validation.UserAttributeSimilarityValidator\",\n },\n {\n \"NAME\": \"django.contrib.auth.password_validation.MinimumLengthValidator\",\n },\n {\n \"NAME\": \"django.contrib.auth.password_validation.CommonPasswordValidator\",\n },\n {\n \"NAME\": \"django.contrib.auth.password_validation.NumericPasswordValidator\",\n },\n]\n\n# Website context path.\nLD_CONTEXT_PATH = os.getenv(\"LD_CONTEXT_PATH\", \"\")\n\nLOGIN_URL = \"/\" + LD_CONTEXT_PATH + \"login\"\nLOGIN_REDIRECT_URL = \"/\" + LD_CONTEXT_PATH + \"bookmarks\"\nLOGOUT_REDIRECT_URL = \"/\" + LD_CONTEXT_PATH + \"login\"\n\n# Internationalization\n# https://docs.djangoproject.com/en/2.2/topics/i18n/\n\nLANGUAGE_CODE = \"en-us\"\n\nTIME_ZONE = os.getenv(\"TZ\", \"UTC\")\n\nUSE_I18N = True\n\nUSE_TZ = True\n\n# Static files (CSS, JavaScript, Images)\n# https://docs.djangoproject.com/en/2.2/howto/static-files/\n\nSTATIC_URL = \"/\" + LD_CONTEXT_PATH + \"static/\"\n\n# Collect static files in static folder\nSTATIC_ROOT = os.path.join(BASE_DIR, \"static\")\n\n# REST framework\nREST_FRAMEWORK = {\n \"DEFAULT_AUTHENTICATION_CLASSES\": [\n \"bookmarks.api.auth.LinkdingTokenAuthentication\",\n \"rest_framework.authentication.SessionAuthentication\",\n ],\n \"DEFAULT_PERMISSION_CLASSES\": [\"rest_framework.permissions.IsAuthenticated\"],\n \"DEFAULT_PAGINATION_CLASS\": \"rest_framework.pagination.LimitOffsetPagination\",\n \"PAGE_SIZE\": 100,\n}\n\n# URL validation flag\nLD_DISABLE_URL_VALIDATION = os.getenv(\"LD_DISABLE_URL_VALIDATION\", False) in (\n True,\n \"True\",\n \"true\",\n \"1\",\n)\n\n# Background task enabled setting\nLD_DISABLE_BACKGROUND_TASKS = os.getenv(\"LD_DISABLE_BACKGROUND_TASKS\", False) in (\n True,\n \"True\",\n \"true\",\n \"1\",\n)\n\n# Huey task queue\nHUEY = {\n \"huey_class\": \"huey.SqliteHuey\",\n \"filename\": os.path.join(BASE_DIR, \"data\", \"tasks.sqlite3\"),\n \"immediate\": False,\n \"results\": False,\n \"store_none\": False,\n \"utc\": True,\n \"consumer\": {\n \"workers\": 2,\n \"worker_type\": \"thread\",\n \"initial_delay\": 5,\n \"backoff\": 1.15,\n \"max_delay\": 10,\n \"scheduler_interval\": 10,\n \"periodic\": True,\n \"check_worker_health\": True,\n \"health_check_interval\": 10,\n },\n}\n\n# Disable login form if configured\nLD_DISABLE_LOGIN_FORM = os.getenv(\"LD_DISABLE_LOGIN_FORM\", False) in (\n True,\n \"True\",\n \"true\",\n \"1\",\n)\n\n# Enable OICD support if configured\nLD_ENABLE_OIDC = os.getenv(\"LD_ENABLE_OIDC\", False) in (True, \"True\", \"true\", \"1\")\n\nAUTHENTICATION_BACKENDS = [\"django.contrib.auth.backends.ModelBackend\"]\n\nif LD_ENABLE_OIDC:\n AUTHENTICATION_BACKENDS.append(\"mozilla_django_oidc.auth.OIDCAuthenticationBackend\")\n\n OIDC_USERNAME_ALGO = \"bookmarks.utils.generate_username\"\n OIDC_OP_AUTHORIZATION_ENDPOINT = os.getenv(\"OIDC_OP_AUTHORIZATION_ENDPOINT\")\n OIDC_OP_TOKEN_ENDPOINT = os.getenv(\"OIDC_OP_TOKEN_ENDPOINT\")\n OIDC_OP_USER_ENDPOINT = os.getenv(\"OIDC_OP_USER_ENDPOINT\")\n OIDC_OP_JWKS_ENDPOINT = os.getenv(\"OIDC_OP_JWKS_ENDPOINT\")\n OIDC_RP_CLIENT_ID = os.getenv(\"OIDC_RP_CLIENT_ID\")\n OIDC_RP_CLIENT_SECRET = os.getenv(\"OIDC_RP_CLIENT_SECRET\")\n OIDC_RP_SIGN_ALGO = os.getenv(\"OIDC_RP_SIGN_ALGO\", \"RS256\")\n OIDC_RP_SCOPES = os.getenv(\"OIDC_RP_SCOPES\", \"openid email profile\")\n OIDC_USE_PKCE = os.getenv(\"OIDC_USE_PKCE\", True) in (True, \"True\", \"true\", \"1\")\n OIDC_VERIFY_SSL = os.getenv(\"OIDC_VERIFY_SSL\", True) in (True, \"True\", \"true\", \"1\")\n OIDC_USERNAME_CLAIM = os.getenv(\"OIDC_USERNAME_CLAIM\", \"email\")\n\n# Enable authentication proxy support if configured\nLD_ENABLE_AUTH_PROXY = os.getenv(\"LD_ENABLE_AUTH_PROXY\", False) in (\n True,\n \"True\",\n \"true\",\n \"1\",\n)\nLD_AUTH_PROXY_USERNAME_HEADER = os.getenv(\n \"LD_AUTH_PROXY_USERNAME_HEADER\", \"REMOTE_USER\"\n)\nLD_AUTH_PROXY_LOGOUT_URL = os.getenv(\"LD_AUTH_PROXY_LOGOUT_URL\", None)\n\nif LD_ENABLE_AUTH_PROXY:\n # Add middleware that automatically authenticates requests that have a known username\n # in the LD_AUTH_PROXY_USERNAME_HEADER request header\n MIDDLEWARE.append(\"bookmarks.middlewares.CustomRemoteUserMiddleware\")\n # Configure auth backend that does not require a password credential\n AUTHENTICATION_BACKENDS = [\"django.contrib.auth.backends.RemoteUserBackend\"]\n # Configure logout URL\n if LD_AUTH_PROXY_LOGOUT_URL:\n LOGOUT_REDIRECT_URL = LD_AUTH_PROXY_LOGOUT_URL\n\n# CSRF trusted origins\ntrusted_origins = os.getenv(\"LD_CSRF_TRUSTED_ORIGINS\", \"\")\nif trusted_origins:\n CSRF_TRUSTED_ORIGINS = trusted_origins.split(\",\")\n\n# Database\n# https://docs.djangoproject.com/en/2.2/ref/settings/#databases\n\nLD_DB_ENGINE = os.getenv(\"LD_DB_ENGINE\", \"sqlite\")\nLD_DB_HOST = os.getenv(\"LD_DB_HOST\", \"localhost\")\nLD_DB_DATABASE = os.getenv(\"LD_DB_DATABASE\", \"linkding\")\nLD_DB_USER = os.getenv(\"LD_DB_USER\", \"linkding\")\nLD_DB_PASSWORD = os.getenv(\"LD_DB_PASSWORD\", None)\nLD_DB_PORT = os.getenv(\"LD_DB_PORT\", None)\nLD_DB_OPTIONS = json.loads(os.getenv(\"LD_DB_OPTIONS\") or \"{}\")\n\nif LD_DB_ENGINE == \"postgres\":\n default_database = {\n \"ENGINE\": \"django.db.backends.postgresql_psycopg2\",\n \"NAME\": LD_DB_DATABASE,\n \"USER\": LD_DB_USER,\n \"PASSWORD\": LD_DB_PASSWORD,\n \"HOST\": LD_DB_HOST,\n \"PORT\": LD_DB_PORT,\n \"OPTIONS\": LD_DB_OPTIONS,\n }\nelse:\n default_database = {\n \"ENGINE\": \"django.db.backends.sqlite3\",\n \"NAME\": os.path.join(BASE_DIR, \"data\", \"db.sqlite3\"),\n \"OPTIONS\": LD_DB_OPTIONS,\n # Creating a connection loads the ICU extension into the SQLite\n # connection, and also loads an ICU collation. The latter causes a\n # memory leak, so try to counter that by making connections indefinitely\n # persistent.\n \"CONN_MAX_AGE\": None,\n }\n\nDATABASES = {\"default\": default_database}\n\nSQLITE_ICU_EXTENSION_PATH = \"./libicu.so\"\nUSE_SQLITE = default_database[\"ENGINE\"] == \"django.db.backends.sqlite3\"\nUSE_SQLITE_ICU_EXTENSION = USE_SQLITE and os.path.exists(SQLITE_ICU_EXTENSION_PATH)\n\n# Favicons\nLD_DEFAULT_FAVICON_PROVIDER = \"https://t1.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url={url}&size=32\"\nLD_FAVICON_PROVIDER = os.getenv(\"LD_FAVICON_PROVIDER\", LD_DEFAULT_FAVICON_PROVIDER)\nLD_FAVICON_FOLDER = os.path.join(BASE_DIR, \"data\", \"favicons\")\nLD_ENABLE_REFRESH_FAVICONS = os.getenv(\"LD_ENABLE_REFRESH_FAVICONS\", True) in (\n True,\n \"True\",\n \"true\",\n \"1\",\n)\n\n# Previews settings\nLD_PREVIEW_FOLDER = os.path.join(BASE_DIR, \"data\", \"previews\")\nLD_PREVIEW_MAX_SIZE = int(os.getenv(\"LD_PREVIEW_MAX_SIZE\", 5242880))\nLD_PREVIEW_ALLOWED_EXTENSIONS = [\n \".jpg\",\n \".jpeg\",\n \".png\",\n \".gif\",\n \".svg\",\n \".webp\",\n]\n\n# Asset / snapshot settings\nLD_ASSET_FOLDER = os.path.join(BASE_DIR, \"data\", \"assets\")\n\nLD_ENABLE_SNAPSHOTS = os.getenv(\"LD_ENABLE_SNAPSHOTS\", False) in (\n True,\n \"True\",\n \"true\",\n \"1\",\n)\nLD_DISABLE_ASSET_UPLOAD = os.getenv(\"LD_DISABLE_ASSET_UPLOAD\", False) in (\n True,\n \"True\",\n \"true\",\n \"1\",\n)\nLD_SINGLEFILE_PATH = os.getenv(\"LD_SINGLEFILE_PATH\", \"single-file\")\nLD_SINGLEFILE_UBLOCK_OPTIONS = os.getenv(\n \"LD_SINGLEFILE_UBLOCK_OPTIONS\",\n shlex.join(\n [\n '--browser-arg=\"--headless=new\"',\n '--browser-arg=\"--user-data-dir=./chromium-profile\"',\n '--browser-arg=\"--no-sandbox\"',\n '--browser-arg=\"--load-extension=uBOLite.chromium.mv3\"',\n ]\n ),\n)\nLD_SINGLEFILE_OPTIONS = os.getenv(\"LD_SINGLEFILE_OPTIONS\", \"\")\nLD_SINGLEFILE_TIMEOUT_SEC = float(os.getenv(\"LD_SINGLEFILE_TIMEOUT_SEC\", 120))\nLD_SNAPSHOT_PDF_MAX_SIZE = int(os.getenv(\"LD_SNAPSHOT_PDF_MAX_SIZE\", 15728640)) # 15MB\n\n# Monolith isn't used at the moment, as the local snapshot implementation\n# switched to single-file after the prototype. Keeping this around in case\n# it turns out to be useful in the future.\nLD_MONOLITH_PATH = os.getenv(\"LD_MONOLITH_PATH\", \"monolith\")\nLD_MONOLITH_OPTIONS = os.getenv(\"LD_MONOLITH_OPTIONS\", \"-a -v -s\")\n"
},
{
"path": "bookmarks/settings/custom.py",
"content": "# Placeholder, can be mounted in a Docker container with a custom settings\n"
},
{
"path": "bookmarks/settings/dev.py",
"content": "\"\"\"\nDevelopment settings for linkding webapp\n\"\"\"\n\n# ruff: noqa\n\n# Start from development settings\n# noinspection PyUnresolvedReferences\nfrom .base import *\n\n# Turn on debug mode\nDEBUG = True\n\n# Enable debug toolbar\n# INSTALLED_APPS.append(\"debug_toolbar\")\n# MIDDLEWARE.append(\"debug_toolbar.middleware.DebugToolbarMiddleware\")\n\nINTERNAL_IPS = [\n \"127.0.0.1\",\n]\n\n# Allow access through ngrok\nCSRF_TRUSTED_ORIGINS = [\"https://*.ngrok-free.app\"]\n\nSTATICFILES_DIRS = [\n # Resolve theme files from style source folder\n os.path.join(BASE_DIR, \"bookmarks\", \"styles\"),\n # Resolve downloaded files in dev environment\n os.path.join(BASE_DIR, \"data\", \"favicons\"),\n os.path.join(BASE_DIR, \"data\", \"previews\"),\n]\n\n# Enable debug logging\nLOGGING = {\n \"version\": 1,\n \"disable_existing_loggers\": False,\n \"formatters\": {\n \"simple\": {\n \"format\": \"{levelname} {asctime} {module}: {message}\",\n \"style\": \"{\",\n },\n },\n \"handlers\": {\"console\": {\"class\": \"logging.StreamHandler\", \"formatter\": \"simple\"}},\n \"root\": {\n \"handlers\": [\"console\"],\n \"level\": \"WARNING\",\n },\n \"loggers\": {\n \"django.db.backends\": {\n \"level\": \"ERROR\", # Set to DEBUG to log all SQL calls\n \"handlers\": [\"console\"],\n },\n \"bookmarks\": { # Log importer debug output\n \"level\": \"DEBUG\",\n \"handlers\": [\"console\"],\n \"propagate\": False,\n },\n \"huey\": { # Huey\n \"level\": \"INFO\",\n \"handlers\": [\"console\"],\n \"propagate\": False,\n },\n },\n}\n\n# Import custom settings\n# noinspection PyUnresolvedReferences\nfrom .custom import *\n"
},
{
"path": "bookmarks/settings/prod.py",
"content": "\"\"\"\nProduction settings for linkding webapp\n\"\"\"\n\n# ruff: noqa\n\n# Start from development settings\n# noinspection PyUnresolvedReferences\nimport os\n\nfrom django.core.management.utils import get_random_secret_key\nfrom .base import *\n\n# Turn of debug mode\nDEBUG = False\n\n# Try read secret key from file\ntry:\n with open(os.path.join(BASE_DIR, \"data\", \"secretkey.txt\")) as f:\n SECRET_KEY = f.read().strip()\nexcept:\n SECRET_KEY = get_random_secret_key()\n\n# Set ALLOWED_HOSTS\n# By default look in the HOST_NAME environment variable, if that is not set then allow all hosts\nhost_name = os.environ.get(\"HOST_NAME\")\nif host_name:\n ALLOWED_HOSTS = [host_name]\nelse:\n ALLOWED_HOSTS = [\"*\"]\n\n# Logging\nLOGGING = {\n \"version\": 1,\n \"disable_existing_loggers\": False,\n \"formatters\": {\n \"simple\": {\n \"format\": \"{asctime} {levelname} {message}\",\n \"style\": \"{\",\n },\n },\n \"handlers\": {\"console\": {\"class\": \"logging.StreamHandler\", \"formatter\": \"simple\"}},\n \"root\": {\n \"handlers\": [\"console\"],\n \"level\": \"WARN\",\n },\n \"loggers\": {\n \"bookmarks\": {\n \"level\": \"INFO\",\n \"handlers\": [\"console\"],\n \"propagate\": False,\n },\n \"huey\": {\n \"level\": \"INFO\",\n \"handlers\": [\"console\"],\n \"propagate\": False,\n },\n },\n}\n\n# Import custom settings\n# noinspection PyUnresolvedReferences\nfrom .custom import *\n"
},
{
"path": "bookmarks/signals.py",
"content": "from django.conf import settings\nfrom django.db.backends.signals import connection_created\nfrom django.dispatch import receiver\n\n\n@receiver(connection_created)\ndef extend_sqlite(connection=None, **kwargs):\n # Load ICU extension into Sqlite connection to support case-insensitive\n # comparisons with unicode characters\n if connection.vendor == \"sqlite\" and settings.USE_SQLITE_ICU_EXTENSION:\n connection.connection.enable_load_extension(True)\n connection.connection.load_extension(\n settings.SQLITE_ICU_EXTENSION_PATH.rstrip(\".so\")\n )\n\n with connection.cursor() as cursor:\n # Load an ICU collation for case-insensitive ordering.\n # The first param can be a specific locale, it seems that not\n # providing one will use a default collation from the ICU project\n # that works reasonably for multiple languages\n cursor.execute(\"SELECT icu_load_collation('', 'ICU');\")\n"
},
{
"path": "bookmarks/static/live-reload.js",
"content": "const RELOAD_URL = \"/live_reload\";\n\nlet eventSource = null;\nlet serverId = null;\n\nfunction connect() {\n console.debug(\"[live-reload] Connecting to\", RELOAD_URL);\n\n eventSource = new EventSource(RELOAD_URL);\n\n eventSource.addEventListener(\"connected\", (event) => {\n const data = JSON.parse(event.data);\n\n if (serverId && serverId !== data.server_id) {\n console.log(\"[live-reload] Server restarted, reloading page\");\n window.location.reload();\n return;\n }\n\n console.debug(\"[live-reload] Connected, server ID:\", data.server_id);\n serverId = data.server_id;\n });\n\n eventSource.addEventListener(\"file_change\", (event) => {\n const data = JSON.parse(event.data);\n console.log(\"[live-reload] File changed:\", data);\n\n if (data.file_path.endsWith(\".html\") || data.file_path.endsWith(\".css\") || data.file_path.endsWith(\".js\")) {\n console.log(\"[live-reload] Asset changed, reloading page\");\n window.location.reload();\n }\n });\n\n eventSource.onerror = (error) => {\n console.debug(\"[live-reload] Disconnected\", error);\n eventSource.close();\n eventSource = null;\n\n // Reconnect after a delay\n setTimeout(connect, 1000);\n };\n}\n\nconnect();\n"
},
{
"path": "bookmarks/static/robots.txt",
"content": "User-agent: *\nDisallow: /\n"
},
{
"path": "bookmarks/static/vendor/Readability.js",
"content": "/*\n * Copyright (c) 2010 Arc90 Inc\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/*\n * This code is heavily based on Arc90's readability.js (1.7.1) script\n * available at: http://code.google.com/p/arc90labs-readability\n */\n\n/**\n * Public constructor.\n * @param {HTMLDocument} doc The document to parse.\n * @param {Object} options The options object.\n */\nfunction Readability(doc, options) {\n // In some older versions, people passed a URI as the first argument. Cope:\n if (options && options.documentElement) {\n doc = options;\n options = arguments[2];\n } else if (!doc || !doc.documentElement) {\n throw new Error(\"First argument to Readability constructor should be a document object.\");\n }\n options = options || {};\n\n this._doc = doc;\n this._docJSDOMParser = this._doc.firstChild.__JSDOMParser__;\n this._articleTitle = null;\n this._articleByline = null;\n this._articleDir = null;\n this._articleSiteName = null;\n this._attempts = [];\n\n // Configurable options\n this._debug = !!options.debug;\n this._maxElemsToParse = options.maxElemsToParse || this.DEFAULT_MAX_ELEMS_TO_PARSE;\n this._nbTopCandidates = options.nbTopCandidates || this.DEFAULT_N_TOP_CANDIDATES;\n this._charThreshold = options.charThreshold || this.DEFAULT_CHAR_THRESHOLD;\n this._classesToPreserve = this.CLASSES_TO_PRESERVE.concat(options.classesToPreserve || []);\n this._keepClasses = !!options.keepClasses;\n this._serializer = options.serializer || function(el) {\n return el.innerHTML;\n };\n this._disableJSONLD = !!options.disableJSONLD;\n this._allowedVideoRegex = options.allowedVideoRegex || this.REGEXPS.videos;\n\n // Start with all flags set\n this._flags = this.FLAG_STRIP_UNLIKELYS |\n this.FLAG_WEIGHT_CLASSES |\n this.FLAG_CLEAN_CONDITIONALLY;\n\n\n // Control whether log messages are sent to the console\n if (this._debug) {\n let logNode = function(node) {\n if (node.nodeType == node.TEXT_NODE) {\n return `${node.nodeName} (\"${node.textContent}\")`;\n }\n let attrPairs = Array.from(node.attributes || [], function(attr) {\n return `${attr.name}=\"${attr.value}\"`;\n }).join(\" \");\n return `<${node.localName} ${attrPairs}>`;\n };\n this.log = function () {\n if (typeof console !== \"undefined\") {\n let args = Array.from(arguments, arg => {\n if (arg && arg.nodeType == this.ELEMENT_NODE) {\n return logNode(arg);\n }\n return arg;\n });\n args.unshift(\"Reader: (Readability)\");\n console.log.apply(console, args);\n } else if (typeof dump !== \"undefined\") {\n /* global dump */\n var msg = Array.prototype.map.call(arguments, function(x) {\n return (x && x.nodeName) ? logNode(x) : x;\n }).join(\" \");\n dump(\"Reader: (Readability) \" + msg + \"\\n\");\n }\n };\n } else {\n this.log = function () {};\n }\n}\n\nReadability.prototype = {\n FLAG_STRIP_UNLIKELYS: 0x1,\n FLAG_WEIGHT_CLASSES: 0x2,\n FLAG_CLEAN_CONDITIONALLY: 0x4,\n\n // https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeType\n ELEMENT_NODE: 1,\n TEXT_NODE: 3,\n\n // Max number of nodes supported by this parser. Default: 0 (no limit)\n DEFAULT_MAX_ELEMS_TO_PARSE: 0,\n\n // The number of top candidates to consider when analysing how\n // tight the competition is among candidates.\n DEFAULT_N_TOP_CANDIDATES: 5,\n\n // Element tags to score by default.\n DEFAULT_TAGS_TO_SCORE: \"section,h2,h3,h4,h5,h6,p,td,pre\".toUpperCase().split(\",\"),\n\n // The default number of chars an article must have in order to return a result\n DEFAULT_CHAR_THRESHOLD: 500,\n\n // All of the regular expressions in use within readability.\n // Defined up here so we don't instantiate them repeatedly in loops.\n REGEXPS: {\n // NOTE: These two regular expressions are duplicated in\n // Readability-readerable.js. Please keep both copies in sync.\n unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,\n okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i,\n\n positive: /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,\n negative: /-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i,\n extraneous: /print|archive|comment|discuss|e[\\-]?mail|share|reply|all|login|sign|single|utility/i,\n byline: /byline|author|dateline|writtenby|p-author/i,\n replaceFonts: /<(\\/?)font[^>]*>/gi,\n normalize: /\\s{2,}/g,\n videos: /\\/\\/(www\\.)?((dailymotion|youtube|youtube-nocookie|player\\.vimeo|v\\.qq)\\.com|(archive|upload\\.wikimedia)\\.org|player\\.twitch\\.tv)/i,\n shareElements: /(\\b|_)(share|sharedaddy)(\\b|_)/i,\n nextLink: /(next|weiter|continue|>([^\\|]|$)|ยป([^\\|]|$))/i,\n prevLink: /(prev|earl|old|new|<|ยซ)/i,\n tokenize: /\\W+/g,\n whitespace: /^\\s*$/,\n hasContent: /\\S$/,\n hashUrl: /^#.+/,\n srcsetUrl: /(\\S+)(\\s+[\\d.]+[xw])?(\\s*(?:,|$))/g,\n b64DataUrl: /^data:\\s*([^\\s;,]+)\\s*;\\s*base64\\s*,/i,\n // Commas as used in Latin, Sindhi, Chinese and various other scripts.\n // see: https://en.wikipedia.org/wiki/Comma#Comma_variants\n commas: /\\u002C|\\u060C|\\uFE50|\\uFE10|\\uFE11|\\u2E41|\\u2E34|\\u2E32|\\uFF0C/g,\n // See: https://schema.org/Article\n jsonLdArticleTypes: /^Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|AskPublicNewsArticle|BackgroundNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|SatiricalArticle|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference$/\n },\n\n UNLIKELY_ROLES: [ \"menu\", \"menubar\", \"complementary\", \"navigation\", \"alert\", \"alertdialog\", \"dialog\" ],\n\n DIV_TO_P_ELEMS: new Set([ \"BLOCKQUOTE\", \"DL\", \"DIV\", \"IMG\", \"OL\", \"P\", \"PRE\", \"TABLE\", \"UL\" ]),\n\n ALTER_TO_DIV_EXCEPTIONS: [\"DIV\", \"ARTICLE\", \"SECTION\", \"P\"],\n\n PRESENTATIONAL_ATTRIBUTES: [ \"align\", \"background\", \"bgcolor\", \"border\", \"cellpadding\", \"cellspacing\", \"frame\", \"hspace\", \"rules\", \"style\", \"valign\", \"vspace\" ],\n\n DEPRECATED_SIZE_ATTRIBUTE_ELEMS: [ \"TABLE\", \"TH\", \"TD\", \"HR\", \"PRE\" ],\n\n // The commented out elements qualify as phrasing content but tend to be\n // removed by readability when put into paragraphs, so we ignore them here.\n PHRASING_ELEMS: [\n // \"CANVAS\", \"IFRAME\", \"SVG\", \"VIDEO\",\n \"ABBR\", \"AUDIO\", \"B\", \"BDO\", \"BR\", \"BUTTON\", \"CITE\", \"CODE\", \"DATA\",\n \"DATALIST\", \"DFN\", \"EM\", \"EMBED\", \"I\", \"IMG\", \"INPUT\", \"KBD\", \"LABEL\",\n \"MARK\", \"MATH\", \"METER\", \"NOSCRIPT\", \"OBJECT\", \"OUTPUT\", \"PROGRESS\", \"Q\",\n \"RUBY\", \"SAMP\", \"SCRIPT\", \"SELECT\", \"SMALL\", \"SPAN\", \"STRONG\", \"SUB\",\n \"SUP\", \"TEXTAREA\", \"TIME\", \"VAR\", \"WBR\"\n ],\n\n // These are the classes that readability sets itself.\n CLASSES_TO_PRESERVE: [ \"page\" ],\n\n // These are the list of HTML entities that need to be escaped.\n HTML_ESCAPE_MAP: {\n \"lt\": \"<\",\n \"gt\": \">\",\n \"amp\": \"&\",\n \"quot\": '\"',\n \"apos\": \"'\",\n },\n\n /**\n * Run any post-process modifications to article content as necessary.\n *\n * @param Element\n * @return void\n **/\n _postProcessContent: function(articleContent) {\n // Readability cannot open relative uris so we convert them to absolute uris.\n this._fixRelativeUris(articleContent);\n\n this._simplifyNestedElements(articleContent);\n\n if (!this._keepClasses) {\n // Remove classes.\n this._cleanClasses(articleContent);\n }\n },\n\n /**\n * Iterates over a NodeList, calls `filterFn` for each node and removes node\n * if function returned `true`.\n *\n * If function is not passed, removes all the nodes in node list.\n *\n * @param NodeList nodeList The nodes to operate on\n * @param Function filterFn the function to use as a filter\n * @return void\n */\n _removeNodes: function(nodeList, filterFn) {\n // Avoid ever operating on live node lists.\n if (this._docJSDOMParser && nodeList._isLiveNodeList) {\n throw new Error(\"Do not pass live node lists to _removeNodes\");\n }\n for (var i = nodeList.length - 1; i >= 0; i--) {\n var node = nodeList[i];\n var parentNode = node.parentNode;\n if (parentNode) {\n if (!filterFn || filterFn.call(this, node, i, nodeList)) {\n parentNode.removeChild(node);\n }\n }\n }\n },\n\n /**\n * Iterates over a NodeList, and calls _setNodeTag for each node.\n *\n * @param NodeList nodeList The nodes to operate on\n * @param String newTagName the new tag name to use\n * @return void\n */\n _replaceNodeTags: function(nodeList, newTagName) {\n // Avoid ever operating on live node lists.\n if (this._docJSDOMParser && nodeList._isLiveNodeList) {\n throw new Error(\"Do not pass live node lists to _replaceNodeTags\");\n }\n for (const node of nodeList) {\n this._setNodeTag(node, newTagName);\n }\n },\n\n /**\n * Iterate over a NodeList, which doesn't natively fully implement the Array\n * interface.\n *\n * For convenience, the current object context is applied to the provided\n * iterate function.\n *\n * @param NodeList nodeList The NodeList.\n * @param Function fn The iterate function.\n * @return void\n */\n _forEachNode: function(nodeList, fn) {\n Array.prototype.forEach.call(nodeList, fn, this);\n },\n\n /**\n * Iterate over a NodeList, and return the first node that passes\n * the supplied test function\n *\n * For convenience, the current object context is applied to the provided\n * test function.\n *\n * @param NodeList nodeList The NodeList.\n * @param Function fn The test function.\n * @return void\n */\n _findNode: function(nodeList, fn) {\n return Array.prototype.find.call(nodeList, fn, this);\n },\n\n /**\n * Iterate over a NodeList, return true if any of the provided iterate\n * function calls returns true, false otherwise.\n *\n * For convenience, the current object context is applied to the\n * provided iterate function.\n *\n * @param NodeList nodeList The NodeList.\n * @param Function fn The iterate function.\n * @return Boolean\n */\n _someNode: function(nodeList, fn) {\n return Array.prototype.some.call(nodeList, fn, this);\n },\n\n /**\n * Iterate over a NodeList, return true if all of the provided iterate\n * function calls return true, false otherwise.\n *\n * For convenience, the current object context is applied to the\n * provided iterate function.\n *\n * @param NodeList nodeList The NodeList.\n * @param Function fn The iterate function.\n * @return Boolean\n */\n _everyNode: function(nodeList, fn) {\n return Array.prototype.every.call(nodeList, fn, this);\n },\n\n /**\n * Concat all nodelists passed as arguments.\n *\n * @return ...NodeList\n * @return Array\n */\n _concatNodeLists: function() {\n var slice = Array.prototype.slice;\n var args = slice.call(arguments);\n var nodeLists = args.map(function(list) {\n return slice.call(list);\n });\n return Array.prototype.concat.apply([], nodeLists);\n },\n\n _getAllNodesWithTag: function(node, tagNames) {\n if (node.querySelectorAll) {\n return node.querySelectorAll(tagNames.join(\",\"));\n }\n return [].concat.apply([], tagNames.map(function(tag) {\n var collection = node.getElementsByTagName(tag);\n return Array.isArray(collection) ? collection : Array.from(collection);\n }));\n },\n\n /**\n * Removes the class=\"\" attribute from every element in the given\n * subtree, except those that match CLASSES_TO_PRESERVE and\n * the classesToPreserve array from the options object.\n *\n * @param Element\n * @return void\n */\n _cleanClasses: function(node) {\n var classesToPreserve = this._classesToPreserve;\n var className = (node.getAttribute(\"class\") || \"\")\n .split(/\\s+/)\n .filter(function(cls) {\n return classesToPreserve.indexOf(cls) != -1;\n })\n .join(\" \");\n\n if (className) {\n node.setAttribute(\"class\", className);\n } else {\n node.removeAttribute(\"class\");\n }\n\n for (node = node.firstElementChild; node; node = node.nextElementSibling) {\n this._cleanClasses(node);\n }\n },\n\n /**\n * Converts each and .\n * Whitespace between abc block.\n var replaced = false;\n\n // If we find a later).\n while ((next = this._nextNode(next)) && (next.tagName == \"BR\")) {\n replaced = true;\n var brSibling = next.nextSibling;\n next.parentNode.removeChild(next);\n next = brSibling;\n }\n\n // If we removed a . Add\n // all sibling nodes as children of the until we hit another .\n if (next.tagName == \"BR\") {\n var nextElem = this._nextNode(next.nextSibling);\n if (nextElem && nextElem.tagName == \"BR\")\n break;\n }\n\n if (!this._isPhrasingContent(next))\n break;\n\n // Otherwise, make this node a child of the new .\n var sibling = next.nextSibling;\n p.appendChild(next);\n next = sibling;\n }\n\n while (p.lastChild && this._isWhitespace(p.lastChild)) {\n p.removeChild(p.lastChild);\n }\n\n if (p.parentNode.tagName === \"P\")\n this._setNodeTag(p.parentNode, \"DIV\");\n }\n });\n },\n\n _setNodeTag: function (node, tag) {\n this.log(\"_setNodeTag\", node, tag);\n if (this._docJSDOMParser) {\n node.localName = tag.toLowerCase();\n node.tagName = tag.toUpperCase();\n return node;\n }\n\n var replacement = node.ownerDocument.createElement(tag);\n while (node.firstChild) {\n replacement.appendChild(node.firstChild);\n }\n node.parentNode.replaceChild(replacement, node);\n if (node.readability)\n replacement.readability = node.readability;\n\n for (var i = 0; i < node.attributes.length; i++) {\n try {\n replacement.setAttribute(node.attributes[i].name, node.attributes[i].value);\n } catch (ex) {\n /* it's possible for setAttribute() to throw if the attribute name\n * isn't a valid XML Name. Such attributes can however be parsed from\n * source in HTML docs, see https://github.com/whatwg/html/issues/4275,\n * so we can hit them here and then throw. We don't care about such\n * attributes so we ignore them.\n */\n }\n }\n return replacement;\n },\n\n /**\n * Prepare the article node for display. Clean out any inline styles,\n * iframes, forms, strip extraneous tags, etc.\n *\n * @param Element\n * @return void\n **/\n _prepArticle: function(articleContent) {\n this._cleanStyles(articleContent);\n\n // Check for data tables before we continue, to avoid removing items in\n // those tables, which will often be isolated even though they're\n // visually linked to other content-ful elements (text, images, etc.).\n this._markDataTables(articleContent);\n\n this._fixLazyImages(articleContent);\n\n // Clean out junk from the article content\n this._cleanConditionally(articleContent, \"form\");\n this._cleanConditionally(articleContent, \"fieldset\");\n this._clean(articleContent, \"object\");\n this._clean(articleContent, \"embed\");\n this._clean(articleContent, \"footer\");\n this._clean(articleContent, \"link\");\n this._clean(articleContent, \"aside\");\n\n // Clean out elements with little content that have \"share\" in their id/class combinations from final top candidates,\n // which means we don't remove the top candidates even they have \"share\".\n\n var shareElementThreshold = this.DEFAULT_CHAR_THRESHOLD;\n\n this._forEachNode(articleContent.children, function (topCandidate) {\n this._cleanMatchedNodes(topCandidate, function (node, matchString) {\n return this.REGEXPS.shareElements.test(matchString) && node.textContent.length < shareElementThreshold;\n });\n });\n\n this._clean(articleContent, \"iframe\");\n this._clean(articleContent, \"input\");\n this._clean(articleContent, \"textarea\");\n this._clean(articleContent, \"select\");\n this._clean(articleContent, \"button\");\n this._cleanHeaders(articleContent);\n\n // Do these last as the previous stuff may have removed junk\n // that will affect these\n this._cleanConditionally(articleContent, \"table\");\n this._cleanConditionally(articleContent, \"ul\");\n this._cleanConditionally(articleContent, \"div\");\n\n // replace H1 with H2 as H1 should be only title that is displayed separately\n this._replaceNodeTags(this._getAllNodesWithTag(articleContent, [\"h1\"]), \"h2\");\n\n // Remove extra paragraphs\n this._removeNodes(this._getAllNodesWithTag(articleContent, [\"p\"]), function (paragraph) {\n var imgCount = paragraph.getElementsByTagName(\"img\").length;\n var embedCount = paragraph.getElementsByTagName(\"embed\").length;\n var objectCount = paragraph.getElementsByTagName(\"object\").length;\n // At this point, nasty iframes have been removed, only remain embedded video ones.\n var iframeCount = paragraph.getElementsByTagName(\"iframe\").length;\n var totalCount = imgCount + embedCount + objectCount + iframeCount;\n\n return totalCount === 0 && !this._getInnerText(paragraph, false);\n });\n\n this._forEachNode(this._getAllNodesWithTag(articleContent, [\"br\"]), function(br) {\n var next = this._nextNode(br.nextSibling);\n if (next && next.tagName == \"P\")\n br.parentNode.removeChild(br);\n });\n\n // Remove single-cell tables\n this._forEachNode(this._getAllNodesWithTag(articleContent, [\"table\"]), function(table) {\n var tbody = this._hasSingleTagInsideElement(table, \"TBODY\") ? table.firstElementChild : table;\n if (this._hasSingleTagInsideElement(tbody, \"TR\")) {\n var row = tbody.firstElementChild;\n if (this._hasSingleTagInsideElement(row, \"TD\")) {\n var cell = row.firstElementChild;\n cell = this._setNodeTag(cell, this._everyNode(cell.childNodes, this._isPhrasingContent) ? \"P\" : \"DIV\");\n table.parentNode.replaceChild(cell, table);\n }\n }\n });\n },\n\n /**\n * Initialize a node with the readability object. Also checks the\n * className/id for special names to add to its score.\n *\n * @param Element\n * @return void\n **/\n _initializeNode: function(node) {\n node.readability = {\"contentScore\": 0};\n\n switch (node.tagName) {\n case \"DIV\":\n node.readability.contentScore += 5;\n break;\n\n case \"PRE\":\n case \"TD\":\n case \"BLOCKQUOTE\":\n node.readability.contentScore += 3;\n break;\n\n case \"ADDRESS\":\n case \"OL\":\n case \"UL\":\n case \"DL\":\n case \"DD\":\n case \"DT\":\n case \"LI\":\n case \"FORM\":\n node.readability.contentScore -= 3;\n break;\n\n case \"H1\":\n case \"H2\":\n case \"H3\":\n case \"H4\":\n case \"H5\":\n case \"H6\":\n case \"TH\":\n node.readability.contentScore -= 5;\n break;\n }\n\n node.readability.contentScore += this._getClassWeight(node);\n },\n\n _removeAndGetNext: function(node) {\n var nextNode = this._getNextNode(node, true);\n node.parentNode.removeChild(node);\n return nextNode;\n },\n\n /**\n * Traverse the DOM from node to node, starting at the node passed in.\n * Pass true for the second parameter to indicate this node itself\n * (and its kids) are going away, and we want the next node over.\n *\n * Calling this in a loop will traverse the DOM depth-first.\n */\n _getNextNode: function(node, ignoreSelfAndKids) {\n // First check for kids if those aren't being ignored\n if (!ignoreSelfAndKids && node.firstElementChild) {\n return node.firstElementChild;\n }\n // Then for siblings...\n if (node.nextElementSibling) {\n return node.nextElementSibling;\n }\n // And finally, move up the parent chain *and* find a sibling\n // (because this is depth-first traversal, we will have already\n // seen the parent nodes themselves).\n do {\n node = node.parentNode;\n } while (node && !node.nextElementSibling);\n return node && node.nextElementSibling;\n },\n\n // compares second text to first one\n // 1 = same text, 0 = completely different text\n // works the way that it splits both texts into words and then finds words that are unique in second text\n // the result is given by the lower length of unique parts\n _textSimilarity: function(textA, textB) {\n var tokensA = textA.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean);\n var tokensB = textB.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean);\n if (!tokensA.length || !tokensB.length) {\n return 0;\n }\n var uniqTokensB = tokensB.filter(token => !tokensA.includes(token));\n var distanceB = uniqTokensB.join(\" \").length / tokensB.join(\" \").length;\n return 1 - distanceB;\n },\n\n _checkByline: function(node, matchString) {\n if (this._articleByline) {\n return false;\n }\n\n if (node.getAttribute !== undefined) {\n var rel = node.getAttribute(\"rel\");\n var itemprop = node.getAttribute(\"itemprop\");\n }\n\n if ((rel === \"author\" || (itemprop && itemprop.indexOf(\"author\") !== -1) || this.REGEXPS.byline.test(matchString)) && this._isValidByline(node.textContent)) {\n this._articleByline = node.textContent.trim();\n return true;\n }\n\n return false;\n },\n\n _getNodeAncestors: function(node, maxDepth) {\n maxDepth = maxDepth || 0;\n var i = 0, ancestors = [];\n while (node.parentNode) {\n ancestors.push(node.parentNode);\n if (maxDepth && ++i === maxDepth)\n break;\n node = node.parentNode;\n }\n return ancestors;\n },\n\n /***\n * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is\n * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.\n *\n * @param page a document to run upon. Needs to be a full document, complete with body.\n * @return Element\n **/\n _grabArticle: function (page) {\n this.log(\"**** grabArticle ****\");\n var doc = this._doc;\n var isPaging = page !== null;\n page = page ? page : this._doc.body;\n\n // We can't grab an article if we don't have a page!\n if (!page) {\n this.log(\"No body found in document. Abort.\");\n return null;\n }\n\n var pageCacheHtml = page.innerHTML;\n\n while (true) {\n this.log(\"Starting grabArticle loop\");\n var stripUnlikelyCandidates = this._flagIsActive(this.FLAG_STRIP_UNLIKELYS);\n\n // First, node prepping. Trash nodes that look cruddy (like ones with the\n // class name \"comment\", etc), and turn divs into P tags where they have been\n // used inappropriately (as in, where they contain no other block level elements.)\n var elementsToScore = [];\n var node = this._doc.documentElement;\n\n let shouldRemoveTitleHeader = true;\n\n while (node) {\n\n if (node.tagName === \"HTML\") {\n this._articleLang = node.getAttribute(\"lang\");\n }\n\n var matchString = node.className + \" \" + node.id;\n\n if (!this._isProbablyVisible(node)) {\n this.log(\"Removing hidden node - \" + matchString);\n node = this._removeAndGetNext(node);\n continue;\n }\n\n // User is not able to see elements applied with both \"aria-modal = true\" and \"role = dialog\"\n if (node.getAttribute(\"aria-modal\") == \"true\" && node.getAttribute(\"role\") == \"dialog\") {\n node = this._removeAndGetNext(node);\n continue;\n }\n\n // Check to see if this node is a byline, and remove it if it is.\n if (this._checkByline(node, matchString)) {\n node = this._removeAndGetNext(node);\n continue;\n }\n\n if (shouldRemoveTitleHeader && this._headerDuplicatesTitle(node)) {\n this.log(\"Removing header: \", node.textContent.trim(), this._articleTitle.trim());\n shouldRemoveTitleHeader = false;\n node = this._removeAndGetNext(node);\n continue;\n }\n\n // Remove unlikely candidates\n if (stripUnlikelyCandidates) {\n if (this.REGEXPS.unlikelyCandidates.test(matchString) &&\n !this.REGEXPS.okMaybeItsACandidate.test(matchString) &&\n !this._hasAncestorTag(node, \"table\") &&\n !this._hasAncestorTag(node, \"code\") &&\n node.tagName !== \"BODY\" &&\n node.tagName !== \"A\") {\n this.log(\"Removing unlikely candidate - \" + matchString);\n node = this._removeAndGetNext(node);\n continue;\n }\n\n if (this.UNLIKELY_ROLES.includes(node.getAttribute(\"role\"))) {\n this.log(\"Removing content with role \" + node.getAttribute(\"role\") + \" - \" + matchString);\n node = this._removeAndGetNext(node);\n continue;\n }\n }\n\n // Remove DIV, SECTION, and HEADER nodes without any content(e.g. text, image, video, or iframe).\n if ((node.tagName === \"DIV\" || node.tagName === \"SECTION\" || node.tagName === \"HEADER\" ||\n node.tagName === \"H1\" || node.tagName === \"H2\" || node.tagName === \"H3\" ||\n node.tagName === \"H4\" || node.tagName === \"H5\" || node.tagName === \"H6\") &&\n this._isElementWithoutContent(node)) {\n node = this._removeAndGetNext(node);\n continue;\n }\n\n if (this.DEFAULT_TAGS_TO_SCORE.indexOf(node.tagName) !== -1) {\n elementsToScore.push(node);\n }\n\n // Turn all divs that don't have children block level elements into p's\n if (node.tagName === \"DIV\") {\n // Put phrasing content into paragraphs.\n var p = null;\n var childNode = node.firstChild;\n while (childNode) {\n var nextSibling = childNode.nextSibling;\n if (this._isPhrasingContent(childNode)) {\n if (p !== null) {\n p.appendChild(childNode);\n } else if (!this._isWhitespace(childNode)) {\n p = doc.createElement(\"p\");\n node.replaceChild(p, childNode);\n p.appendChild(childNode);\n }\n } else if (p !== null) {\n while (p.lastChild && this._isWhitespace(p.lastChild)) {\n p.removeChild(p.lastChild);\n }\n p = null;\n }\n childNode = nextSibling;\n }\n\n // Sites like http://mobile.slate.com encloses each paragraph with a DIV\n // element. DIVs with only a P element inside and no text content can be\n // safely converted into plain P elements to avoid confusing the scoring\n // algorithm with DIVs with are, in practice, paragraphs.\n if (this._hasSingleTagInsideElement(node, \"P\") && this._getLinkDensity(node) < 0.25) {\n var newNode = node.children[0];\n node.parentNode.replaceChild(newNode, node);\n node = newNode;\n elementsToScore.push(node);\n } else if (!this._hasChildBlockElement(node)) {\n node = this._setNodeTag(node, \"P\");\n elementsToScore.push(node);\n }\n }\n node = this._getNextNode(node);\n }\n\n /**\n * Loop through all paragraphs, and assign a score to them based on how content-y they look.\n * Then add their score to their parent node.\n *\n * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.\n **/\n var candidates = [];\n this._forEachNode(elementsToScore, function(elementToScore) {\n if (!elementToScore.parentNode || typeof(elementToScore.parentNode.tagName) === \"undefined\")\n return;\n\n // If this paragraph is less than 25 characters, don't even count it.\n var innerText = this._getInnerText(elementToScore);\n if (innerText.length < 25)\n return;\n\n // Exclude nodes with no ancestor.\n var ancestors = this._getNodeAncestors(elementToScore, 5);\n if (ancestors.length === 0)\n return;\n\n var contentScore = 0;\n\n // Add a point for the paragraph itself as a base.\n contentScore += 1;\n\n // Add points for any commas within this paragraph.\n contentScore += innerText.split(this.REGEXPS.commas).length;\n\n // For every 100 characters in this paragraph, add another point. Up to 3 points.\n contentScore += Math.min(Math.floor(innerText.length / 100), 3);\n\n // Initialize and score ancestors.\n this._forEachNode(ancestors, function(ancestor, level) {\n if (!ancestor.tagName || !ancestor.parentNode || typeof(ancestor.parentNode.tagName) === \"undefined\")\n return;\n\n if (typeof(ancestor.readability) === \"undefined\") {\n this._initializeNode(ancestor);\n candidates.push(ancestor);\n }\n\n // Node score divider:\n // - parent: 1 (no division)\n // - grandparent: 2\n // - great grandparent+: ancestor level * 3\n if (level === 0)\n var scoreDivider = 1;\n else if (level === 1)\n scoreDivider = 2;\n else\n scoreDivider = level * 3;\n ancestor.readability.contentScore += contentScore / scoreDivider;\n });\n });\n\n // After we've calculated scores, loop through all of the possible\n // candidate nodes we found and find the one with the highest score.\n var topCandidates = [];\n for (var c = 0, cl = candidates.length; c < cl; c += 1) {\n var candidate = candidates[c];\n\n // Scale the final candidates score based on link density. Good content\n // should have a relatively small link density (5% or less) and be mostly\n // unaffected by this operation.\n var candidateScore = candidate.readability.contentScore * (1 - this._getLinkDensity(candidate));\n candidate.readability.contentScore = candidateScore;\n\n this.log(\"Candidate:\", candidate, \"with score \" + candidateScore);\n\n for (var t = 0; t < this._nbTopCandidates; t++) {\n var aTopCandidate = topCandidates[t];\n\n if (!aTopCandidate || candidateScore > aTopCandidate.readability.contentScore) {\n topCandidates.splice(t, 0, candidate);\n if (topCandidates.length > this._nbTopCandidates)\n topCandidates.pop();\n break;\n }\n }\n }\n\n var topCandidate = topCandidates[0] || null;\n var neededToCreateTopCandidate = false;\n var parentOfTopCandidate;\n\n // If we still have no top candidate, just use the body as a last resort.\n // We also have to copy the body node so it is something we can modify.\n if (topCandidate === null || topCandidate.tagName === \"BODY\") {\n // Move all of the page's children into topCandidate\n topCandidate = doc.createElement(\"DIV\");\n neededToCreateTopCandidate = true;\n // Move everything (not just elements, also text nodes etc.) into the container\n // so we even include text directly in the body:\n while (page.firstChild) {\n this.log(\"Moving child out:\", page.firstChild);\n topCandidate.appendChild(page.firstChild);\n }\n\n page.appendChild(topCandidate);\n\n this._initializeNode(topCandidate);\n } else if (topCandidate) {\n // Find a better top candidate node if it contains (at least three) nodes which belong to `topCandidates` array\n // and whose scores are quite closed with current `topCandidate` node.\n var alternativeCandidateAncestors = [];\n for (var i = 1; i < topCandidates.length; i++) {\n if (topCandidates[i].readability.contentScore / topCandidate.readability.contentScore >= 0.75) {\n alternativeCandidateAncestors.push(this._getNodeAncestors(topCandidates[i]));\n }\n }\n var MINIMUM_TOPCANDIDATES = 3;\n if (alternativeCandidateAncestors.length >= MINIMUM_TOPCANDIDATES) {\n parentOfTopCandidate = topCandidate.parentNode;\n while (parentOfTopCandidate.tagName !== \"BODY\") {\n var listsContainingThisAncestor = 0;\n for (var ancestorIndex = 0; ancestorIndex < alternativeCandidateAncestors.length && listsContainingThisAncestor < MINIMUM_TOPCANDIDATES; ancestorIndex++) {\n listsContainingThisAncestor += Number(alternativeCandidateAncestors[ancestorIndex].includes(parentOfTopCandidate));\n }\n if (listsContainingThisAncestor >= MINIMUM_TOPCANDIDATES) {\n topCandidate = parentOfTopCandidate;\n break;\n }\n parentOfTopCandidate = parentOfTopCandidate.parentNode;\n }\n }\n if (!topCandidate.readability) {\n this._initializeNode(topCandidate);\n }\n\n // Because of our bonus system, parents of candidates might have scores\n // themselves. They get half of the node. There won't be nodes with higher\n // scores than our topCandidate, but if we see the score going *up* in the first\n // few steps up the tree, that's a decent sign that there might be more content\n // lurking in other places that we want to unify in. The sibling stuff\n // below does some of that - but only if we've looked high enough up the DOM\n // tree.\n parentOfTopCandidate = topCandidate.parentNode;\n var lastScore = topCandidate.readability.contentScore;\n // The scores shouldn't get too low.\n var scoreThreshold = lastScore / 3;\n while (parentOfTopCandidate.tagName !== \"BODY\") {\n if (!parentOfTopCandidate.readability) {\n parentOfTopCandidate = parentOfTopCandidate.parentNode;\n continue;\n }\n var parentScore = parentOfTopCandidate.readability.contentScore;\n if (parentScore < scoreThreshold)\n break;\n if (parentScore > lastScore) {\n // Alright! We found a better parent to use.\n topCandidate = parentOfTopCandidate;\n break;\n }\n lastScore = parentOfTopCandidate.readability.contentScore;\n parentOfTopCandidate = parentOfTopCandidate.parentNode;\n }\n\n // If the top candidate is the only child, use parent instead. This will help sibling\n // joining logic when adjacent content is actually located in parent's sibling node.\n parentOfTopCandidate = topCandidate.parentNode;\n while (parentOfTopCandidate.tagName != \"BODY\" && parentOfTopCandidate.children.length == 1) {\n topCandidate = parentOfTopCandidate;\n parentOfTopCandidate = topCandidate.parentNode;\n }\n if (!topCandidate.readability) {\n this._initializeNode(topCandidate);\n }\n }\n\n // Now that we have the top candidate, look through its siblings for content\n // that might also be related. Things like preambles, content split by ads\n // that we removed, etc.\n var articleContent = doc.createElement(\"DIV\");\n if (isPaging)\n articleContent.id = \"readability-content\";\n\n var siblingScoreThreshold = Math.max(10, topCandidate.readability.contentScore * 0.2);\n // Keep potential top candidate's parent node to try to get text direction of it later.\n parentOfTopCandidate = topCandidate.parentNode;\n var siblings = parentOfTopCandidate.children;\n\n for (var s = 0, sl = siblings.length; s < sl; s++) {\n var sibling = siblings[s];\n var append = false;\n\n this.log(\"Looking at sibling node:\", sibling, sibling.readability ? (\"with score \" + sibling.readability.contentScore) : \"\");\n this.log(\"Sibling has score\", sibling.readability ? sibling.readability.contentScore : \"Unknown\");\n\n if (sibling === topCandidate) {\n append = true;\n } else {\n var contentBonus = 0;\n\n // Give a bonus if sibling nodes and top candidates have the example same classname\n if (sibling.className === topCandidate.className && topCandidate.className !== \"\")\n contentBonus += topCandidate.readability.contentScore * 0.2;\n\n if (sibling.readability &&\n ((sibling.readability.contentScore + contentBonus) >= siblingScoreThreshold)) {\n append = true;\n } else if (sibling.nodeName === \"P\") {\n var linkDensity = this._getLinkDensity(sibling);\n var nodeContent = this._getInnerText(sibling);\n var nodeLength = nodeContent.length;\n\n if (nodeLength > 80 && linkDensity < 0.25) {\n append = true;\n } else if (nodeLength < 80 && nodeLength > 0 && linkDensity === 0 &&\n nodeContent.search(/\\.( |$)/) !== -1) {\n append = true;\n }\n }\n }\n\n if (append) {\n this.log(\"Appending node:\", sibling);\n\n if (this.ALTER_TO_DIV_EXCEPTIONS.indexOf(sibling.nodeName) === -1) {\n // We have a node that isn't a common block level element, like a form or td tag.\n // Turn it into a div so it doesn't get filtered out later by accident.\n this.log(\"Altering sibling:\", sibling, \"to div.\");\n\n sibling = this._setNodeTag(sibling, \"DIV\");\n }\n\n articleContent.appendChild(sibling);\n // Fetch children again to make it compatible\n // with DOM parsers without live collection support.\n siblings = parentOfTopCandidate.children;\n // siblings is a reference to the children array, and\n // sibling is removed from the array when we call appendChild().\n // As a result, we must revisit this index since the nodes\n // have been shifted.\n s -= 1;\n sl -= 1;\n }\n }\n\n if (this._debug)\n this.log(\"Article content pre-prep: \" + articleContent.innerHTML);\n // So we have all of the content that we need. Now we clean it up for presentation.\n this._prepArticle(articleContent);\n if (this._debug)\n this.log(\"Article content post-prep: \" + articleContent.innerHTML);\n\n if (neededToCreateTopCandidate) {\n // We already created a fake div thing, and there wouldn't have been any siblings left\n // for the previous loop, so there's no point trying to create a new div, and then\n // move all the children over. Just assign IDs and class names here. No need to append\n // because that already happened anyway.\n topCandidate.id = \"readability-page-1\";\n topCandidate.className = \"page\";\n } else {\n var div = doc.createElement(\"DIV\");\n div.id = \"readability-page-1\";\n div.className = \"page\";\n while (articleContent.firstChild) {\n div.appendChild(articleContent.firstChild);\n }\n articleContent.appendChild(div);\n }\n\n if (this._debug)\n this.log(\"Article content after paging: \" + articleContent.innerHTML);\n\n var parseSuccessful = true;\n\n // Now that we've gone through the full algorithm, check to see if\n // we got any meaningful content. If we didn't, we may need to re-run\n // grabArticle with different flags set. This gives us a higher likelihood of\n // finding the content, and the sieve approach gives us a higher likelihood of\n // finding the -right- content.\n var textLength = this._getInnerText(articleContent, true).length;\n if (textLength < this._charThreshold) {\n parseSuccessful = false;\n page.innerHTML = pageCacheHtml;\n\n if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) {\n this._removeFlag(this.FLAG_STRIP_UNLIKELYS);\n this._attempts.push({articleContent: articleContent, textLength: textLength});\n } else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) {\n this._removeFlag(this.FLAG_WEIGHT_CLASSES);\n this._attempts.push({articleContent: articleContent, textLength: textLength});\n } else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) {\n this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY);\n this._attempts.push({articleContent: articleContent, textLength: textLength});\n } else {\n this._attempts.push({articleContent: articleContent, textLength: textLength});\n // No luck after removing flags, just return the longest text we found during the different loops\n this._attempts.sort(function (a, b) {\n return b.textLength - a.textLength;\n });\n\n // But first check if we actually have something\n if (!this._attempts[0].textLength) {\n return null;\n }\n\n articleContent = this._attempts[0].articleContent;\n parseSuccessful = true;\n }\n }\n\n if (parseSuccessful) {\n // Find out text direction from ancestors of final top candidate.\n var ancestors = [parentOfTopCandidate, topCandidate].concat(this._getNodeAncestors(parentOfTopCandidate));\n this._someNode(ancestors, function(ancestor) {\n if (!ancestor.tagName)\n return false;\n var articleDir = ancestor.getAttribute(\"dir\");\n if (articleDir) {\n this._articleDir = articleDir;\n return true;\n }\n return false;\n });\n return articleContent;\n }\n }\n },\n\n /**\n * Check whether the input string could be a byline.\n * This verifies that the input is a string, and that the length\n * is less than 100 chars.\n *\n * @param possibleByline {string} - a string to check whether its a byline.\n * @return Boolean - whether the input string is a byline.\n */\n _isValidByline: function(byline) {\n if (typeof byline == \"string\" || byline instanceof String) {\n byline = byline.trim();\n return (byline.length > 0) && (byline.length < 100);\n }\n return false;\n },\n\n /**\n * Converts some of the common HTML entities in string to their corresponding characters.\n *\n * @param str {string} - a string to unescape.\n * @return string without HTML entity.\n */\n _unescapeHtmlEntities: function(str) {\n if (!str) {\n return str;\n }\n\n var htmlEscapeMap = this.HTML_ESCAPE_MAP;\n return str.replace(/&(quot|amp|apos|lt|gt);/g, function(_, tag) {\n return htmlEscapeMap[tag];\n }).replace(/&#(?:x([0-9a-z]{1,4})|([0-9]{1,4}));/gi, function(_, hex, numStr) {\n var num = parseInt(hex || numStr, hex ? 16 : 10);\n return String.fromCharCode(num);\n });\n },\n\n /**\n * Try to extract metadata from JSON-LD object.\n * For now, only Schema.org objects of type Article or its subtypes are supported.\n * @return Object with any metadata that could be extracted (possibly none)\n */\n _getJSONLD: function (doc) {\n var scripts = this._getAllNodesWithTag(doc, [\"script\"]);\n\n var metadata;\n\n this._forEachNode(scripts, function(jsonLdElement) {\n if (!metadata && jsonLdElement.getAttribute(\"type\") === \"application/ld+json\") {\n try {\n // Strip CDATA markers if present\n var content = jsonLdElement.textContent.replace(/^\\s*\\s*$/g, \"\");\n var parsed = JSON.parse(content);\n if (\n !parsed[\"@context\"] ||\n !parsed[\"@context\"].match(/^https?\\:\\/\\/schema\\.org$/)\n ) {\n return;\n }\n\n if (!parsed[\"@type\"] && Array.isArray(parsed[\"@graph\"])) {\n parsed = parsed[\"@graph\"].find(function(it) {\n return (it[\"@type\"] || \"\").match(\n this.REGEXPS.jsonLdArticleTypes\n );\n });\n }\n\n if (\n !parsed ||\n !parsed[\"@type\"] ||\n !parsed[\"@type\"].match(this.REGEXPS.jsonLdArticleTypes)\n ) {\n return;\n }\n\n metadata = {};\n\n if (typeof parsed.name === \"string\" && typeof parsed.headline === \"string\" && parsed.name !== parsed.headline) {\n // we have both name and headline element in the JSON-LD. They should both be the same but some websites like aktualne.cz\n // put their own name into \"name\" and the article title to \"headline\" which confuses Readability. So we try to check if either\n // \"name\" or \"headline\" closely matches the html title, and if so, use that one. If not, then we use \"name\" by default.\n\n var title = this._getArticleTitle();\n var nameMatches = this._textSimilarity(parsed.name, title) > 0.75;\n var headlineMatches = this._textSimilarity(parsed.headline, title) > 0.75;\n\n if (headlineMatches && !nameMatches) {\n metadata.title = parsed.headline;\n } else {\n metadata.title = parsed.name;\n }\n } else if (typeof parsed.name === \"string\") {\n metadata.title = parsed.name.trim();\n } else if (typeof parsed.headline === \"string\") {\n metadata.title = parsed.headline.trim();\n }\n if (parsed.author) {\n if (typeof parsed.author.name === \"string\") {\n metadata.byline = parsed.author.name.trim();\n } else if (Array.isArray(parsed.author) && parsed.author[0] && typeof parsed.author[0].name === \"string\") {\n metadata.byline = parsed.author\n .filter(function(author) {\n return author && typeof author.name === \"string\";\n })\n .map(function(author) {\n return author.name.trim();\n })\n .join(\", \");\n }\n }\n if (typeof parsed.description === \"string\") {\n metadata.excerpt = parsed.description.trim();\n }\n if (\n parsed.publisher &&\n typeof parsed.publisher.name === \"string\"\n ) {\n metadata.siteName = parsed.publisher.name.trim();\n }\n if (typeof parsed.datePublished === \"string\") {\n metadata.datePublished = parsed.datePublished.trim();\n }\n return;\n } catch (err) {\n this.log(err.message);\n }\n }\n });\n return metadata ? metadata : {};\n },\n\n /**\n * Attempts to get excerpt and byline metadata for the article.\n *\n * @param {Object} jsonld โ object containing any metadata that\n * could be extracted from JSON-LD object.\n *\n * @return Object with optional \"excerpt\" and \"byline\" properties\n */\n _getArticleMetadata: function(jsonld) {\n var metadata = {};\n var values = {};\n var metaElements = this._doc.getElementsByTagName(\"meta\");\n\n // property is a space-separated list of values\n var propertyPattern = /\\s*(article|dc|dcterm|og|twitter)\\s*:\\s*(author|creator|description|published_time|title|site_name)\\s*/gi;\n\n // name is a single value\n var namePattern = /^\\s*(?:(dc|dcterm|og|twitter|weibo:(article|webpage))\\s*[\\.:]\\s*)?(author|creator|description|title|site_name)\\s*$/i;\n\n // Find description tags.\n this._forEachNode(metaElements, function(element) {\n var elementName = element.getAttribute(\"name\");\n var elementProperty = element.getAttribute(\"property\");\n var content = element.getAttribute(\"content\");\n if (!content) {\n return;\n }\n var matches = null;\n var name = null;\n\n if (elementProperty) {\n matches = elementProperty.match(propertyPattern);\n if (matches) {\n // Convert to lowercase, and remove any whitespace\n // so we can match below.\n name = matches[0].toLowerCase().replace(/\\s/g, \"\");\n // multiple authors\n values[name] = content.trim();\n }\n }\n if (!matches && elementName && namePattern.test(elementName)) {\n name = elementName;\n if (content) {\n // Convert to lowercase, remove any whitespace, and convert dots\n // to colons so we can match below.\n name = name.toLowerCase().replace(/\\s/g, \"\").replace(/\\./g, \":\");\n values[name] = content.trim();\n }\n }\n });\n\n // get title\n metadata.title = jsonld.title ||\n values[\"dc:title\"] ||\n values[\"dcterm:title\"] ||\n values[\"og:title\"] ||\n values[\"weibo:article:title\"] ||\n values[\"weibo:webpage:title\"] ||\n values[\"title\"] ||\n values[\"twitter:title\"];\n\n if (!metadata.title) {\n metadata.title = this._getArticleTitle();\n }\n\n // get author\n metadata.byline = jsonld.byline ||\n values[\"dc:creator\"] ||\n values[\"dcterm:creator\"] ||\n values[\"author\"];\n\n // get description\n metadata.excerpt = jsonld.excerpt ||\n values[\"dc:description\"] ||\n values[\"dcterm:description\"] ||\n values[\"og:description\"] ||\n values[\"weibo:article:description\"] ||\n values[\"weibo:webpage:description\"] ||\n values[\"description\"] ||\n values[\"twitter:description\"];\n\n // get site name\n metadata.siteName = jsonld.siteName ||\n values[\"og:site_name\"];\n\n // get article published time\n metadata.publishedTime = jsonld.datePublished ||\n values[\"article:published_time\"] || null;\n\n // in many sites the meta value is escaped with HTML entities,\n // so here we need to unescape it\n metadata.title = this._unescapeHtmlEntities(metadata.title);\n metadata.byline = this._unescapeHtmlEntities(metadata.byline);\n metadata.excerpt = this._unescapeHtmlEntities(metadata.excerpt);\n metadata.siteName = this._unescapeHtmlEntities(metadata.siteName);\n metadata.publishedTime = this._unescapeHtmlEntities(metadata.publishedTime);\n\n return metadata;\n },\n\n /**\n * Check if node is image, or if node contains exactly only one image\n * whether as a direct child or as its descendants.\n *\n * @param Element\n **/\n _isSingleImage: function(node) {\n if (node.tagName === \"IMG\") {\n return true;\n }\n\n if (node.children.length !== 1 || node.textContent.trim() !== \"\") {\n return false;\n }\n\n return this._isSingleImage(node.children[0]);\n },\n\n /**\n * Find all uri in the given element to an absolute URI,\n * ignoring #ref URIs.\n *\n * @param Element\n * @return void\n */\n _fixRelativeUris: function(articleContent) {\n var baseURI = this._doc.baseURI;\n var documentURI = this._doc.documentURI;\n function toAbsoluteURI(uri) {\n // Leave hash links alone if the base URI matches the document URI:\n if (baseURI == documentURI && uri.charAt(0) == \"#\") {\n return uri;\n }\n\n // Otherwise, resolve against base URI:\n try {\n return new URL(uri, baseURI).href;\n } catch (ex) {\n // Something went wrong, just return the original:\n }\n return uri;\n }\n\n var links = this._getAllNodesWithTag(articleContent, [\"a\"]);\n this._forEachNode(links, function(link) {\n var href = link.getAttribute(\"href\");\n if (href) {\n // Remove links with javascript: URIs, since\n // they won't work after scripts have been removed from the page.\n if (href.indexOf(\"javascript:\") === 0) {\n // if the link only contains simple text content, it can be converted to a text node\n if (link.childNodes.length === 1 && link.childNodes[0].nodeType === this.TEXT_NODE) {\n var text = this._doc.createTextNode(link.textContent);\n link.parentNode.replaceChild(text, link);\n } else {\n // if the link has multiple children, they should all be preserved\n var container = this._doc.createElement(\"span\");\n while (link.firstChild) {\n container.appendChild(link.firstChild);\n }\n link.parentNode.replaceChild(container, link);\n }\n } else {\n link.setAttribute(\"href\", toAbsoluteURI(href));\n }\n }\n });\n\n var medias = this._getAllNodesWithTag(articleContent, [\n \"img\", \"picture\", \"figure\", \"video\", \"audio\", \"source\"\n ]);\n\n this._forEachNode(medias, function(media) {\n var src = media.getAttribute(\"src\");\n var poster = media.getAttribute(\"poster\");\n var srcset = media.getAttribute(\"srcset\");\n\n if (src) {\n media.setAttribute(\"src\", toAbsoluteURI(src));\n }\n\n if (poster) {\n media.setAttribute(\"poster\", toAbsoluteURI(poster));\n }\n\n if (srcset) {\n var newSrcset = srcset.replace(this.REGEXPS.srcsetUrl, function(_, p1, p2, p3) {\n return toAbsoluteURI(p1) + (p2 || \"\") + p3;\n });\n\n media.setAttribute(\"srcset\", newSrcset);\n }\n });\n },\n\n _simplifyNestedElements: function(articleContent) {\n var node = articleContent;\n\n while (node) {\n if (node.parentNode && [\"DIV\", \"SECTION\"].includes(node.tagName) && !(node.id && node.id.startsWith(\"readability\"))) {\n if (this._isElementWithoutContent(node)) {\n node = this._removeAndGetNext(node);\n continue;\n } else if (this._hasSingleTagInsideElement(node, \"DIV\") || this._hasSingleTagInsideElement(node, \"SECTION\")) {\n var child = node.children[0];\n for (var i = 0; i < node.attributes.length; i++) {\n child.setAttribute(node.attributes[i].name, node.attributes[i].value);\n }\n node.parentNode.replaceChild(child, node);\n node = child;\n continue;\n }\n }\n\n node = this._getNextNode(node);\n }\n },\n\n /**\n * Get the article title as an H1.\n *\n * @return string\n **/\n _getArticleTitle: function() {\n var doc = this._doc;\n var curTitle = \"\";\n var origTitle = \"\";\n\n try {\n curTitle = origTitle = doc.title.trim();\n\n // If they had an element with id \"title\" in their HTML\n if (typeof curTitle !== \"string\")\n curTitle = origTitle = this._getInnerText(doc.getElementsByTagName(\"title\")[0]);\n } catch (e) {/* ignore exceptions setting the title. */}\n\n var titleHadHierarchicalSeparators = false;\n function wordCount(str) {\n return str.split(/\\s+/).length;\n }\n\n // If there's a separator in the title, first remove the final part\n if ((/ [\\|\\-\\\\\\/>ยป] /).test(curTitle)) {\n titleHadHierarchicalSeparators = / [\\\\\\/>ยป] /.test(curTitle);\n curTitle = origTitle.replace(/(.*)[\\|\\-\\\\\\/>ยป] .*/gi, \"$1\");\n\n // If the resulting title is too short (3 words or fewer), remove\n // the first part instead:\n if (wordCount(curTitle) < 3)\n curTitle = origTitle.replace(/[^\\|\\-\\\\\\/>ยป]*[\\|\\-\\\\\\/>ยป](.*)/gi, \"$1\");\n } else if (curTitle.indexOf(\": \") !== -1) {\n // Check if we have an heading containing this exact string, so we\n // could assume it's the full title.\n var headings = this._concatNodeLists(\n doc.getElementsByTagName(\"h1\"),\n doc.getElementsByTagName(\"h2\")\n );\n var trimmedTitle = curTitle.trim();\n var match = this._someNode(headings, function(heading) {\n return heading.textContent.trim() === trimmedTitle;\n });\n\n // If we don't, let's extract the title out of the original title string.\n if (!match) {\n curTitle = origTitle.substring(origTitle.lastIndexOf(\":\") + 1);\n\n // If the title is now too short, try the first colon instead:\n if (wordCount(curTitle) < 3) {\n curTitle = origTitle.substring(origTitle.indexOf(\":\") + 1);\n // But if we have too many words before the colon there's something weird\n // with the titles and the H tags so let's just use the original title instead\n } else if (wordCount(origTitle.substr(0, origTitle.indexOf(\":\"))) > 5) {\n curTitle = origTitle;\n }\n }\n } else if (curTitle.length > 150 || curTitle.length < 15) {\n var hOnes = doc.getElementsByTagName(\"h1\");\n\n if (hOnes.length === 1)\n curTitle = this._getInnerText(hOnes[0]);\n }\n\n curTitle = curTitle.trim().replace(this.REGEXPS.normalize, \" \");\n // If we now have 4 words or fewer as our title, and either no\n // 'hierarchical' separators (\\, /, > or ยป) were found in the original\n // title or we decreased the number of words by more than 1 word, use\n // the original title.\n var curTitleWordCount = wordCount(curTitle);\n if (curTitleWordCount <= 4 &&\n (!titleHadHierarchicalSeparators ||\n curTitleWordCount != wordCount(origTitle.replace(/[\\|\\-\\\\\\/>ยป]+/g, \"\")) - 1)) {\n curTitle = origTitle;\n }\n\n return curTitle;\n },\n\n /**\n * Prepare the HTML document for readability to scrape it.\n * This includes things like stripping javascript, CSS, and handling terrible markup.\n *\n * @return void\n **/\n _prepDocument: function() {\n var doc = this._doc;\n\n // Remove all style tags in head\n this._removeNodes(this._getAllNodesWithTag(doc, [\"style\"]));\n\n if (doc.body) {\n this._replaceBrs(doc.body);\n }\n\n this._replaceNodeTags(this._getAllNodesWithTag(doc, [\"font\"]), \"SPAN\");\n },\n\n /**\n * Finds the next node, starting from the given node, and ignoring\n * whitespace in between. If the given node is an element, the same node is\n * returned.\n */\n _nextNode: function (node) {\n var next = node;\n while (next\n && (next.nodeType != this.ELEMENT_NODE)\n && this.REGEXPS.whitespace.test(next.textContent)) {\n next = next.nextSibling;\n }\n return next;\n },\n\n /**\n * Replaces 2 or more successive
elements with a single
elements are ignored. For example:\n * foo\n * will become:\n *
bar
abcfoo\n */\n _replaceBrs: function (elem) {\n this._forEachNode(this._getAllNodesWithTag(elem, [\"br\"]), function(br) {\n var next = br.nextSibling;\n\n // Whether 2 or more
bar
elements have been found and replaced with a\n //
chain, remove the
s until we hit another node\n // or non-whitespace. This leaves behind the first
in the chain\n // (which will be replaced with a
chain, replace the remaining
with a
\n // chain.\n if (replaced) {\n var p = this._doc.createElement(\"p\");\n br.parentNode.replaceChild(p, br);\n\n next = p.nextSibling;\n while (next) {\n // If we've hit another
, we're done adding children to this ![](\"/linkding-screenshot.png\"/)
![](\"/linkding-screenshot-dark.png\"/)