Repository: sissbruecker/linkding Branch: master Commit: 573b6f5411ea Files: 405 Total size: 1.7 MB Directory structure: gitextract_7gdsh6yf/ ├── .coveragerc ├── .devcontainer/ │ └── devcontainer.json ├── .dockerignore ├── .gitattributes ├── .github/ │ └── workflows/ │ ├── build-test.yaml │ ├── build.yaml │ └── main.yaml ├── .gitignore ├── CHANGELOG.md ├── LICENSE.txt ├── Makefile ├── README.md ├── SECURITY.md ├── assets/ │ ├── header.afdesign │ ├── logo-inset.afdesign │ ├── logo.afdesign │ └── social-preview.afdesign ├── bookmarks/ │ ├── __init__.py │ ├── admin.py │ ├── api/ │ │ ├── __init__.py │ │ ├── auth.py │ │ ├── routes.py │ │ └── serializers.py │ ├── apps.py │ ├── context_processors.py │ ├── feeds.py │ ├── forms.py │ ├── frontend/ │ │ ├── api.js │ │ ├── components/ │ │ │ ├── bookmark-page.js │ │ │ ├── clear-button.js │ │ │ ├── confirm-dropdown.js │ │ │ ├── details-modal.js │ │ │ ├── dev-tool.js │ │ │ ├── dropdown.js │ │ │ ├── filter-drawer.js │ │ │ ├── form.js │ │ │ ├── modal.js │ │ │ ├── search-autocomplete.js │ │ │ ├── tag-autocomplete.js │ │ │ └── upload-button.js │ │ ├── index.js │ │ ├── shortcuts.js │ │ └── utils/ │ │ ├── element.js │ │ ├── focus.js │ │ ├── input.js │ │ ├── position-controller.js │ │ ├── search-history.js │ │ └── tag-cache.js │ ├── management/ │ │ └── commands/ │ │ ├── backup.py │ │ ├── create_initial_superuser.py │ │ ├── enable_wal.py │ │ ├── ensure_superuser.py │ │ ├── full_backup.py │ │ ├── generate_secret_key.py │ │ ├── import_netscape.py │ │ └── migrate_tasks.py │ ├── middlewares.py │ ├── migrations/ │ │ ├── 0001_initial.py │ │ ├── 0002_auto_20190629_2303.py │ │ ├── 0003_auto_20200913_0656.py │ │ ├── 0004_auto_20200926_1028.py │ │ ├── 0005_auto_20210103_1212.py │ │ ├── 0006_bookmark_is_archived.py │ │ ├── 0007_userprofile.py │ │ ├── 0008_userprofile_bookmark_date_display.py │ │ ├── 0009_bookmark_web_archive_snapshot_url.py │ │ ├── 0010_userprofile_bookmark_link_target.py │ │ ├── 0011_userprofile_web_archive_integration.py │ │ ├── 0012_toast.py │ │ ├── 0013_web_archive_optin_toast.py │ │ ├── 0014_alter_bookmark_unread.py │ │ ├── 0015_feedtoken.py │ │ ├── 0016_bookmark_shared.py │ │ ├── 0017_userprofile_enable_sharing.py │ │ ├── 0018_bookmark_favicon_file.py │ │ ├── 0019_userprofile_enable_favicons.py │ │ ├── 0020_userprofile_tag_search.py │ │ ├── 0021_userprofile_display_url.py │ │ ├── 0022_bookmark_notes.py │ │ ├── 0023_userprofile_permanent_notes.py │ │ ├── 0024_userprofile_enable_public_sharing.py │ │ ├── 0025_userprofile_search_preferences.py │ │ ├── 0026_userprofile_custom_css.py │ │ ├── 0027_userprofile_bookmark_description_display_and_more.py │ │ ├── 0028_userprofile_display_archive_bookmark_action_and_more.py │ │ ├── 0029_bookmark_list_actions_toast.py │ │ ├── 0030_bookmarkasset.py │ │ ├── 0031_userprofile_enable_automatic_html_snapshots.py │ │ ├── 0032_html_snapshots_hint_toast.py │ │ ├── 0033_userprofile_default_mark_unread.py │ │ ├── 0034_bookmark_preview_image_file_and_more.py │ │ ├── 0035_userprofile_tag_grouping.py │ │ ├── 0036_userprofile_auto_tagging_rules.py │ │ ├── 0037_globalsettings.py │ │ ├── 0038_globalsettings_guest_profile_user.py │ │ ├── 0039_globalsettings_enable_link_prefetch.py │ │ ├── 0040_userprofile_items_per_page_and_more.py │ │ ├── 0041_merge_metadata.py │ │ ├── 0042_userprofile_custom_css_hash.py │ │ ├── 0043_userprofile_collapse_side_panel.py │ │ ├── 0044_bookmark_latest_snapshot.py │ │ ├── 0045_userprofile_hide_bundles_bookmarkbundle.py │ │ ├── 0046_add_url_normalized_field.py │ │ ├── 0047_populate_url_normalized_field.py │ │ ├── 0048_userprofile_default_mark_shared.py │ │ ├── 0049_userprofile_legacy_search.py │ │ ├── 0050_new_search_toast.py │ │ ├── 0051_fix_normalized_url.py │ │ ├── 0052_apitoken.py │ │ ├── 0053_migrate_api_tokens.py │ │ ├── 0054_bookmarkbundle_filter_shared_and_more.py │ │ └── __init__.py │ ├── models.py │ ├── queries.py │ ├── services/ │ │ ├── __init__.py │ │ ├── assets.py │ │ ├── auto_tagging.py │ │ ├── bookmarks.py │ │ ├── bundles.py │ │ ├── exporter.py │ │ ├── favicon_loader.py │ │ ├── importer.py │ │ ├── monolith.py │ │ ├── parser.py │ │ ├── preview_image_loader.py │ │ ├── search_query_parser.py │ │ ├── singlefile.py │ │ ├── tags.py │ │ ├── tasks.py │ │ ├── wayback.py │ │ └── website_loader.py │ ├── settings/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── custom.py │ │ ├── dev.py │ │ └── prod.py │ ├── signals.py │ ├── static/ │ │ ├── live-reload.js │ │ ├── robots.txt │ │ └── vendor/ │ │ └── Readability.js │ ├── styles/ │ │ ├── auth.css │ │ ├── bookmark-details.css │ │ ├── bookmark-form.css │ │ ├── bookmark-page.css │ │ ├── bundles.css │ │ ├── components.css │ │ ├── crud.css │ │ ├── layout.css │ │ ├── markdown.css │ │ ├── reader-mode.css │ │ ├── responsive.css │ │ ├── settings.css │ │ ├── tags.css │ │ ├── theme/ │ │ │ ├── LICENSE │ │ │ ├── _normalize.css │ │ │ ├── animations.css │ │ │ ├── asian.css │ │ │ ├── autocomplete.css │ │ │ ├── badges.css │ │ │ ├── base.css │ │ │ ├── buttons.css │ │ │ ├── code.css │ │ │ ├── dropdowns.css │ │ │ ├── empty.css │ │ │ ├── forms.css │ │ │ ├── menus.css │ │ │ ├── modals.css │ │ │ ├── pagination.css │ │ │ ├── tables.css │ │ │ ├── tabs.css │ │ │ ├── toasts.css │ │ │ ├── typography.css │ │ │ ├── utilities.css │ │ │ └── variables.css │ │ ├── theme-dark.css │ │ └── theme-light.css │ ├── tasks.py │ ├── templates/ │ │ ├── admin/ │ │ │ └── background_tasks.html │ │ ├── bookmarks/ │ │ │ ├── bookmark_list.html │ │ │ ├── bookmark_page.html │ │ │ ├── bulk_edit_bar.html │ │ │ ├── bundle_section.html │ │ │ ├── close.html │ │ │ ├── details/ │ │ │ │ ├── asset_icon.html │ │ │ │ ├── assets.html │ │ │ │ ├── form.html │ │ │ │ └── modal.html │ │ │ ├── edit.html │ │ │ ├── empty_bookmarks.html │ │ │ ├── form.html │ │ │ ├── new.html │ │ │ ├── read.html │ │ │ ├── search.html │ │ │ ├── tag_cloud.html │ │ │ ├── tag_section.html │ │ │ └── user_section.html │ │ ├── bundles/ │ │ │ ├── edit.html │ │ │ ├── form.html │ │ │ ├── index.html │ │ │ ├── new.html │ │ │ └── preview.html │ │ ├── opensearch.xml │ │ ├── registration/ │ │ │ ├── login.html │ │ │ ├── password_change_done.html │ │ │ └── password_change_form.html │ │ ├── settings/ │ │ │ ├── bookmarklet.js │ │ │ ├── bookmarklet_clientside.js │ │ │ ├── create_api_token_modal.html │ │ │ ├── general.html │ │ │ └── integrations.html │ │ ├── shared/ │ │ │ ├── dev_tool.html │ │ │ ├── error_list.html │ │ │ ├── head.html │ │ │ ├── layout.html │ │ │ ├── messages.html │ │ │ ├── modal_header.html │ │ │ ├── nav_menu.html │ │ │ ├── pagination.html │ │ │ └── top_frame.html │ │ └── tags/ │ │ ├── edit.html │ │ ├── form.html │ │ ├── index.html │ │ ├── merge.html │ │ └── new.html │ ├── templatetags/ │ │ ├── __init__.py │ │ ├── bookmarks.py │ │ ├── pagination.py │ │ └── shared.py │ ├── tests/ │ │ ├── __init__.py │ │ ├── helpers.py │ │ ├── resources/ │ │ │ ├── simple_valid_import_file.html │ │ │ └── simple_valid_import_file_with_one_invalid_bookmark.html │ │ ├── test_app_options.py │ │ ├── test_assets_service.py │ │ ├── test_auth_api.py │ │ ├── test_auth_proxy_support.py │ │ ├── test_auto_tagging.py │ │ ├── test_bookmark_action_view.py │ │ ├── test_bookmark_archived_view.py │ │ ├── test_bookmark_archived_view_performance.py │ │ ├── test_bookmark_asset_view.py │ │ ├── test_bookmark_assets.py │ │ ├── test_bookmark_assets_api.py │ │ ├── test_bookmark_details_modal.py │ │ ├── test_bookmark_edit_view.py │ │ ├── test_bookmark_index_view.py │ │ ├── test_bookmark_index_view_performance.py │ │ ├── test_bookmark_new_view.py │ │ ├── test_bookmark_previews.py │ │ ├── test_bookmark_search_form.py │ │ ├── test_bookmark_search_model.py │ │ ├── test_bookmark_search_tag.py │ │ ├── test_bookmark_shared_view.py │ │ ├── test_bookmark_shared_view_performance.py │ │ ├── test_bookmark_validation.py │ │ ├── test_bookmarks_api.py │ │ ├── test_bookmarks_api_performance.py │ │ ├── test_bookmarks_api_permissions.py │ │ ├── test_bookmarks_list_template.py │ │ ├── test_bookmarks_model.py │ │ ├── test_bookmarks_service.py │ │ ├── test_bookmarks_tasks.py │ │ ├── test_bundles_api.py │ │ ├── test_bundles_edit_view.py │ │ ├── test_bundles_index_view.py │ │ ├── test_bundles_new_view.py │ │ ├── test_bundles_preview_view.py │ │ ├── test_context_path.py │ │ ├── test_create_initial_superuser_command.py │ │ ├── test_custom_css_view.py │ │ ├── test_exporter.py │ │ ├── test_exporter_performance.py │ │ ├── test_favicon_loader.py │ │ ├── test_feeds.py │ │ ├── test_feeds_performance.py │ │ ├── test_health_view.py │ │ ├── test_importer.py │ │ ├── test_layout.py │ │ ├── test_linkding_middleware.py │ │ ├── test_login_view.py │ │ ├── test_metadata_view.py │ │ ├── test_monolith_service.py │ │ ├── test_oidc_support.py │ │ ├── test_opensearch_view.py │ │ ├── test_pagination_tag.py │ │ ├── test_parser.py │ │ ├── test_password_change_view.py │ │ ├── test_preview_image_loader.py │ │ ├── test_queries.py │ │ ├── test_root_view.py │ │ ├── test_search_query_parser.py │ │ ├── test_settings_export_view.py │ │ ├── test_settings_general_view.py │ │ ├── test_settings_import_view.py │ │ ├── test_settings_integrations_view.py │ │ ├── test_singlefile_service.py │ │ ├── test_tag_cloud_template.py │ │ ├── test_tags_edit_view.py │ │ ├── test_tags_index_view.py │ │ ├── test_tags_merge_view.py │ │ ├── test_tags_model.py │ │ ├── test_tags_new_view.py │ │ ├── test_tags_service.py │ │ ├── test_toasts_view.py │ │ ├── test_user_profile_model.py │ │ ├── test_user_select_tag.py │ │ ├── test_utils.py │ │ └── test_website_loader.py │ ├── tests_e2e/ │ │ ├── __init__.py │ │ ├── e2e_test_a11y_navigation_focus.py │ │ ├── e2e_test_bookmark_details_modal.py │ │ ├── e2e_test_bookmark_item.py │ │ ├── e2e_test_bookmark_page_bulk_edit.py │ │ ├── e2e_test_bookmark_page_partial_updates.py │ │ ├── e2e_test_bundle_preview.py │ │ ├── e2e_test_collapse_side_panel.py │ │ ├── e2e_test_dropdown.py │ │ ├── e2e_test_edit_bookmark_form.py │ │ ├── e2e_test_filter_drawer.py │ │ ├── e2e_test_global_shortcuts.py │ │ ├── e2e_test_new_bookmark_form.py │ │ ├── e2e_test_settings_general.py │ │ ├── e2e_test_settings_integrations.py │ │ ├── e2e_test_tag_management.py │ │ └── helpers.py │ ├── type_defs.py │ ├── urls.py │ ├── utils.py │ ├── validators.py │ ├── views/ │ │ ├── __init__.py │ │ ├── access.py │ │ ├── assets.py │ │ ├── auth.py │ │ ├── bookmarks.py │ │ ├── bundles.py │ │ ├── contexts.py │ │ ├── custom_css.py │ │ ├── health.py │ │ ├── manifest.py │ │ ├── opensearch.py │ │ ├── reload.py │ │ ├── root.py │ │ ├── settings.py │ │ ├── tags.py │ │ ├── toasts.py │ │ └── turbo.py │ ├── widgets.py │ └── wsgi.py ├── bootstrap.sh ├── docker/ │ ├── alpine.Dockerfile │ └── default.Dockerfile ├── docker-compose.yml ├── docs/ │ ├── .gitignore │ ├── README.md │ ├── astro.config.mjs │ ├── package.json │ ├── src/ │ │ ├── assets/ │ │ │ ├── Add To Linkding.shortcut │ │ │ └── linkding_shortcut.json │ │ ├── components/ │ │ │ ├── Card.astro │ │ │ └── icons.ts │ │ ├── content/ │ │ │ ├── config.ts │ │ │ └── docs/ │ │ │ ├── acknowledgements.md │ │ │ ├── admin.md │ │ │ ├── api.md │ │ │ ├── archiving.md │ │ │ ├── auto-tagging.md │ │ │ ├── backups.md │ │ │ ├── browser-extension.md │ │ │ ├── community.md │ │ │ ├── how-to.md │ │ │ ├── index.mdx │ │ │ ├── installation.md │ │ │ ├── managed-hosting.md │ │ │ ├── options.md │ │ │ ├── search.md │ │ │ ├── shortcuts.md │ │ │ └── troubleshooting.md │ │ ├── env.d.ts │ │ └── styles/ │ │ └── custom.css │ └── tsconfig.json ├── install-linkding.sh ├── manage.py ├── package.json ├── postcss.config.js ├── pyproject.toml ├── pytest.ini ├── rollup.config.mjs ├── scripts/ │ ├── build-docker.sh │ ├── coverage.sh │ ├── generate-changelog.py │ ├── release.sh │ ├── run-docker.sh │ ├── run-postgres.sh │ ├── setup-ublock.sh │ ├── test-environments/ │ │ ├── authelia-oidc/ │ │ │ ├── authelia/ │ │ │ │ ├── configuration.yml │ │ │ │ └── users_database.yml │ │ │ ├── compose.yml │ │ │ ├── setup.sh │ │ │ └── traefik/ │ │ │ └── certificates.yml │ │ └── postgres/ │ │ └── compose.yml │ └── test-postgres.sh ├── supervisord-all.conf ├── supervisord-tasks.conf ├── uwsgi.ini └── version.txt ================================================ FILE CONTENTS ================================================ ================================================ FILE: .coveragerc ================================================ [run] source = bookmarks omit = bookmarks/tests/* ================================================ FILE: .devcontainer/devcontainer.json ================================================ // For format details, see https://aka.ms/devcontainer.json. For config options, see the // README at: https://github.com/devcontainers/templates/tree/main/src/python { "name": "Python 3", "image": "mcr.microsoft.com/devcontainers/python:3.13", "features": { "ghcr.io/devcontainers/features/node:1": {} }, // Features to add to the dev container. More info: https://containers.dev/features. // "features": {}, // Use 'forwardPorts' to make a list of ports inside the container available locally. "forwardPorts": [8000], // Use 'postCreateCommand' to run commands after the container is created. "postCreateCommand": "pip install uv && uv sync --group dev && npm install && mkdir -p data && uv run manage.py migrate", // Configure tool-specific properties. "customizations": { "vscode": { "extensions": [ "ms-python.python" ] } }, "remoteUser": "vscode" } ================================================ FILE: .dockerignore ================================================ # Ignore everything * # Include files required for build or at runtime !/bookmarks !/bootstrap.sh !/LICENSE.txt !/manage.py !/package.json !/package-lock.json !/postcss.config.js !/pyproject.toml !/rollup.config.mjs !/supervisord-tasks.conf !/supervisord-all.conf !/uv.lock !/uwsgi.ini !/version.txt # Remove dev settings /bookmarks/settings/dev.py ================================================ FILE: .gitattributes ================================================ * text=auto *.sh text eol=lf ================================================ FILE: .github/workflows/build-test.yaml ================================================ name: build-test on: workflow_dispatch jobs: build: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Login to Docker Hub uses: docker/login-action@v3 with: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ github.token }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build latest uses: docker/build-push-action@v6 with: context: . file: ./docker/default.Dockerfile platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | ghcr.io/sissbruecker/linkding:test target: linkding push: true cache-from: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache cache-to: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache,mode=max - name: Build latest-alpine uses: docker/build-push-action@v6 with: context: . file: ./docker/alpine.Dockerfile platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | ghcr.io/sissbruecker/linkding:test-alpine target: linkding push: true cache-from: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache-alpine cache-to: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache-alpine,mode=max - name: Build latest-plus uses: docker/build-push-action@v6 with: context: . file: ./docker/default.Dockerfile platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | ghcr.io/sissbruecker/linkding:test-plus target: linkding-plus push: true cache-from: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache cache-to: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache,mode=max - name: Build latest-plus-alpine uses: docker/build-push-action@v6 with: context: . file: ./docker/alpine.Dockerfile platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | ghcr.io/sissbruecker/linkding:test-plus-alpine target: linkding-plus push: true cache-from: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache-alpine cache-to: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache-alpine,mode=max ================================================ FILE: .github/workflows/build.yaml ================================================ name: build on: workflow_dispatch jobs: build: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 - name: Read version from file id: get_version run: echo "VERSION=$(cat version.txt)" >> $GITHUB_ENV - name: Login to Docker Hub uses: docker/login-action@v3 with: username: ${{ vars.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ github.token }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build latest uses: docker/build-push-action@v6 with: context: . file: ./docker/default.Dockerfile platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | sissbruecker/linkding:latest sissbruecker/linkding:${{ env.VERSION }} ghcr.io/sissbruecker/linkding:latest ghcr.io/sissbruecker/linkding:${{ env.VERSION }} target: linkding push: true cache-from: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache cache-to: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache,mode=max - name: Build latest-alpine uses: docker/build-push-action@v6 with: context: . file: ./docker/alpine.Dockerfile platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | sissbruecker/linkding:latest-alpine sissbruecker/linkding:${{ env.VERSION }}-alpine ghcr.io/sissbruecker/linkding:latest-alpine ghcr.io/sissbruecker/linkding:${{ env.VERSION }}-alpine target: linkding push: true cache-from: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache-alpine cache-to: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache-alpine,mode=max - name: Build latest-plus uses: docker/build-push-action@v6 with: context: . file: ./docker/default.Dockerfile platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | sissbruecker/linkding:latest-plus sissbruecker/linkding:${{ env.VERSION }}-plus ghcr.io/sissbruecker/linkding:latest-plus ghcr.io/sissbruecker/linkding:${{ env.VERSION }}-plus target: linkding-plus push: true cache-from: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache cache-to: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache,mode=max - name: Build latest-plus-alpine uses: docker/build-push-action@v6 with: context: . file: ./docker/alpine.Dockerfile platforms: linux/amd64,linux/arm64,linux/arm/v7 tags: | sissbruecker/linkding:latest-plus-alpine sissbruecker/linkding:${{ env.VERSION }}-plus-alpine ghcr.io/sissbruecker/linkding:latest-plus-alpine ghcr.io/sissbruecker/linkding:${{ env.VERSION }}-plus-alpine target: linkding-plus push: true cache-from: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache-alpine cache-to: type=registry,ref=ghcr.io/sissbruecker/linkding:buildcache-alpine,mode=max ================================================ FILE: .github/workflows/main.yaml ================================================ name: linkding CI on: pull_request: push: branches: - master jobs: unit_tests: name: Unit Tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.13" - name: Install uv uses: astral-sh/setup-uv@v6 - name: Set up Node uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' - name: Install Node dependencies run: npm ci - name: Setup Python environment run: | uv sync mkdir data - name: Run tests run: uv run pytest -n auto e2e_tests: name: E2E Tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.13" - name: Install uv uses: astral-sh/setup-uv@v6 - name: Set up Node uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' - name: Install Node dependencies run: npm ci - name: Setup Python environment run: | uv sync uv run playwright install chromium mkdir data - name: Run build run: | npm run build uv run manage.py collectstatic - name: Run tests run: uv run pytest bookmarks/tests_e2e -n auto -o "python_files=e2e_test_*.py" - name: Upload screenshots if: failure() uses: actions/upload-artifact@v4 with: name: e2e-screenshots path: test-results/screenshots ================================================ FILE: .gitignore ================================================ # Created by .ignore support plugin (hsz.mobi) ### JetBrains template # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 # File-based project format *.iws # IntelliJ .idea *.iml out/ ### Python template # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ .pytest_cache/ test-results/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ ### Node template # Logs logs *.log npm-debug.log* yarn-debug.log* yarn-error.log* # Runtime data pids *.pid *.seed *.pid.lock # Directory for instrumented libs generated by jscoverage/JSCover lib-cov # Coverage directory used by tools like istanbul coverage # nyc test coverage .nyc_output # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) .grunt # Bower dependency directory (https://bower.io/) bower_components # node-waf configuration .lock-wscript # Compiled binary addons (https://nodejs.org/api/addons.html) build/Release # Dependency directories node_modules/ jspm_packages/ # TypeScript v1 declaration files typings/ # Optional npm cache directory .npm # Optional eslint cache .eslintcache # Optional REPL history .node_repl_history # Output of 'npm pack' *.tgz # Yarn Integrity file .yarn-integrity # dotenv environment variables file .env # next.js build output .next ### Custom # Rollup compilation output /bookmarks/static/bundle.js* # CSS compilation output /bookmarks/static/theme-*.css* # Collected static files for deployment /static # Build output, etc. /tmp # Database file /data # ublock + chromium /uBOLite.chromium.mv3 /chromium-profile # direnv /.direnv # Test setups /scripts/unsecure-test-setups/authelia-oidc/authelia/db.sqlite3 /scripts/unsecure-test-setups/authelia-oidc/traefik/certs ================================================ FILE: CHANGELOG.md ================================================ # Changelog ## v1.45.0 (06/01/2026) ### What's Changed * API token management by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1248 * Add option to disable login form by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1269 * Turn scheme-less URLs into HTTPS instead of HTTP links by @Maaxxs in https://github.com/sissbruecker/linkding/pull/1225 * Disable bulk execute button when no bookmarks selected by @emanuelebeffa in https://github.com/sissbruecker/linkding/pull/1241 * Add option to run supervisor as main process by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1270 * Allow setting date_added and date_modified for new bookmarks through REST API by @jmason in https://github.com/sissbruecker/linkding/pull/1063 * Download PDF instead of creating HTML snapshot if URL points at PDF by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1271 * Allow sandboxed scripts when viewing assets by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1252 * Allow viewing video assets by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1259 * Remove absolute URIs from settings page by @packrat386 in https://github.com/sissbruecker/linkding/pull/1261 * Move tag management forms into dialogs by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1253 * Move bulk edit checkboxes into bookmark list container by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1257 * Remove registration switch by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1268 * Add linkdinger to community projects by @lmmendes in https://github.com/sissbruecker/linkding/pull/1266 ### New Contributors * @packrat386 made their first contribution in https://github.com/sissbruecker/linkding/pull/1261 * @lmmendes made their first contribution in https://github.com/sissbruecker/linkding/pull/1266 * @Maaxxs made their first contribution in https://github.com/sissbruecker/linkding/pull/1225 * @emanuelebeffa made their first contribution in https://github.com/sissbruecker/linkding/pull/1241 * @jmason made their first contribution in https://github.com/sissbruecker/linkding/pull/1063 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.44.2...v1.45.0 --- ## v1.44.2 (13/12/2025) ### What's Changed > [!WARNING] > *This resolves a [security vulnerability](https://github.com/sissbruecker/linkding/security/advisories/GHSA-3pf9-5cjv-2w7q) in linkding. Everyone is encouraged to upgrade to the latest version as soon as possible.* * Use sandbox CSP for viewing assets by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1245 * Fix devcontainer by @m3eno in https://github.com/sissbruecker/linkding/pull/1208 * Fix tag cloud highlighting first char when tags are not grouped by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1209 * Bump supervisor to 4.3.0 to fix warning by @simonhammes in https://github.com/sissbruecker/linkding/pull/1216 * Added Javascript client and library for Linkding REST API by @vbsampath in https://github.com/sissbruecker/linkding/pull/1195 * Add Komrade project to community resources by @dev-inside in https://github.com/sissbruecker/linkding/pull/1236 ### New Contributors * @m3eno made their first contribution in https://github.com/sissbruecker/linkding/pull/1208 * @vbsampath made their first contribution in https://github.com/sissbruecker/linkding/pull/1195 * @dev-inside made their first contribution in https://github.com/sissbruecker/linkding/pull/1236 * @simonhammes made their first contribution in https://github.com/sissbruecker/linkding/pull/1216 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.44.1...v1.44.2 --- ## v1.44.1 (11/10/2025) ### What's Changed * Fix normalized URL not being generated in bookmark import by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1202 * Fix missing tags causing errors in import with Postgres by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1203 * Check for dupes by exact URL if normalized URL is missing by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1204 * Attempt to fix botched normalized URL migration from 1.43.0 by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1205 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.44.0...v1.44.1 --- ## v1.44.0 (05/10/2025) ### What's Changed * Add new search engine that supports logical expressions (and, or, not) by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1198 * Fix pagination links to use relative URLs by @dunlor in https://github.com/sissbruecker/linkding/pull/1186 * Fix queued tasks link when context path is used by @dunlor in https://github.com/sissbruecker/linkding/pull/1187 * Fix bundle preview pagination resetting to first page by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1194 ### New Contributors * @dunlor made their first contribution in https://github.com/sissbruecker/linkding/pull/1186 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.43.0...v1.44.0 --- ## v1.43.0 (28/09/2025) ### What's Changed * Add basic tag management by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1175 * Normalize URLs when checking for duplicates by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1169 * Add option to mark bookmarks as shared by default by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1170 * Use modal dialog for confirming actions by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1168 * Fix error when filtering bookmark assets in the admin UI by @proog in https://github.com/sissbruecker/linkding/pull/1162 * Document API bundle filter by @proog in https://github.com/sissbruecker/linkding/pull/1161 * Add alfred-linkding-bookmarks to community.md by @FireFingers21 in https://github.com/sissbruecker/linkding/pull/1160 * Switch to uv by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1172 * Replace Svelte components with Lit elements by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1174 * Bump versions by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1173 * Bump astro from 5.12.8 to 5.13.2 in /docs by @dependabot[bot] in https://github.com/sissbruecker/linkding/pull/1166 * Bump vite from 6.3.5 to 6.3.6 in /docs by @dependabot[bot] in https://github.com/sissbruecker/linkding/pull/1184 ### New Contributors * @FireFingers21 made their first contribution in https://github.com/sissbruecker/linkding/pull/1160 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.42.0...v1.43.0 --- ## v1.42.0 (16/08/2025) ### What's Changed * Bulk create HTML snapshots by @Tql-ws1 in https://github.com/sissbruecker/linkding/pull/1132 * Create bundle from current search query by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1154 * Add alternative bookmarklet that uses browser metadata by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1159 * Add date and time to HTML export filename by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1101 * Automatically compress uploads with gzip by @hkclark in https://github.com/sissbruecker/linkding/pull/1087 * Show bookmark bundles in admin by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1110 * Allow filtering feeds by bundle by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1152 * Submit bookmark form with Ctrl/Cmd + Enter by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1158 * Improve bookmark form accessibility by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1116 * Fix custom CSS not being used in reader mode by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1102 * Use filename when downloading asset through UI by @proog in https://github.com/sissbruecker/linkding/pull/1146 * Update order when deleting bundle by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1114 * Wrap long titles in bookmark details modal by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1150 * Ignore tags with just whitespace by @pvl in https://github.com/sissbruecker/linkding/pull/1125 * Ignore tags that exceed length limit during import by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1153 * Add CloudBreak on Managed Hosting by @benjaminoakes in https://github.com/sissbruecker/linkding/pull/1079 * Add Pocket migration to to community page by @hkclark in https://github.com/sissbruecker/linkding/pull/1112 * Add linkding-media-archiver to community.md by @proog in https://github.com/sissbruecker/linkding/pull/1144 * Bump astro from 5.7.13 to 5.12.8 in /docs by @dependabot[bot] in https://github.com/sissbruecker/linkding/pull/1147 ### New Contributors * @hkclark made their first contribution in https://github.com/sissbruecker/linkding/pull/1087 * @benjaminoakes made their first contribution in https://github.com/sissbruecker/linkding/pull/1079 * @proog made their first contribution in https://github.com/sissbruecker/linkding/pull/1146 * @pvl made their first contribution in https://github.com/sissbruecker/linkding/pull/1125 * @Tql-ws1 made their first contribution in https://github.com/sissbruecker/linkding/pull/1132 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.41.0...v1.42.0 --- ## v1.41.0 (19/06/2025) ### What's Changed * Add bundles for organizing bookmarks by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1097 * Add REST API for bookmark bundles by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1100 * Add date filters for REST API by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1080 * Fix side panel not being hidden on smaller viewports by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1089 * Fix assets not using correct icon by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1098 * Add LinkBuddy to community section by @peterto in https://github.com/sissbruecker/linkding/pull/1088 * Bump tar-fs in /docs by @dependabot in https://github.com/sissbruecker/linkding/pull/1084 * Bump django from 5.1.9 to 5.1.10 by @dependabot in https://github.com/sissbruecker/linkding/pull/1086 * Bump requests from 2.32.3 to 2.32.4 by @dependabot in https://github.com/sissbruecker/linkding/pull/1090 * Bump urllib3 from 2.2.3 to 2.5.0 by @dependabot in https://github.com/sissbruecker/linkding/pull/1096 ### New Contributors * @peterto made their first contribution in https://github.com/sissbruecker/linkding/pull/1088 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.40.0...v1.41.0 --- ## v1.40.0 (17/05/2025) ### What's Changed * Add bulk and single bookmark metadata refresh by @Teknicallity in https://github.com/sissbruecker/linkding/pull/999 * Prefer local snapshot over web archive link in bookmark list links by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1021 * Push Docker images to GHCR in addition to Docker Hub by @caycehouse in https://github.com/sissbruecker/linkding/pull/1024 * Allow auto tagging rules to match URL fragments by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1045 * Linkify plain URLs in notes by @sonicdoe in https://github.com/sissbruecker/linkding/pull/1051 * Add opensearch declaration by @jzorn in https://github.com/sissbruecker/linkding/pull/1058 * Allow pre-filling tags in new bookmark form by @dasrecht in https://github.com/sissbruecker/linkding/pull/1060 * Handle lowercase "true" in environment variables by @jose-elias-alvarez in https://github.com/sissbruecker/linkding/pull/1020 * Accessibility improvements in page structure by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1014 * Improve announcements after navigation by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1015 * Fix OIDC login link by @cite in https://github.com/sissbruecker/linkding/pull/1019 * Fix bookmark asset download endpoint by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1033 * Add docs for auto tagging by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1009 * Fix typo in index.mdx tagline by @cenviity in https://github.com/sissbruecker/linkding/pull/1052 * Add how-to for using linkding PWA in native Android share sheet by @kzshantonu in https://github.com/sissbruecker/linkding/pull/1055 * Adding linktiles to community projects by @haondt in https://github.com/sissbruecker/linkding/pull/1025 * Bump django from 5.1.5 to 5.1.7 by @dependabot in https://github.com/sissbruecker/linkding/pull/1007 * Bump django from 5.1.7 to 5.1.8 by @dependabot in https://github.com/sissbruecker/linkding/pull/1030 * Bump tar-fs in /docs by @dependabot in https://github.com/sissbruecker/linkding/pull/1028 * Bump prismjs from 1.29.0 to 1.30.0 in /docs by @dependabot in https://github.com/sissbruecker/linkding/pull/1034 * Bump @babel/helpers from 7.26.7 to 7.27.0 in /docs by @dependabot in https://github.com/sissbruecker/linkding/pull/1035 * Bump vite from 5.4.14 to 5.4.17 in /docs by @dependabot in https://github.com/sissbruecker/linkding/pull/1036 * Bump esbuild, @astrojs/starlight and astro in /docs by @dependabot in https://github.com/sissbruecker/linkding/pull/1037 * Bump django from 5.1.8 to 5.1.9 by @dependabot in https://github.com/sissbruecker/linkding/pull/1059 ### New Contributors * @cite made their first contribution in https://github.com/sissbruecker/linkding/pull/1019 * @jose-elias-alvarez made their first contribution in https://github.com/sissbruecker/linkding/pull/1020 * @Teknicallity made their first contribution in https://github.com/sissbruecker/linkding/pull/999 * @haondt made their first contribution in https://github.com/sissbruecker/linkding/pull/1025 * @caycehouse made their first contribution in https://github.com/sissbruecker/linkding/pull/1024 * @cenviity made their first contribution in https://github.com/sissbruecker/linkding/pull/1052 * @sonicdoe made their first contribution in https://github.com/sissbruecker/linkding/pull/1051 * @jzorn made their first contribution in https://github.com/sissbruecker/linkding/pull/1058 * @dasrecht made their first contribution in https://github.com/sissbruecker/linkding/pull/1060 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.39.1...v1.40.0 --- ## v1.39.1 (06/03/2025) > [!WARNING] > Due to changes in the release process the `1.39.0` Docker image accidentally runs the application in debug mode. Please upgrade to `1.39.1` instead. --- ## v1.39.0 (06/03/2025) ### What's Changed * Add REST endpoint for uploading snapshots from the Singlefile extension by @sissbruecker in https://github.com/sissbruecker/linkding/pull/996 * Add bookmark assets API by @sissbruecker in https://github.com/sissbruecker/linkding/pull/1003 * Allow providing REST API authentication token with Bearer keyword by @sissbruecker in https://github.com/sissbruecker/linkding/pull/995 * Add Telegram bot to community section by @marb08 in https://github.com/sissbruecker/linkding/pull/1001 * Adding linklater to community projects by @nsartor in https://github.com/sissbruecker/linkding/pull/1002 ### New Contributors * @marb08 made their first contribution in https://github.com/sissbruecker/linkding/pull/1001 * @nsartor made their first contribution in https://github.com/sissbruecker/linkding/pull/1002 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.38.1...v1.39.0 --- ## v1.38.1 (22/02/2025) ### What's Changed * Remove preview image when bookmark is deleted by @sissbruecker in https://github.com/sissbruecker/linkding/pull/989 * Try limit uwsgi memory usage by configuring file descriptor limit by @sissbruecker in https://github.com/sissbruecker/linkding/pull/990 * Add note about OIDC and LD_SUPERUSER_NAME combination by @tebriel in https://github.com/sissbruecker/linkding/pull/992 * Return web archive fallback URL from REST API by @sissbruecker in https://github.com/sissbruecker/linkding/pull/993 * Fix auth proxy logout by @sissbruecker in https://github.com/sissbruecker/linkding/pull/994 ### New Contributors * @tebriel made their first contribution in https://github.com/sissbruecker/linkding/pull/992 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.38.0...v1.38.1 --- ## v1.38.0 (09/02/2025) ### What's Changed * Fix nav menu closing on mousedown in Safari by @sissbruecker in https://github.com/sissbruecker/linkding/pull/965 * Allow customizing username when creating user through OIDC by @kyuuk in https://github.com/sissbruecker/linkding/pull/971 * Improve accessibility of modal dialogs by @sissbruecker in https://github.com/sissbruecker/linkding/pull/974 * Add option to collapse side panel by @sissbruecker in https://github.com/sissbruecker/linkding/pull/975 * Convert tag modal into drawer by @sissbruecker in https://github.com/sissbruecker/linkding/pull/977 * Add RSS link to shared bookmarks page by @sissbruecker in https://github.com/sissbruecker/linkding/pull/984 * Add Additional iOS Shortcut to community section by @joshdick in https://github.com/sissbruecker/linkding/pull/968 ### New Contributors * @kyuuk made their first contribution in https://github.com/sissbruecker/linkding/pull/971 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.37.0...v1.38.0 --- ## v1.37.0 (26/01/2025) ### What's Changed * Add option to disable request logs by @dmarcoux in https://github.com/sissbruecker/linkding/pull/887 * Add default robots.txt to block crawlers by @sissbruecker in https://github.com/sissbruecker/linkding/pull/959 * Fix menu dropdown focus traps by @sissbruecker in https://github.com/sissbruecker/linkding/pull/944 * Provide accessible name to radio groups by @sissbruecker in https://github.com/sissbruecker/linkding/pull/945 * Add serchding to community projects, sort the list by alphabetical order by @ldwgchen in https://github.com/sissbruecker/linkding/pull/880 * Add cosmicding To Community Resources by @vkhitrin in https://github.com/sissbruecker/linkding/pull/892 * Add 3 new community projects by @sebw in https://github.com/sissbruecker/linkding/pull/949 * Add a rust client library to community.md by @zbrox in https://github.com/sissbruecker/linkding/pull/914 * Update community.md by @justusthane in https://github.com/sissbruecker/linkding/pull/897 * Bump astro from 4.15.8 to 4.16.3 in /docs by @dependabot in https://github.com/sissbruecker/linkding/pull/884 * Bump vite from 5.4.9 to 5.4.14 in /docs by @dependabot in https://github.com/sissbruecker/linkding/pull/953 * Bump django from 5.1.1 to 5.1.5 by @dependabot in https://github.com/sissbruecker/linkding/pull/947 * Bump nanoid from 3.3.7 to 3.3.8 by @dependabot in https://github.com/sissbruecker/linkding/pull/928 * Bump astro from 4.16.3 to 4.16.18 in /docs by @dependabot in https://github.com/sissbruecker/linkding/pull/929 * Bump nanoid from 3.3.7 to 3.3.8 in /docs by @dependabot in https://github.com/sissbruecker/linkding/pull/962 ### New Contributors * @ldwgchen made their first contribution in https://github.com/sissbruecker/linkding/pull/880 * @dmarcoux made their first contribution in https://github.com/sissbruecker/linkding/pull/887 * @vkhitrin made their first contribution in https://github.com/sissbruecker/linkding/pull/892 * @sebw made their first contribution in https://github.com/sissbruecker/linkding/pull/949 * @justusthane made their first contribution in https://github.com/sissbruecker/linkding/pull/897 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.36.0...v1.37.0 --- ## v1.36.0 (02/10/2024) ### What's Changed * Replace uBlock Origin with uBlock Origin Lite by @sissbruecker in https://github.com/sissbruecker/linkding/pull/866 * Add LAST_MODIFIED attribute when exporting by @ixzhao in https://github.com/sissbruecker/linkding/pull/860 * Return client error status code for invalid form submissions by @sissbruecker in https://github.com/sissbruecker/linkding/pull/849 * Fix header.svg text by @vladh in https://github.com/sissbruecker/linkding/pull/850 * Do not clear fields in POST requests (API behavior change) by @sissbruecker in https://github.com/sissbruecker/linkding/pull/852 * Prevent duplicates when editing by @sissbruecker in https://github.com/sissbruecker/linkding/pull/853 * Fix jumping details modal on back navigation by @sissbruecker in https://github.com/sissbruecker/linkding/pull/854 * Fix select dropdown menu background in dark theme by @sissbruecker in https://github.com/sissbruecker/linkding/pull/858 * Do not escape valid characters in custom CSS by @sissbruecker in https://github.com/sissbruecker/linkding/pull/863 * Simplify Docker build by @sissbruecker in https://github.com/sissbruecker/linkding/pull/865 * Improve error handling for auto tagging by @sissbruecker in https://github.com/sissbruecker/linkding/pull/855 * Bump rollup from 4.13.0 to 4.22.4 by @dependabot in https://github.com/sissbruecker/linkding/pull/851 * Bump rollup from 4.21.3 to 4.22.4 in /docs by @dependabot in https://github.com/sissbruecker/linkding/pull/856 ### New Contributors * @vladh made their first contribution in https://github.com/sissbruecker/linkding/pull/850 * @ixzhao made their first contribution in https://github.com/sissbruecker/linkding/pull/860 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.35.0...v1.36.0 --- ## v1.35.0 (23/09/2024) ### What's Changed * Add configuration options for pagination by @sissbruecker in https://github.com/sissbruecker/linkding/pull/835 * Show placeholder if there is no preview image by @sissbruecker in https://github.com/sissbruecker/linkding/pull/842 * Allow bookmarks to have empty title and description by @sissbruecker in https://github.com/sissbruecker/linkding/pull/843 * Add clear buttons in bookmark form by @sissbruecker in https://github.com/sissbruecker/linkding/pull/846 * Add basic fail2ban support by @sissbruecker in https://github.com/sissbruecker/linkding/pull/847 * Add documentation website by @sissbruecker in https://github.com/sissbruecker/linkding/pull/833 * Add go-linkding to community projects by @piero-vic in https://github.com/sissbruecker/linkding/pull/836 * Fix a broken link to options documentation by @zbrox in https://github.com/sissbruecker/linkding/pull/844 * Use HTTPS repository link for devcontainer by @voltagex in https://github.com/sissbruecker/linkding/pull/837 * Bump requests version to 3.23.3 by @voltagex in https://github.com/sissbruecker/linkding/pull/839 * Bump path-to-regexp and astro in /docs by @dependabot in https://github.com/sissbruecker/linkding/pull/840 * Bump dependencies by @sissbruecker in https://github.com/sissbruecker/linkding/pull/841 ### New Contributors * @piero-vic made their first contribution in https://github.com/sissbruecker/linkding/pull/836 * @voltagex made their first contribution in https://github.com/sissbruecker/linkding/pull/839 * @zbrox made their first contribution in https://github.com/sissbruecker/linkding/pull/844 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.34.0...v1.35.0 --- ## v1.34.0 (16/09/2024) ### What's Changed * Fix several issues around browser back navigation by @sissbruecker in https://github.com/sissbruecker/linkding/pull/825 * Speed up response times for certain actions by @sissbruecker in https://github.com/sissbruecker/linkding/pull/829 * Implement IPv6 capability by @itz-Jana in https://github.com/sissbruecker/linkding/pull/826 ### New Contributors * @itz-Jana made their first contribution in https://github.com/sissbruecker/linkding/pull/826 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.33.0...v1.34.0 --- ## v1.33.0 (14/09/2024) ### What's Changed * Theme improvements by @sissbruecker in https://github.com/sissbruecker/linkding/pull/822 * Speed up navigation by @sissbruecker in https://github.com/sissbruecker/linkding/pull/824 * Rename "SingeFileError" to "SingleFileError" by @curiousleo in https://github.com/sissbruecker/linkding/pull/823 * Bump svelte from 4.2.12 to 4.2.19 by @dependabot in https://github.com/sissbruecker/linkding/pull/806 ### New Contributors * @curiousleo made their first contribution in https://github.com/sissbruecker/linkding/pull/823 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.32.0...v1.33.0 --- ## v1.32.0 (10/09/2024) ### What's Changed * Allow configuring landing page for unauthenticated users by @sissbruecker in https://github.com/sissbruecker/linkding/pull/808 * Allow configuring guest user profile by @sissbruecker in https://github.com/sissbruecker/linkding/pull/809 * Return bookmark tags in RSS feeds by @sissbruecker in https://github.com/sissbruecker/linkding/pull/810 * Additional filter parameters for RSS feeds by @sissbruecker in https://github.com/sissbruecker/linkding/pull/811 * Allow pre-filling notes in new bookmark form by @sissbruecker in https://github.com/sissbruecker/linkding/pull/812 * Fix inconsistent tag order in bookmarks by @sissbruecker in https://github.com/sissbruecker/linkding/pull/819 * Fix auto-tagging when URL includes port by @sissbruecker in https://github.com/sissbruecker/linkding/pull/820 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.31.1...v1.32.0 --- ## v1.31.1 (30/08/2024) ### What's Changed * Include favicons and thumbnails in REST API by @sissbruecker in https://github.com/sissbruecker/linkding/pull/763 * Add Pinkt to the Community section by @fibelatti in https://github.com/sissbruecker/linkding/pull/772 * removed version line from docker compose yaml by @volumedata21 in https://github.com/sissbruecker/linkding/pull/800 * Add resource linkding logo by @QYG2297248353 in https://github.com/sissbruecker/linkding/pull/788 * Allow use of standard docker `TZ` env var by @watsonbox in https://github.com/sissbruecker/linkding/pull/765 * Add OCI source annotation to link back to source repo by @Ramblurr in https://github.com/sissbruecker/linkding/pull/701 * Generate fallback URLs for web archive links by @sissbruecker in https://github.com/sissbruecker/linkding/pull/804 * Fix overflow in settings page by @sissbruecker in https://github.com/sissbruecker/linkding/pull/805 * Bump django from 5.0.3 to 5.0.8 by @dependabot in https://github.com/sissbruecker/linkding/pull/795 * Bump certifi from 2023.11.17 to 2024.7.4 by @dependabot in https://github.com/sissbruecker/linkding/pull/775 * Bump djangorestframework from 3.14.0 to 3.15.2 by @dependabot in https://github.com/sissbruecker/linkding/pull/769 * Bump urllib3 from 2.1.0 to 2.2.2 by @dependabot in https://github.com/sissbruecker/linkding/pull/762 ### New Contributors * @fibelatti made their first contribution in https://github.com/sissbruecker/linkding/pull/772 * @volumedata21 made their first contribution in https://github.com/sissbruecker/linkding/pull/800 * @QYG2297248353 made their first contribution in https://github.com/sissbruecker/linkding/pull/788 * @watsonbox made their first contribution in https://github.com/sissbruecker/linkding/pull/765 * @Ramblurr made their first contribution in https://github.com/sissbruecker/linkding/pull/701 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.31.0...v1.31.1 --- ## v1.31.0 (16/06/2024) ### What's Changed * Add support for bookmark thumbnails by @vslinko in https://github.com/sissbruecker/linkding/pull/721 * Automatically add tags to bookmarks based on URL pattern by @vslinko in https://github.com/sissbruecker/linkding/pull/736 * Load bookmark thumbnails after import by @vslinko in https://github.com/sissbruecker/linkding/pull/724 * Load missing thumbnails after enabling the feature by @sissbruecker in https://github.com/sissbruecker/linkding/pull/725 * Thumbnails lazy loading by @vslinko in https://github.com/sissbruecker/linkding/pull/734 * Add option for disabling tag grouping by @vslinko in https://github.com/sissbruecker/linkding/pull/735 * Preview auto tags in bookmark form by @sissbruecker in https://github.com/sissbruecker/linkding/pull/737 * Hide tooltip on mobile by @vslinko in https://github.com/sissbruecker/linkding/pull/733 * Bump requests from 2.31.0 to 2.32.0 by @dependabot in https://github.com/sissbruecker/linkding/pull/740 ### New Contributors * @vslinko made their first contribution in https://github.com/sissbruecker/linkding/pull/721 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.30.0...v1.31.0 --- ## v1.30.0 (20/04/2024) ### What's Changed * Add reader mode by @sissbruecker in https://github.com/sissbruecker/linkding/pull/703 * Allow uploading custom files for bookmarks by @sissbruecker in https://github.com/sissbruecker/linkding/pull/713 * Add option for marking bookmarks as unread by default by @ab623 in https://github.com/sissbruecker/linkding/pull/706 * Make blocking cookie banners more reliable by @sissbruecker in https://github.com/sissbruecker/linkding/pull/699 * Close bookmark details with escape by @sissbruecker in https://github.com/sissbruecker/linkding/pull/702 * Show proper name for bookmark assets in admin by @ab623 in https://github.com/sissbruecker/linkding/pull/708 * Bump sqlparse from 0.4.4 to 0.5.0 by @dependabot in https://github.com/sissbruecker/linkding/pull/704 ### New Contributors * @ab623 made their first contribution in https://github.com/sissbruecker/linkding/pull/706 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.29.0...v1.30.0 --- ## v1.29.0 (14/04/2024) ### What's Changed * Remove ads and cookie banners from HTML snapshots by @sissbruecker in https://github.com/sissbruecker/linkding/pull/695 * Add button for creating missing HTML snapshots by @sissbruecker in https://github.com/sissbruecker/linkding/pull/696 * Refresh file list when there are queued snapshots by @sissbruecker in https://github.com/sissbruecker/linkding/pull/697 * Bump idna from 3.6 to 3.7 by @dependabot in https://github.com/sissbruecker/linkding/pull/694 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.28.0...v1.29.0 --- ## v1.28.0 (09/04/2024) ### What's Changed * Add option to disable SSL verification for OIDC by @akaSyntaax in https://github.com/sissbruecker/linkding/pull/684 * Add full backup method by @sissbruecker in https://github.com/sissbruecker/linkding/pull/686 * Truncate snapshot filename for long URLs by @sissbruecker in https://github.com/sissbruecker/linkding/pull/687 * Add option for customizing single-file timeout by @pettijohn in https://github.com/sissbruecker/linkding/pull/688 * Add option for passing arguments to single-file command by @pettijohn in https://github.com/sissbruecker/linkding/pull/691 * Fix typo by @tianheg in https://github.com/sissbruecker/linkding/pull/689 ### New Contributors * @akaSyntaax made their first contribution in https://github.com/sissbruecker/linkding/pull/684 * @pettijohn made their first contribution in https://github.com/sissbruecker/linkding/pull/688 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.27.1...v1.28.0 --- ## v1.27.1 (07/04/2024) ### What's Changed * Fix HTML snapshot errors related to single-file-cli by @sissbruecker in https://github.com/sissbruecker/linkding/pull/683 * Replace django-background-tasks with huey by @sissbruecker in https://github.com/sissbruecker/linkding/pull/657 * Add Authelia OIDC example to docs by @hugo-vrijswijk in https://github.com/sissbruecker/linkding/pull/675 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.27.0...v1.27.1 --- ## v1.27.0 (01/04/2024) ### What's Changed * Archive snapshots of websites locally by @sissbruecker in https://github.com/sissbruecker/linkding/pull/672 * Add Railway hosting option by @tianheg in https://github.com/sissbruecker/linkding/pull/661 * Add how to for increasing the font size by @sissbruecker in https://github.com/sissbruecker/linkding/pull/667 ### New Contributors * @tianheg made their first contribution in https://github.com/sissbruecker/linkding/pull/661 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.26.0...v1.27.0 --- ## v1.26.0 (30/03/2024) ### What's Changed * Add option for showing bookmark description as separate block by @sissbruecker in https://github.com/sissbruecker/linkding/pull/663 * Add bookmark details view by @sissbruecker in https://github.com/sissbruecker/linkding/pull/665 * Make bookmark list actions configurable by @sissbruecker in https://github.com/sissbruecker/linkding/pull/666 * Bump black from 24.1.1 to 24.3.0 by @dependabot in https://github.com/sissbruecker/linkding/pull/662 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.25.0...v1.26.0 --- ## v1.25.0 (18/03/2024) ### What's Changed * Improve PWA capabilities by @hugo-vrijswijk in https://github.com/sissbruecker/linkding/pull/630 * build improvements by @hugo-vrijswijk in https://github.com/sissbruecker/linkding/pull/649 * Add support for oidc by @Nighmared in https://github.com/sissbruecker/linkding/pull/389 * Add option for custom CSS by @sissbruecker in https://github.com/sissbruecker/linkding/pull/652 * Update backup location to safe directory by @bphenriques in https://github.com/sissbruecker/linkding/pull/653 * Include web archive link in /api/bookmarks/ by @sissbruecker in https://github.com/sissbruecker/linkding/pull/655 * Add RSS feeds for shared bookmarks by @sissbruecker in https://github.com/sissbruecker/linkding/pull/656 * Bump django from 5.0.2 to 5.0.3 by @dependabot in https://github.com/sissbruecker/linkding/pull/658 ### New Contributors * @hugo-vrijswijk made their first contribution in https://github.com/sissbruecker/linkding/pull/630 * @Nighmared made their first contribution in https://github.com/sissbruecker/linkding/pull/389 * @bphenriques made their first contribution in https://github.com/sissbruecker/linkding/pull/653 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.24.2...v1.25.0 --- ## v1.24.2 (16/03/2024) ### What's Changed * Fix logout button by @sissbruecker in https://github.com/sissbruecker/linkding/pull/648 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.24.1...v1.24.2 --- ## v1.24.1 (16/03/2024) ### What's Changed * Bump dependencies by @sissbruecker in https://github.com/sissbruecker/linkding/pull/618 * Persist secret key in data folder by @sissbruecker in https://github.com/sissbruecker/linkding/pull/620 * Group ideographic characters in tag cloud by @jonathan-s in https://github.com/sissbruecker/linkding/pull/613 * Bump django from 5.0.1 to 5.0.2 by @dependabot in https://github.com/sissbruecker/linkding/pull/625 * Add k8s setup to community section by @jzck in https://github.com/sissbruecker/linkding/pull/633 * Added a new Linkding client to community section by @JGeek00 in https://github.com/sissbruecker/linkding/pull/638 ### New Contributors * @jzck made their first contribution in https://github.com/sissbruecker/linkding/pull/633 * @JGeek00 made their first contribution in https://github.com/sissbruecker/linkding/pull/638 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.24.0...v1.24.1 --- ## v1.24.0 (27/01/2024) ### What's Changed * Support Open Graph description by @jonathan-s in https://github.com/sissbruecker/linkding/pull/602 * Add tooltip to truncated bookmark titles by @jonathan-s in https://github.com/sissbruecker/linkding/pull/607 * Improve bulk tag performance by @sissbruecker in https://github.com/sissbruecker/linkding/pull/612 * Increase tag limit in tag autocomplete by @hypebeast in https://github.com/sissbruecker/linkding/pull/581 * Add CapRover as managed hosting option by @adamshand in https://github.com/sissbruecker/linkding/pull/585 * Bump playwright dependencies by @jonathan-s in https://github.com/sissbruecker/linkding/pull/601 * Adjust archive.org donation link in general.html by @JnsDornbusch in https://github.com/sissbruecker/linkding/pull/603 ### New Contributors * @hypebeast made their first contribution in https://github.com/sissbruecker/linkding/pull/581 * @adamshand made their first contribution in https://github.com/sissbruecker/linkding/pull/585 * @jonathan-s made their first contribution in https://github.com/sissbruecker/linkding/pull/601 * @JnsDornbusch made their first contribution in https://github.com/sissbruecker/linkding/pull/603 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.23.1...v1.24.0 --- ## v1.23.1 (08/12/2023) ### What's Changed * Properly encode search query param by @sissbruecker in https://github.com/sissbruecker/linkding/pull/587 > [!WARNING] > *This resolves a security vulnerability in linkding. Everyone is encouraged to upgrade to the latest version as soon as possible.* **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.23.0...v1.23.1 --- ## v1.23.0 (24/11/2023) ### What's Changed * Add Alpine based Docker image (experimental) by @sissbruecker in https://github.com/sissbruecker/linkding/pull/570 * Add backup CLI command by @sissbruecker in https://github.com/sissbruecker/linkding/pull/571 * Update browser extension links by @OPerepadia in https://github.com/sissbruecker/linkding/pull/574 * Include archived bookmarks in export by @sissbruecker in https://github.com/sissbruecker/linkding/pull/579 ### New Contributors * @OPerepadia made their first contribution in https://github.com/sissbruecker/linkding/pull/574 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.22.3...v1.23.0 --- ## v1.22.3 (04/11/2023) ### What's Changed * Fix RSS feed not handling None values by @vitormarcal in https://github.com/sissbruecker/linkding/pull/569 * Bump django from 4.1.10 to 4.1.13 by @dependabot in https://github.com/sissbruecker/linkding/pull/567 ### New Contributors * @vitormarcal made their first contribution in https://github.com/sissbruecker/linkding/pull/569 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.22.2...v1.22.3 --- ## v1.22.2 (27/10/2023) ### What's Changed * Fix search options not opening on iOS by @sissbruecker in https://github.com/sissbruecker/linkding/pull/549 * Bump urllib3 from 1.26.11 to 1.26.17 by @dependabot in https://github.com/sissbruecker/linkding/pull/542 * Add iOS shortcut to community section by @andrewdolphin in https://github.com/sissbruecker/linkding/pull/550 * Disable editing of search preferences in user admin by @sissbruecker in https://github.com/sissbruecker/linkding/pull/555 * Add feed2linkding to community section by @Strubbl in https://github.com/sissbruecker/linkding/pull/544 * Sanitize RSS feed to remove control characters by @sissbruecker in https://github.com/sissbruecker/linkding/pull/565 * Bump urllib3 from 1.26.17 to 1.26.18 by @dependabot in https://github.com/sissbruecker/linkding/pull/560 ### New Contributors * @andrewdolphin made their first contribution in https://github.com/sissbruecker/linkding/pull/550 * @Strubbl made their first contribution in https://github.com/sissbruecker/linkding/pull/544 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.22.1...v1.22.2 --- ## v1.22.1 (06/10/2023) ### What's Changed * Fix memory leak with SQLite by @sissbruecker in https://github.com/sissbruecker/linkding/pull/548 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.22.0...v1.22.1 --- ## v1.22.0 (01/10/2023) ### What's Changed * Fix case-insensitive search for unicode characters in SQLite by @sissbruecker in https://github.com/sissbruecker/linkding/pull/520 * Add sort option to bookmark list by @sissbruecker in https://github.com/sissbruecker/linkding/pull/522 * Add button to show tags on smaller screens by @sissbruecker in https://github.com/sissbruecker/linkding/pull/529 * Make code blocks in notes scrollable by @sissbruecker in https://github.com/sissbruecker/linkding/pull/530 * Add filter for shared state by @sissbruecker in https://github.com/sissbruecker/linkding/pull/531 * Add support for exporting/importing bookmark notes by @sissbruecker in https://github.com/sissbruecker/linkding/pull/532 * Add filter for unread state by @sissbruecker in https://github.com/sissbruecker/linkding/pull/535 * Allow saving search preferences by @sissbruecker in https://github.com/sissbruecker/linkding/pull/540 * Add user profile endpoint by @sissbruecker in https://github.com/sissbruecker/linkding/pull/541 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.21.0...v1.22.0 --- ## v1.21.1 (26/09/2023) ### What's Changed * Fix bulk edit to respect searched tags by @sissbruecker in https://github.com/sissbruecker/linkding/pull/537 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.21.0...v1.21.1 --- ## v1.21.0 (25/08/2023) ### What's Changed * Make search autocomplete respect link target setting by @sissbruecker in https://github.com/sissbruecker/linkding/pull/513 * Various CSS improvements by @sissbruecker in https://github.com/sissbruecker/linkding/pull/514 * Display shared state in bookmark list by @sissbruecker in https://github.com/sissbruecker/linkding/pull/515 * Allow bulk editing unread and shared state of bookmarks by @sissbruecker in https://github.com/sissbruecker/linkding/pull/517 * Bump uwsgi from 2.0.20 to 2.0.22 by @dependabot in https://github.com/sissbruecker/linkding/pull/516 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.20.1...v1.21.0 --- ## v1.20.1 (23/08/2023) ### What's Changed * Update cached styles and scripts after version change by @sissbruecker in https://github.com/sissbruecker/linkding/pull/510 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.20.0...v1.20.1 --- ## v1.20.0 (22/08/2023) ### What's Changed * Add option to share bookmarks publicly by @sissbruecker in https://github.com/sissbruecker/linkding/pull/503 * Various improvements to favicons by @sissbruecker in https://github.com/sissbruecker/linkding/pull/504 * Add support for PRIVATE flag in import and export by @sissbruecker in https://github.com/sissbruecker/linkding/pull/505 * Avoid page reload when triggering actions in bookmark list by @sissbruecker in https://github.com/sissbruecker/linkding/pull/506 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.19.1...v1.20.0 --- ## v1.19.1 (29/07/2023) ### What's Changed * Add Postman Collection to Community section of README by @gingerbeardman in https://github.com/sissbruecker/linkding/pull/476 * Added Dev Container support by @acbgbca in https://github.com/sissbruecker/linkding/pull/474 * Added Apple web-app meta tag #358 by @acbgbca in https://github.com/sissbruecker/linkding/pull/359 * Bump requests from 2.28.1 to 2.31.0 by @dependabot in https://github.com/sissbruecker/linkding/pull/478 * Allow passing title and description to new bookmark form by @acbgbca in https://github.com/sissbruecker/linkding/pull/479 * Enable WAL to avoid locked database lock errors by @sissbruecker in https://github.com/sissbruecker/linkding/pull/480 * Fix website loader content encoding detection by @sissbruecker in https://github.com/sissbruecker/linkding/pull/482 * Bump certifi from 2022.12.7 to 2023.7.22 by @dependabot in https://github.com/sissbruecker/linkding/pull/497 * Bump django from 4.1.9 to 4.1.10 by @dependabot in https://github.com/sissbruecker/linkding/pull/494 ### New Contributors * @gingerbeardman made their first contribution in https://github.com/sissbruecker/linkding/pull/476 * @acbgbca made their first contribution in https://github.com/sissbruecker/linkding/pull/474 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.19.0...v1.19.1 --- ## v1.19.0 (20/05/2023) ### What's Changed * Add notes to bookmarks by @sissbruecker in https://github.com/sissbruecker/linkding/pull/472 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.18.0...v1.19.0 --- ## v1.18.0 (18/05/2023) ### What's Changed * Make search case-insensitive on Postgres by @sissbruecker in https://github.com/sissbruecker/linkding/pull/432 * Allow searching for tags without hash character by @sissbruecker in https://github.com/sissbruecker/linkding/pull/449 * Prevent zoom-in after focusing an input on small viewports on iOS devices by @puresick in https://github.com/sissbruecker/linkding/pull/440 * Add database options by @plockaby in https://github.com/sissbruecker/linkding/pull/406 * Allow to log real client ip in logs when using a reverse proxy by @fmenabe in https://github.com/sissbruecker/linkding/pull/398 * Add option to display URL below title by @bah0 in https://github.com/sissbruecker/linkding/pull/365 * Add LinkThing iOS app to community section by @amoscardino in https://github.com/sissbruecker/linkding/pull/446 * Bump django from 4.1.7 to 4.1.9 by @dependabot in https://github.com/sissbruecker/linkding/pull/466 * Bump sqlparse from 0.4.2 to 0.4.4 by @dependabot in https://github.com/sissbruecker/linkding/pull/455 ### New Contributors * @amoscardino made their first contribution in https://github.com/sissbruecker/linkding/pull/446 * @puresick made their first contribution in https://github.com/sissbruecker/linkding/pull/440 * @plockaby made their first contribution in https://github.com/sissbruecker/linkding/pull/406 * @fmenabe made their first contribution in https://github.com/sissbruecker/linkding/pull/398 * @bah0 made their first contribution in https://github.com/sissbruecker/linkding/pull/365 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.17.2...v1.18.0 --- ## v1.17.2 (18/02/2023) ### What's Changed * Escape texts in exported HTML by @sissbruecker in https://github.com/sissbruecker/linkding/pull/429 * Bump django from 4.1.2 to 4.1.7 by @dependabot in https://github.com/sissbruecker/linkding/pull/427 * Make health check in Dockerfile honor context path setting by @mrex in https://github.com/sissbruecker/linkding/pull/407 * Disable autocapitalization for tag input form by @joshdick in https://github.com/sissbruecker/linkding/pull/395 ### New Contributors * @mrex made their first contribution in https://github.com/sissbruecker/linkding/pull/407 * @joshdick made their first contribution in https://github.com/sissbruecker/linkding/pull/395 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.17.1...v1.17.2 --- ## v1.17.1 (22/01/2023) ### What's Changed * Fix favicon being cleared by web archive snapshot task by @sissbruecker in https://github.com/sissbruecker/linkding/pull/405 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.17.0...v1.17.1 --- ## v1.17.0 (21/01/2023) ### What's Changed * Add Health Check endpoint by @mckennajones in https://github.com/sissbruecker/linkding/pull/392 * Cache website metadata to avoid duplicate scraping by @sissbruecker in https://github.com/sissbruecker/linkding/pull/401 * Prefill form if URL is already bookmarked by @sissbruecker in https://github.com/sissbruecker/linkding/pull/402 * Add option for showing bookmark favicons by @sissbruecker in https://github.com/sissbruecker/linkding/pull/390 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.16.1...v1.17.0 --- ## v1.16.1 (20/01/2023) ### What's Changed * Fix bookmark website metadata not being updated when URL changes by @sissbruecker in https://github.com/sissbruecker/linkding/pull/400 * Bump django from 4.1 to 4.1.2 by @dependabot in https://github.com/sissbruecker/linkding/pull/391 * Bump certifi from 2022.6.15 to 2022.12.7 by @dependabot in https://github.com/sissbruecker/linkding/pull/374 * Bump minimatch from 3.0.4 to 3.1.2 by @dependabot in https://github.com/sissbruecker/linkding/pull/366 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.16.0...v1.16.1 --- ## v1.16.0 (12/01/2023) ### What's Changed * Add postgres as database engine by @tomamplius in https://github.com/sissbruecker/linkding/pull/388 * Gracefully stop docker container when it receives SIGTERM by @mckennajones in https://github.com/sissbruecker/linkding/pull/368 * Limit document size for website scraper by @sissbruecker in https://github.com/sissbruecker/linkding/pull/354 * Add error handling for checking latest version by @sissbruecker in https://github.com/sissbruecker/linkding/pull/360 * Trim website metadata title and description by @luca1197 in https://github.com/sissbruecker/linkding/pull/383 * Only show admin link for superusers by @AlexanderS in https://github.com/sissbruecker/linkding/pull/384 * Add apache reverse proxy documentation. by @jhauris in https://github.com/sissbruecker/linkding/pull/371 * Correct LD_ENABLE_AUTH_PROXY documentation by @jhauris in https://github.com/sissbruecker/linkding/pull/379 * Android HTTP shortcuts v3 by @kzshantonu in https://github.com/sissbruecker/linkding/pull/387 ### New Contributors * @jhauris made their first contribution in https://github.com/sissbruecker/linkding/pull/371 * @AlexanderS made their first contribution in https://github.com/sissbruecker/linkding/pull/384 * @mckennajones made their first contribution in https://github.com/sissbruecker/linkding/pull/368 * @tomamplius made their first contribution in https://github.com/sissbruecker/linkding/pull/388 * @luca1197 made their first contribution in https://github.com/sissbruecker/linkding/pull/383 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.15.1...v1.16.0 --- ## v1.15.1 (05/10/2022) ### What's Changed * Fix static file dir warning by @sissbruecker in https://github.com/sissbruecker/linkding/pull/350 * Add setting and documentation for fixing CSRF errors by @sissbruecker in https://github.com/sissbruecker/linkding/pull/349 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.15.0...v1.15.1 --- ## v1.15.0 (11/09/2022) ### What's Changed * Bump Django and other dependencies by @sissbruecker in https://github.com/sissbruecker/linkding/pull/331 * Add option to create initial superuser by @sissbruecker in https://github.com/sissbruecker/linkding/pull/323 * Improved Android HTTP Shortcuts doc by @kzshantonu in https://github.com/sissbruecker/linkding/pull/330 * Minify bookmark list HTML by @sissbruecker in https://github.com/sissbruecker/linkding/pull/332 * Bump python version to 3.10 by @sissbruecker in https://github.com/sissbruecker/linkding/pull/333 * Fix error when deleting all bookmarks in admin by @sissbruecker in https://github.com/sissbruecker/linkding/pull/336 * Improve bookmark query performance by @sissbruecker in https://github.com/sissbruecker/linkding/pull/334 * Prevent rate limit errors in wayback machine API by @sissbruecker in https://github.com/sissbruecker/linkding/pull/339 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.14.0...v1.15.0 --- ## v1.14.0 (14/08/2022) ### What's Changed * Add support for context path by @s2marine in https://github.com/sissbruecker/linkding/pull/313 * Add support for authentication proxies by @sissbruecker in https://github.com/sissbruecker/linkding/pull/321 * Add bookmark list keyboard navigation by @sissbruecker in https://github.com/sissbruecker/linkding/pull/320 * Skip updating website metadata on edit unless URL has changed by @sissbruecker in https://github.com/sissbruecker/linkding/pull/318 * Add simple docs of the new `shared` API parameter by @bachya in https://github.com/sissbruecker/linkding/pull/312 * Add project linka to community section in README by @cmsax in https://github.com/sissbruecker/linkding/pull/319 * Order tags in test_should_create_new_bookmark by @RoGryza in https://github.com/sissbruecker/linkding/pull/310 * Bump django from 3.2.14 to 3.2.15 by @dependabot in https://github.com/sissbruecker/linkding/pull/316 ### New Contributors * @s2marine made their first contribution in https://github.com/sissbruecker/linkding/pull/313 * @RoGryza made their first contribution in https://github.com/sissbruecker/linkding/pull/310 * @cmsax made their first contribution in https://github.com/sissbruecker/linkding/pull/319 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.13.0...v1.14.0 --- ## v1.13.0 (04/08/2022) ### What's Changed * Add bookmark sharing by @sissbruecker in https://github.com/sissbruecker/linkding/pull/311 * Display selected tags in tag cloud by @sissbruecker and @jhauris in https://github.com/sissbruecker/linkding/pull/307 * Update unread flag when saving duplicate URL by @sissbruecker in https://github.com/sissbruecker/linkding/pull/306 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.12.0...v1.13.0 --- ## v1.12.0 (23/07/2022) ### What's Changed * Add read it later functionality by @sissbruecker in https://github.com/sissbruecker/linkding/pull/304 * Add RSS feeds by @sissbruecker in https://github.com/sissbruecker/linkding/pull/305 * Add bookmarklet to community by @ukcuddlyguy in https://github.com/sissbruecker/linkding/pull/293 * Shorten and simplify example bookmarklet in documentation by @FunctionDJ in https://github.com/sissbruecker/linkding/pull/297 * Fix typo by @kianmeng in https://github.com/sissbruecker/linkding/pull/295 * Bump django from 3.2.13 to 3.2.14 by @dependabot in https://github.com/sissbruecker/linkding/pull/294 * Bump svelte from 3.46.4 to 3.49.0 by @dependabot in https://github.com/sissbruecker/linkding/pull/299 * Bump terser from 5.5.1 to 5.14.2 by @dependabot in https://github.com/sissbruecker/linkding/pull/302 ### New Contributors * @ukcuddlyguy made their first contribution in https://github.com/sissbruecker/linkding/pull/293 * @FunctionDJ made their first contribution in https://github.com/sissbruecker/linkding/pull/297 * @kianmeng made their first contribution in https://github.com/sissbruecker/linkding/pull/295 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.11.1...v1.12.0 --- ## v1.11.1 (03/07/2022) ### What's Changed * Fix duplicate tags on import by @wahlm in https://github.com/sissbruecker/linkding/pull/289 * Add apple-touch-icon by @daveonkels in https://github.com/sissbruecker/linkding/pull/282 * Bump waybackpy to 3.0.6 by @dustinblackman in https://github.com/sissbruecker/linkding/pull/281 ### New Contributors * @wahlm made their first contribution in https://github.com/sissbruecker/linkding/pull/289 * @daveonkels made their first contribution in https://github.com/sissbruecker/linkding/pull/282 * @dustinblackman made their first contribution in https://github.com/sissbruecker/linkding/pull/281 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.11.0...v1.11.1 --- ## v1.11.0 (26/05/2022) ### What's Changed * Add background tasks to admin by @sissbruecker in https://github.com/sissbruecker/linkding/pull/264 * Improve about section by @sissbruecker in https://github.com/sissbruecker/linkding/pull/265 * Allow creating archived bookmark through REST API by @kencx in https://github.com/sissbruecker/linkding/pull/268 * Add PATCH support to bookmarks endpoint by @sissbruecker in https://github.com/sissbruecker/linkding/pull/269 * Add community reference to linkding-cli by @bachya in https://github.com/sissbruecker/linkding/pull/270 ### New Contributors * @kencx made their first contribution in https://github.com/sissbruecker/linkding/pull/268 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.10.1...v1.11.0 --- ## v1.10.1 (21/05/2022) ### What's Changed * Fake request headers to reduce bot detection by @sissbruecker in https://github.com/sissbruecker/linkding/pull/263 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.10.0...v1.10.1 --- ## v1.10.0 (21/05/2022) ### What's Changed * Add to managed hosting options by @m3nu in https://github.com/sissbruecker/linkding/pull/253 * Add community reference to aiolinkding by @bachya in https://github.com/sissbruecker/linkding/pull/259 * Improve import performance by @sissbruecker in https://github.com/sissbruecker/linkding/pull/261 * Update how-to.md to fix unclear/paraphrased Safari action in IOS Shortcuts by @feoh in https://github.com/sissbruecker/linkding/pull/260 * Allow searching for untagged bookmarks by @sissbruecker in https://github.com/sissbruecker/linkding/pull/226 ### New Contributors * @m3nu made their first contribution in https://github.com/sissbruecker/linkding/pull/253 * @bachya made their first contribution in https://github.com/sissbruecker/linkding/pull/259 * @feoh made their first contribution in https://github.com/sissbruecker/linkding/pull/260 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.9.0...v1.10.0 --- ## v1.9.0 (14/05/2022) ### What's Changed * Scroll menu items into view when using keyboard by @sissbruecker in https://github.com/sissbruecker/linkding/pull/248 * Add whitespace after auto-completed tag by @sissbruecker in https://github.com/sissbruecker/linkding/pull/249 * Bump django from 3.2.12 to 3.2.13 by @dependabot in https://github.com/sissbruecker/linkding/pull/244 * Add community helm chart reference to readme by @pascaliske in https://github.com/sissbruecker/linkding/pull/242 * Feature: Shortcut key for new bookmark by @rithask in https://github.com/sissbruecker/linkding/pull/241 * Clarify archive.org feature by @clach04 in https://github.com/sissbruecker/linkding/pull/229 * Make Internet Archive integration opt-in by @sissbruecker in https://github.com/sissbruecker/linkding/pull/250 ### New Contributors * @pascaliske made their first contribution in https://github.com/sissbruecker/linkding/pull/242 * @rithask made their first contribution in https://github.com/sissbruecker/linkding/pull/241 * @clach04 made their first contribution in https://github.com/sissbruecker/linkding/pull/229 **Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.8.8...v1.9.0 --- ## v1.8.8 (27/03/2022) - [**bug**] Prevent bookmark actions through get requests - [**bug**] Prevent external redirects --- ## v1.8.7 (26/03/2022) - [**bug**] Increase request buffer size [#28](https://github.com/sissbruecker/linkding/issues/28) - [**enhancement**] Allow specifying port through LINKDING_PORT environment variable [#156](https://github.com/sissbruecker/linkding/pull/156) - [**chore**] Bump NPM packages [#224](https://github.com/sissbruecker/linkding/pull/224) --- ## v1.8.6 (25/03/2022) - [bug] fix bookmark access restrictions - [bug] prevent external redirects - [chore] bump dependencies --- ## v1.8.5 (12/12/2021) - [**bug**] Ensure tag names do not contain spaces [#182](https://github.com/sissbruecker/linkding/issues/182) - [**bug**] Consider not copying whole GIT repository to Docker image [#174](https://github.com/sissbruecker/linkding/issues/174) - [**enhancement**] Make bookmarks count column in admin sortable [#183](https://github.com/sissbruecker/linkding/pull/183) --- ## v1.8.4 (16/10/2021) - [**enhancement**] Allow non-admin users to change their password [#166](https://github.com/sissbruecker/linkding/issues/166) --- ## v1.8.3 (03/10/2021) - [**enhancement**] Enhancement: let user configure to open links in same tab instead on a new window/tab [#27](https://github.com/sissbruecker/linkding/issues/27) --- ## v1.8.2 (02/10/2021) - [**bug**] Fix jumping search box [#163](https://github.com/sissbruecker/linkding/pull/163) --- ## v1.8.1 (01/10/2021) - [**enhancement**] Add global shortcut for search [#161](https://github.com/sissbruecker/linkding/pull/161) - allows to press `s` to focus the search input --- ## v1.8.0 (04/09/2021) - [**enhancement**] Wayback Machine Integration [#59](https://github.com/sissbruecker/linkding/issues/59) - Automatically creates snapshots of bookmarked websites on [web archive](https://archive.org/web/) - This is one of the largest changes yet and adds a task processor that runs as a separate process in the background. If you run into issues with this feature, it can be disabled using the [LD_DISABLE_BACKGROUND_TASKS](https://github.com/sissbruecker/linkding/blob/master/docs/Options.md#ld_disable_background_tasks) option --- ## v1.7.2 (26/08/2021) - [**enhancement**] Add support for nanosecond resolution timestamps for bookmark import (e.g. Google Bookmarks) [#146](https://github.com/sissbruecker/linkding/issues/146) --- ## v1.7.1 (25/08/2021) - [**bug**] umlaut/non-ascii characters broken when using bookmarklet (firefox) [#148](https://github.com/sissbruecker/linkding/issues/148) - [**bug**] Bookmark import accepts empty URL values [#124](https://github.com/sissbruecker/linkding/issues/124) - [**enhancement**] Show the version in the settings [#104](https://github.com/sissbruecker/linkding/issues/104) --- ## v1.7.0 (17/08/2021) - Upgrade to Django 3 - Bump other dependencies --- ## v1.6.5 (15/08/2021) - [**enhancement**] query with multiple hashtags very slow [#112](https://github.com/sissbruecker/linkding/issues/112) --- ## v1.6.4 (13/05/2021) - Update dependencies for security fixes --- ## v1.6.3 (06/04/2021) - [**bug**] relative names use the wrong "today" after day change [#107](https://github.com/sissbruecker/linkding/issues/107) --- ## v1.6.2 (04/04/2021) - [**enhancement**] Expose `date_added` in UI [#85](https://github.com/sissbruecker/linkding/issues/85) - [**closed**] Archived bookmarks - no result when searching for a word which is used only as tag [#83](https://github.com/sissbruecker/linkding/issues/83) - [**closed**] Add archive/unarchive button to edit bookmark page [#82](https://github.com/sissbruecker/linkding/issues/82) - [**enhancement**] Make scraped title and description editable [#80](https://github.com/sissbruecker/linkding/issues/80) --- ## v1.6.1 (31/03/2021) - Expose date_added in UI [#85](https://github.com/sissbruecker/linkding/issues/85) --- ## v1.6.0 (28/03/2021) - Bulk edit mode [#101](https://github.com/sissbruecker/linkding/pull/101) --- ## v1.5.0 (28/03/2021) - [**closed**] Add a dark mode [#49](https://github.com/sissbruecker/linkding/issues/49) --- ## v1.4.1 (20/03/2021) - Security patches - Documentation improvements --- ## v1.4.0 (24/02/2021) - [**enhancement**] Improve admin utilization [#76](https://github.com/sissbruecker/linkding/issues/76) --- ## v1.3.3 (18/02/2021) - [**closed**] Missing "description" request body parameter in API causes 500 [#78](https://github.com/sissbruecker/linkding/issues/78) --- ## v1.3.2 (18/02/2021) - [**closed**] /archive and /unarchive API routes return 404 [#77](https://github.com/sissbruecker/linkding/issues/77) - [**closed**] API - /api/check_url?url= with token authetification [#55](https://github.com/sissbruecker/linkding/issues/55) --- ## v1.3.1 (15/02/2021) [enhancement] Enhance delete links with inline confirmation --- ## v1.3.0 (14/02/2021) - [**closed**] Novice help. [#71](https://github.com/sissbruecker/linkding/issues/71) - [**closed**] Option to create bookmarks public [#70](https://github.com/sissbruecker/linkding/issues/70) - [**enhancement**] Show URL if title is not available [#64](https://github.com/sissbruecker/linkding/issues/64) - [**bug**] minor ui nitpicks [#62](https://github.com/sissbruecker/linkding/issues/62) - [**enhancement**] add an archive function [#46](https://github.com/sissbruecker/linkding/issues/46) - [**closed**] remove non fqdn check and alert [#36](https://github.com/sissbruecker/linkding/issues/36) - [**closed**] Add Lotus Notes links [#22](https://github.com/sissbruecker/linkding/issues/22) --- ## v1.2.1 (12/01/2021) - [**bug**] Bug: Two equal tags with different capitalisation lead to 500 server errors [#65](https://github.com/sissbruecker/linkding/issues/65) - [**closed**] Enhancement: category and pagination [#11](https://github.com/sissbruecker/linkding/issues/11) --- ## v1.2.0 (09/01/2021) - [**closed**] Add Favicon [#58](https://github.com/sissbruecker/linkding/issues/58) - [**closed**] Make tags case-insensitive [#45](https://github.com/sissbruecker/linkding/issues/45) --- ## v1.1.1 (01/01/2021) - [**enhancement**] Add docker-compose support [#54](https://github.com/sissbruecker/linkding/pull/54) --- ## v1.1.0 (31/12/2020) - [**enhancement**] Search autocomplete [#52](https://github.com/sissbruecker/linkding/issues/52) - [**enhancement**] Improve Netscape bookmarks file parsing [#50](https://github.com/sissbruecker/linkding/issues/50) --- ## v1.0.0 (31/12/2020) - [**bug**] Import does not import bookmark descriptions [#47](https://github.com/sissbruecker/linkding/issues/47) - [**enhancement**] Enhancement: return to same page we were on after editing a bookmark [#26](https://github.com/sissbruecker/linkding/issues/26) - [**bug**] Increase limit on bookmark URL length [#25](https://github.com/sissbruecker/linkding/issues/25) - [**enhancement**] API for app development [#24](https://github.com/sissbruecker/linkding/issues/24) - [**enhancement**] Enhancement: detect duplicates at entry time [#23](https://github.com/sissbruecker/linkding/issues/23) - [**bug**] Error importing bookmarks [#18](https://github.com/sissbruecker/linkding/issues/18) - [**enhancement**] Enhancement: better administration page [#4](https://github.com/sissbruecker/linkding/issues/4) - [**enhancement**] Bug: Navigation bar active link stays on add bookmark [#3](https://github.com/sissbruecker/linkding/issues/3) - [**bug**] CSS Stylesheet presented as text/plain [#2](https://github.com/sissbruecker/linkding/issues/2) ================================================ FILE: LICENSE.txt ================================================ The MIT License (MIT) Copyright (c) 2019 Sascha Ißbrücker Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ .PHONY: serve init: uv sync [ -d data ] || mkdir data data/assets data/favicons data/previews uv run manage.py migrate npm install serve: uv run manage.py runserver tasks: uv run manage.py run_huey test: uv run pytest -n auto lint: uv run ruff check bookmarks format: uv run ruff format bookmarks uv run djlint bookmarks/templates --reformat --quiet --warn npx prettier bookmarks/frontend --write npx prettier bookmarks/styles --write prepare-e2e: uv run playwright install chromium rm -rf static npm run build uv run manage.py collectstatic --no-input e2e: make prepare-e2e uv run pytest bookmarks/tests_e2e -n auto -o "python_files=e2e_test_*.py" frontend: npm run dev ================================================ FILE: README.md ================================================
## Introduction linkding is a bookmark manager that you can host yourself. It's designed be to be minimal, fast, and easy to set up using Docker. The name comes from: - *link* which is often used as a synonym for URLs and bookmarks in common language - *Ding* which is German for thing - ...so basically something for managing your links **Feature Overview:** - Clean UI optimized for readability - Organize bookmarks with tags - Bulk editing, Markdown notes, read it later functionality - Share bookmarks with other users or guests - Automatically provides titles, descriptions and icons of bookmarked websites - Automatically archive websites, either as local HTML file or on Internet Archive - Import and export bookmarks in Netscape HTML format - Installable as a Progressive Web App (PWA) - Extensions for [Firefox](https://addons.mozilla.org/firefox/addon/linkding-extension/) and [Chrome](https://chrome.google.com/webstore/detail/linkding-extension/beakmhbijpdhipnjhnclmhgjlddhidpe), as well as a bookmarklet - SSO support via OIDC or authentication proxies - REST API for developing 3rd party apps - Admin panel for user self-service and raw data access **Demo:** https://demo.linkding.link/ **Screenshot:**  ## Getting Started The following links help you to get started with linkding: - [Install linkding on your own server](https://linkding.link/installation) or [check managed hosting options](https://linkding.link/managed-hosting) - [Install the browser extension](https://linkding.link/browser-extension) - [Check out community projects](https://linkding.link/community), which include mobile apps, browser extensions, libraries and more ## Documentation The full documentation is now available at [linkding.link](https://linkding.link/). If you want to contribute to the documentation, you can find the source files in the `docs` folder. If you want to contribute a community project, feel free to [submit a PR](https://github.com/sissbruecker/linkding/edit/master/docs/src/content/docs/community.md). ## Contributing Small improvements, bugfixes and documentation improvements are always welcome. If you want to contribute a larger feature, consider opening an issue first to discuss it. I may choose to ignore PRs for features that don't align with the project's goals or that I don't want to maintain. ## Development The application is built using the Django web framework. You can get started by checking out the excellent [Django docs](https://docs.djangoproject.com/en/4.1/). The `bookmarks` folder contains the actual bookmark application. Other than that the code should be self-explanatory / standard Django stuff 🙂. ### Prerequisites - Python 3.13 - [uv](https://docs.astral.sh/uv/getting-started/installation/) - Node.js ### Setup Initialize the development environment with: ``` make init ``` This sets up a virtual environment using uv, installs NPM dependencies and runs migrations to create the initial database. Create a user for the frontend: ``` uv run manage.py createsuperuser --username=joe --email=joe@example.com ``` Run the frontend build for bundling frontend components with: ``` make frontend ``` Then start the Django development server with: ``` make serve ``` The frontend is now available under http://localhost:8000 ### Tests Run all tests with pytest: ``` make test ``` ### Linting Run linting with ruff: ``` make lint ``` ### Formatting Format Python code with ruff, Django templates with djlint, and JavaScript code with prettier: ``` make format ``` ### DevContainers This repository also supports DevContainers: [](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/sissbruecker/linkding.git) Once checked out, only the following commands are required to get started: Create a user for the frontend: ``` uv run manage.py createsuperuser --username=joe --email=joe@example.com ``` Start the Node.js development server (used for compiling JavaScript components like tag auto-completion) with: ``` make frontend ``` Start the Django development server with: ``` make serve ``` The frontend is now available under http://localhost:8000 ================================================ FILE: SECURITY.md ================================================ # Security Policy ## Supported Versions | Version | Supported | | ------- | ------------------ | | 1.10.x | :white_check_mark: | ## Reporting a Vulnerability To report a vulnerability, please send a mail to: 588ex5zl8@mozmail.com I'll try to get back to you as soon as possible. ================================================ FILE: bookmarks/__init__.py ================================================ ================================================ FILE: bookmarks/admin.py ================================================ import os from django import forms from django.contrib import admin, messages from django.contrib.admin import AdminSite from django.contrib.auth.admin import UserAdmin from django.contrib.auth.models import User from django.core.paginator import Paginator from django.db.models import Count, QuerySet from django.shortcuts import render from django.urls import path from django.utils.translation import gettext, ngettext from huey.contrib.djhuey import HUEY as huey from bookmarks.models import ( ApiToken, Bookmark, BookmarkAsset, BookmarkBundle, FeedToken, Tag, Toast, UserProfile, ) from bookmarks.services.bookmarks import archive_bookmark, unarchive_bookmark # Custom paginator to paginate through Huey tasks class TaskPaginator(Paginator): def __init__(self): super().__init__(self, 100) self.task_count = huey.storage.queue_size() @property def count(self): return self.task_count def page(self, number): limit = self.per_page offset = (number - 1) * self.per_page return self._get_page( self.enqueued_items(limit, offset), number, self, ) # Copied from Huey's SqliteStorage with some modifications to allow pagination def enqueued_items(self, limit, offset): def to_bytes(b): return bytes(b) if not isinstance(b, bytes) else b sql = "select data from task where queue=? order by priority desc, id limit ? offset ?" params = (huey.storage.name, limit, offset) serialized_tasks = [ to_bytes(i) for (i,) in huey.storage.sql(sql, params, results=True) ] return [huey.deserialize_task(task) for task in serialized_tasks] # Custom view to display Huey tasks in the admin def background_task_view(request): page_number = int(request.GET.get("p", 1)) paginator = TaskPaginator() page = paginator.get_page(page_number) page_range = paginator.get_elided_page_range(page_number, on_each_side=2, on_ends=2) context = { **linkding_admin_site.each_context(request), "title": "Background tasks", "page": page, "page_range": page_range, "tasks": page.object_list, } return render(request, "admin/background_tasks.html", context) class LinkdingAdminSite(AdminSite): site_header = "linkding administration" site_title = "linkding Admin" def get_urls(self): urls = super().get_urls() custom_urls = [ path("tasks/", background_task_view, name="background_tasks"), ] return custom_urls + urls def get_app_list(self, request, app_label=None): app_list = super().get_app_list(request, app_label) context_path = os.getenv("LD_CONTEXT_PATH", "") app_list += [ { "name": "Huey", "app_label": "huey_app", "models": [ { "name": "Queued tasks", "object_name": "background_tasks", "admin_url": f"/{context_path}admin/tasks/", "view_only": True, } ], } ] return app_list class AdminBookmark(admin.ModelAdmin): list_display = ("resolved_title", "url", "is_archived", "owner", "date_added") search_fields = ( "title", "description", "website_title", "website_description", "url", "tags__name", ) list_filter = ( "owner__username", "is_archived", "unread", "tags", ) ordering = ("-date_added",) actions = [ "delete_selected_bookmarks", "archive_selected_bookmarks", "unarchive_selected_bookmarks", "mark_as_read", "mark_as_unread", ] def get_actions(self, request): actions = super().get_actions(request) # Remove default delete action, which gets replaced by delete_selected_bookmarks below # The default action shows a confirmation page which can fail in production when selecting all bookmarks and the # number of objects to delete exceeds the value in DATA_UPLOAD_MAX_NUMBER_FIELDS (1000 by default) del actions["delete_selected"] return actions def delete_selected_bookmarks(self, request, queryset: QuerySet): bookmarks_count = queryset.count() for bookmark in queryset: bookmark.delete() self.message_user( request, ngettext( "%d bookmark was successfully deleted.", "%d bookmarks were successfully deleted.", bookmarks_count, ) % bookmarks_count, messages.SUCCESS, ) def archive_selected_bookmarks(self, request, queryset: QuerySet): for bookmark in queryset: archive_bookmark(bookmark) bookmarks_count = queryset.count() self.message_user( request, ngettext( "%d bookmark was successfully archived.", "%d bookmarks were successfully archived.", bookmarks_count, ) % bookmarks_count, messages.SUCCESS, ) def unarchive_selected_bookmarks(self, request, queryset: QuerySet): for bookmark in queryset: unarchive_bookmark(bookmark) bookmarks_count = queryset.count() self.message_user( request, ngettext( "%d bookmark was successfully unarchived.", "%d bookmarks were successfully unarchived.", bookmarks_count, ) % bookmarks_count, messages.SUCCESS, ) def mark_as_read(self, request, queryset: QuerySet): bookmarks_count = queryset.count() queryset.update(unread=False) self.message_user( request, ngettext( "%d bookmark marked as read.", "%d bookmarks marked as read.", bookmarks_count, ) % bookmarks_count, messages.SUCCESS, ) def mark_as_unread(self, request, queryset: QuerySet): bookmarks_count = queryset.count() queryset.update(unread=True) self.message_user( request, ngettext( "%d bookmark marked as unread.", "%d bookmarks marked as unread.", bookmarks_count, ) % bookmarks_count, messages.SUCCESS, ) class AdminBookmarkAsset(admin.ModelAdmin): @admin.display(description="Display Name") def custom_display_name(self, obj): return str(obj) list_display = ("custom_display_name", "date_created", "status") search_fields = ( "display_name", "file", ) list_filter = ("status",) class AdminTag(admin.ModelAdmin): list_display = ("name", "bookmarks_count", "owner", "date_added") search_fields = ("name", "owner__username") list_filter = ("owner__username",) ordering = ("-date_added",) actions = ["delete_unused_tags"] def get_queryset(self, request): queryset = super().get_queryset(request) queryset = queryset.annotate(bookmarks_count=Count("bookmark")) return queryset def bookmarks_count(self, obj): return obj.bookmarks_count bookmarks_count.admin_order_field = "bookmarks_count" def delete_unused_tags(self, request, queryset: QuerySet): unused_tags = queryset.filter(bookmark__isnull=True) unused_tags_count = unused_tags.count() for tag in unused_tags: tag.delete() if unused_tags_count > 0: self.message_user( request, ngettext( "%d unused tag was successfully deleted.", "%d unused tags were successfully deleted.", unused_tags_count, ) % unused_tags_count, messages.SUCCESS, ) else: self.message_user( request, gettext( "There were no unused tags in the selection", ), messages.SUCCESS, ) class AdminBookmarkBundle(admin.ModelAdmin): list_display = ( "name", "owner", "order", "search", "any_tags", "all_tags", "excluded_tags", "filter_shared", "filter_unread", "date_created", ) search_fields = ["name", "search", "any_tags", "all_tags", "excluded_tags"] list_filter = ("owner__username",) class AdminUserProfileInline(admin.StackedInline): model = UserProfile can_delete = False verbose_name_plural = "Profile" fk_name = "user" readonly_fields = ("search_preferences",) class AdminCustomUser(UserAdmin): inlines = (AdminUserProfileInline,) def get_inline_instances(self, request, obj=None): if not obj: return list() return super().get_inline_instances(request, obj) class AdminToast(admin.ModelAdmin): list_display = ("key", "message", "owner", "acknowledged") search_fields = ("key", "message") list_filter = ("owner__username",) class AdminFeedToken(admin.ModelAdmin): list_display = ("key", "user") search_fields = ["key"] list_filter = ("user__username",) class ApiTokenAdminForm(forms.ModelForm): class Meta: model = ApiToken fields = ("name", "user") class AdminApiToken(admin.ModelAdmin): form = ApiTokenAdminForm list_display = ("name", "user", "created") search_fields = ["name", "user__username"] list_filter = ("user__username",) ordering = ("-created",) linkding_admin_site = LinkdingAdminSite() linkding_admin_site.register(Bookmark, AdminBookmark) linkding_admin_site.register(BookmarkAsset, AdminBookmarkAsset) linkding_admin_site.register(Tag, AdminTag) linkding_admin_site.register(BookmarkBundle, AdminBookmarkBundle) linkding_admin_site.register(User, AdminCustomUser) linkding_admin_site.register(ApiToken, AdminApiToken) linkding_admin_site.register(Toast, AdminToast) linkding_admin_site.register(FeedToken, AdminFeedToken) ================================================ FILE: bookmarks/api/__init__.py ================================================ ================================================ FILE: bookmarks/api/auth.py ================================================ from django.utils.translation import gettext_lazy as _ from rest_framework import exceptions from rest_framework.authentication import TokenAuthentication, get_authorization_header from bookmarks.models import ApiToken class LinkdingTokenAuthentication(TokenAuthentication): """ Extends DRF TokenAuthentication to add support for multiple keywords and multiple tokens per user. """ model = ApiToken keywords = [keyword.lower().encode() for keyword in ["Token", "Bearer"]] def authenticate(self, request): auth = get_authorization_header(request).split() if not auth or auth[0].lower() not in self.keywords: return None if len(auth) == 1: msg = _("Invalid token header. No credentials provided.") raise exceptions.AuthenticationFailed(msg) elif len(auth) > 2: msg = _("Invalid token header. Token string should not contain spaces.") raise exceptions.AuthenticationFailed(msg) try: token = auth[1].decode() except UnicodeError: msg = _( "Invalid token header. Token string should not contain invalid characters." ) raise exceptions.AuthenticationFailed(msg) from None return self.authenticate_credentials(token) ================================================ FILE: bookmarks/api/routes.py ================================================ import gzip import logging import os from django.conf import settings from django.http import Http404, StreamingHttpResponse from rest_framework import mixins, status, viewsets from rest_framework.decorators import action from rest_framework.permissions import AllowAny from rest_framework.response import Response from rest_framework.routers import DefaultRouter, SimpleRouter from bookmarks import queries from bookmarks.api.serializers import ( BookmarkAssetSerializer, BookmarkBundleSerializer, BookmarkSerializer, TagSerializer, UserProfileSerializer, ) from bookmarks.models import ( Bookmark, BookmarkAsset, BookmarkBundle, BookmarkSearch, Tag, User, ) from bookmarks.services import assets, auto_tagging, bookmarks, bundles, website_loader from bookmarks.type_defs import HttpRequest from bookmarks.views import access logger = logging.getLogger(__name__) class BookmarkViewSet( viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.UpdateModelMixin, mixins.DestroyModelMixin, ): request: HttpRequest serializer_class = BookmarkSerializer def get_permissions(self): # Allow unauthenticated access to shared bookmarks. # The shared action should still filter bookmarks so that # unauthenticated users only see bookmarks from users that have public # sharing explicitly enabled if self.action == "shared": return [AllowAny()] # Otherwise use default permissions which should require authentication return super().get_permissions() def get_queryset(self): # Provide filtered queryset for list actions user = self.request.user search = BookmarkSearch.from_request(self.request, self.request.GET) if self.action == "list": return queries.query_bookmarks(user, user.profile, search) elif self.action == "archived": return queries.query_archived_bookmarks(user, user.profile, search) elif self.action == "shared": user = User.objects.filter(username=search.user).first() public_only = not self.request.user.is_authenticated return queries.query_shared_bookmarks( user, self.request.user_profile, search, public_only ) # For single entity actions return user owned bookmarks return Bookmark.objects.all().filter(owner=user) def get_serializer_context(self): disable_scraping = "disable_scraping" in self.request.GET disable_html_snapshot = "disable_html_snapshot" in self.request.GET return { "request": self.request, "user": self.request.user, "disable_scraping": disable_scraping, "disable_html_snapshot": disable_html_snapshot, } @action(methods=["get"], detail=False) def archived(self, request: HttpRequest): return self.list(request) @action(methods=["get"], detail=False) def shared(self, request: HttpRequest): return self.list(request) @action(methods=["post"], detail=True) def archive(self, request: HttpRequest, pk): bookmark = self.get_object() bookmarks.archive_bookmark(bookmark) return Response(status=status.HTTP_204_NO_CONTENT) @action(methods=["post"], detail=True) def unarchive(self, request: HttpRequest, pk): bookmark = self.get_object() bookmarks.unarchive_bookmark(bookmark) return Response(status=status.HTTP_204_NO_CONTENT) @action(methods=["get"], detail=False) def check(self, request: HttpRequest): url = request.GET.get("url") ignore_cache = request.GET.get("ignore_cache", False) in ["true"] bookmark = Bookmark.query_existing(request.user, url).first() existing_bookmark_data = ( self.get_serializer(bookmark).data if bookmark else None ) metadata = website_loader.load_website_metadata(url, ignore_cache=ignore_cache) # Return tags that would be automatically applied to the bookmark profile = request.user.profile auto_tags = [] if profile.auto_tagging_rules: try: auto_tags = auto_tagging.get_tags(profile.auto_tagging_rules, url) except Exception as e: logger.error( f"Failed to auto-tag bookmark. url={url}", exc_info=e, ) return Response( { "bookmark": existing_bookmark_data, "metadata": metadata.to_dict(), "auto_tags": auto_tags, }, status=status.HTTP_200_OK, ) @action(methods=["post"], detail=False) def singlefile(self, request: HttpRequest): if settings.LD_DISABLE_ASSET_UPLOAD: return Response( {"error": "Asset upload is disabled."}, status=status.HTTP_403_FORBIDDEN, ) url = request.POST.get("url") file = request.FILES.get("file") if not url or not file: return Response( {"error": "Both 'url' and 'file' parameters are required."}, status=status.HTTP_400_BAD_REQUEST, ) bookmark = Bookmark.query_existing(request.user, url).first() if not bookmark: bookmark = Bookmark(url=url) bookmark = bookmarks.create_bookmark( bookmark, "", request.user, disable_html_snapshot=True ) bookmarks.enhance_with_website_metadata(bookmark) assets.upload_snapshot(bookmark, file.read()) return Response( {"message": "Snapshot uploaded successfully."}, status=status.HTTP_201_CREATED, ) class BookmarkAssetViewSet( viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, mixins.DestroyModelMixin, ): request: HttpRequest serializer_class = BookmarkAssetSerializer def get_queryset(self): user = self.request.user # limit access to assets to the owner of the bookmark for now bookmark = access.bookmark_write(self.request, self.kwargs["bookmark_id"]) return BookmarkAsset.objects.filter( bookmark_id=bookmark.id, bookmark__owner=user ) def get_serializer_context(self): return {"user": self.request.user} @action(detail=True, methods=["get"], url_path="download") def download(self, request: HttpRequest, bookmark_id, pk): asset = self.get_object() try: file_path = os.path.join(settings.LD_ASSET_FOLDER, asset.file) content_type = asset.content_type file_stream = ( gzip.GzipFile(file_path, mode="rb") if asset.gzip else open(file_path, "rb") # noqa: SIM115 ) response = StreamingHttpResponse(file_stream, content_type=content_type) response["Content-Disposition"] = ( f'attachment; filename="{asset.download_name}"' ) return response except FileNotFoundError: raise Http404("Asset file does not exist") from None except Exception as e: logger.error( f"Failed to download asset. bookmark_id={bookmark_id}, asset_id={pk}", exc_info=e, ) return Response(status=status.HTTP_500_INTERNAL_SERVER_ERROR) @action(methods=["post"], detail=False) def upload(self, request: HttpRequest, bookmark_id): if settings.LD_DISABLE_ASSET_UPLOAD: return Response( {"error": "Asset upload is disabled."}, status=status.HTTP_403_FORBIDDEN, ) bookmark = access.bookmark_write(request, bookmark_id) upload_file = request.FILES.get("file") if not upload_file: return Response( {"error": "No file provided."}, status=status.HTTP_400_BAD_REQUEST ) try: asset = assets.upload_asset(bookmark, upload_file) serializer = self.get_serializer(asset) return Response(serializer.data, status=status.HTTP_201_CREATED) except Exception as e: logger.error( f"Failed to upload asset file. bookmark_id={bookmark_id}, file={upload_file.name}", exc_info=e, ) return Response( {"error": "Failed to upload asset."}, status=status.HTTP_500_INTERNAL_SERVER_ERROR, ) def perform_destroy(self, instance): assets.remove_asset(instance) class TagViewSet( viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, mixins.CreateModelMixin, ): request: HttpRequest serializer_class = TagSerializer def get_queryset(self): user = self.request.user return Tag.objects.all().filter(owner=user) def get_serializer_context(self): return {"user": self.request.user} class UserViewSet(viewsets.GenericViewSet): @action(methods=["get"], detail=False) def profile(self, request: HttpRequest): return Response(UserProfileSerializer(request.user.profile).data) class BookmarkBundleViewSet( viewsets.GenericViewSet, mixins.ListModelMixin, mixins.RetrieveModelMixin, mixins.CreateModelMixin, mixins.UpdateModelMixin, mixins.DestroyModelMixin, ): request: HttpRequest serializer_class = BookmarkBundleSerializer def get_queryset(self): user = self.request.user return BookmarkBundle.objects.filter(owner=user).order_by("order") def get_serializer_context(self): return {"user": self.request.user} def perform_destroy(self, instance): bundles.delete_bundle(instance) # DRF routers do not support nested view sets such as /bookmarks/") def append_bookmark(doc: BookmarkDocument, bookmark: Bookmark): url = bookmark.url title = html.escape(bookmark.resolved_title or "") desc = html.escape(bookmark.resolved_description or "") if bookmark.notes: desc += f"[linkding-notes]{html.escape(bookmark.notes)}[/linkding-notes]" tag_names = bookmark.tag_names if bookmark.is_archived: tag_names.append("linkding:bookmarks.archived") tags = ",".join(tag_names) toread = "1" if bookmark.unread else "0" private = "0" if bookmark.shared else "1" added = int(bookmark.date_added.timestamp()) modified = int(bookmark.date_modified.timestamp()) doc.append( f'
")
================================================
FILE: bookmarks/services/favicon_loader.py
================================================
import logging
import mimetypes
import os.path
import re
import time
from pathlib import Path
from urllib.parse import urlparse
import requests
from django.conf import settings
max_file_age = 60 * 60 * 24 # 1 day
logger = logging.getLogger(__name__)
# register mime type for .ico files, which is not included in the default
# mimetypes of the Docker image
mimetypes.add_type("image/x-icon", ".ico")
def _ensure_favicon_folder():
Path(settings.LD_FAVICON_FOLDER).mkdir(parents=True, exist_ok=True)
def _url_to_filename(url: str) -> str:
return re.sub(r"\W+", "_", url)
def _get_url_parameters(url: str) -> dict:
parsed_uri = urlparse(url)
return {
# https://example.com/foo?bar -> https://example.com
"url": f"{parsed_uri.scheme}://{parsed_uri.hostname}",
# https://example.com/foo?bar -> example.com
"domain": parsed_uri.hostname,
}
def _get_favicon_path(favicon_file: str) -> Path:
return Path(os.path.join(settings.LD_FAVICON_FOLDER, favicon_file))
def _check_existing_favicon(favicon_name: str):
# return existing file if a file with the same name, ignoring extension,
# exists and is not stale
for filename in os.listdir(settings.LD_FAVICON_FOLDER):
file_base_name, _ = os.path.splitext(filename)
if file_base_name == favicon_name:
favicon_path = _get_favicon_path(filename)
return filename if not _is_stale(favicon_path) else None
return None
def _is_stale(path: Path) -> bool:
stat = path.stat()
file_age = time.time() - stat.st_mtime
return file_age >= max_file_age
def load_favicon(url: str) -> str:
url_parameters = _get_url_parameters(url)
# Create favicon folder if not exists
_ensure_favicon_folder()
# Use scheme+hostname as favicon filename to reuse icon for all pages on the same domain
favicon_name = _url_to_filename(url_parameters["url"])
favicon_file = _check_existing_favicon(favicon_name)
if not favicon_file:
# Load favicon from provider, save to file
favicon_url = settings.LD_FAVICON_PROVIDER.format(**url_parameters)
logger.debug(f"Loading favicon from: {favicon_url}")
with requests.get(favicon_url, stream=True) as response:
content_type = response.headers["Content-Type"]
file_extension = mimetypes.guess_extension(content_type)
favicon_file = f"{favicon_name}{file_extension}"
favicon_path = _get_favicon_path(favicon_file)
with open(favicon_path, "wb") as file:
for chunk in response.iter_content(chunk_size=8192):
file.write(chunk)
logger.debug(f"Saved favicon as: {favicon_path}")
return favicon_file
================================================
FILE: bookmarks/services/importer.py
================================================
import logging
from dataclasses import dataclass
from django.contrib.auth.models import User
from django.utils import timezone
from bookmarks.models import Bookmark, Tag
from bookmarks.services import tasks
from bookmarks.services.parser import NetscapeBookmark, parse
from bookmarks.utils import normalize_url, parse_timestamp
logger = logging.getLogger(__name__)
@dataclass
class ImportResult:
total: int = 0
success: int = 0
failed: int = 0
@dataclass
class ImportOptions:
map_private_flag: bool = False
class TagCache:
def __init__(self, user: User):
self.user = user
self.cache = dict()
# Init cache with all existing tags for that user
tags = Tag.objects.filter(owner=user)
for tag in tags:
self.put(tag)
def get(self, tag_name: str):
tag_name_lowercase = tag_name.lower()
if tag_name_lowercase in self.cache:
return self.cache[tag_name_lowercase]
else:
return None
def get_all(self, tag_names: list[str]):
result = []
for tag_name in tag_names:
tag = self.get(tag_name)
# Tag may not have been created if tag name exceeded maximum length
# Prevent returning duplicates
if tag and tag not in result:
result.append(tag)
return result
def put(self, tag: Tag):
self.cache[tag.name.lower()] = tag
def import_netscape_html(
html: str, user: User, options: ImportOptions | None = None
) -> ImportResult:
if options is None:
options = ImportOptions()
result = ImportResult()
import_start = timezone.now()
try:
netscape_bookmarks = parse(html)
except Exception:
logging.exception("Could not read bookmarks file.")
raise
parse_end = timezone.now()
logger.debug(f"Parse duration: {parse_end - import_start}")
# Create and cache all tags beforehand
_create_missing_tags(netscape_bookmarks, user)
tag_cache = TagCache(user)
# Split bookmarks to import into batches, to keep memory usage for bulk operations manageable
batches = _get_batches(netscape_bookmarks, 200)
for batch in batches:
_import_batch(batch, user, options, tag_cache, result)
# Load favicons for newly imported bookmarks
tasks.schedule_bookmarks_without_favicons(user)
# Load previews for newly imported bookmarks
tasks.schedule_bookmarks_without_previews(user)
end = timezone.now()
logger.debug(f"Import duration: {end - import_start}")
return result
def _create_missing_tags(netscape_bookmarks: list[NetscapeBookmark], user: User):
tag_cache = TagCache(user)
tags_to_create = []
for netscape_bookmark in netscape_bookmarks:
for tag_name in netscape_bookmark.tag_names:
# Skip tag names that exceed the maximum allowed length
if len(tag_name) > 64:
logger.warning(
f"Ignoring tag '{tag_name}' (length {len(tag_name)}) as it exceeds maximum length of 64 characters"
)
continue
tag = tag_cache.get(tag_name)
if not tag:
tag = Tag(name=tag_name, owner=user)
tag.date_added = timezone.now()
tags_to_create.append(tag)
tag_cache.put(tag)
Tag.objects.bulk_create(tags_to_create)
def _get_batches(items: list, batch_size: int):
batches = []
offset = 0
num_items = len(items)
while offset < num_items:
batch = items[offset : min(offset + batch_size, num_items)]
if len(batch) > 0:
batches.append(batch)
offset = offset + batch_size
return batches
def _import_batch(
netscape_bookmarks: list[NetscapeBookmark],
user: User,
options: ImportOptions,
tag_cache: TagCache,
result: ImportResult,
):
# Query existing bookmarks
batch_urls = [bookmark.href for bookmark in netscape_bookmarks]
existing_bookmarks = Bookmark.objects.filter(owner=user, url__in=batch_urls)
# Create or update bookmarks from parsed Netscape bookmarks
bookmarks_to_create = []
bookmarks_to_update = []
for netscape_bookmark in netscape_bookmarks:
result.total = result.total + 1
try:
# Lookup existing bookmark by URL, or create new bookmark if there is no bookmark for that URL yet
bookmark = next(
(
bookmark
for bookmark in existing_bookmarks
if bookmark.url == netscape_bookmark.href
),
None,
)
if not bookmark:
bookmark = Bookmark(owner=user)
is_update = False
else:
is_update = True
# Copy data from parsed bookmark
_copy_bookmark_data(netscape_bookmark, bookmark, options)
# Validate bookmark fields, exclude owner to prevent n+1 database query,
# also there is no specific validation on owner
bookmark.clean_fields(exclude=["owner"])
# Schedule for update or insert
if is_update:
bookmarks_to_update.append(bookmark)
else:
bookmarks_to_create.append(bookmark)
result.success = result.success + 1
except Exception:
shortened_bookmark_tag_str = str(netscape_bookmark)[:100] + "..."
logging.exception("Error importing bookmark: " + shortened_bookmark_tag_str)
result.failed = result.failed + 1
# Bulk update bookmarks in DB
Bookmark.objects.bulk_update(
bookmarks_to_update,
[
"url",
"url_normalized",
"date_added",
"date_modified",
"unread",
"shared",
"title",
"description",
"notes",
"owner",
],
)
# Bulk insert new bookmarks into DB
Bookmark.objects.bulk_create(bookmarks_to_create)
# Bulk assign tags
# In Django 3, bulk_create does not return the auto-generated IDs when bulk inserting,
# so we have to reload the inserted bookmarks, and match them to the parsed bookmarks by URL
existing_bookmarks = Bookmark.objects.filter(owner=user, url__in=batch_urls)
BookmarkToTagRelationShip = Bookmark.tags.through
relationships = []
for netscape_bookmark in netscape_bookmarks:
# Lookup bookmark by URL again
bookmark = next(
(
bookmark
for bookmark in existing_bookmarks
if bookmark.url == netscape_bookmark.href
),
None,
)
if not bookmark:
# Something is wrong, we should have just created this bookmark
shortened_bookmark_tag_str = str(netscape_bookmark)[:100] + "..."
logging.warning(
f"Failed to assign tags to the bookmark: {shortened_bookmark_tag_str}. Could not find bookmark by URL."
)
continue
# Get tag models by string, schedule inserts for bookmark -> tag associations
tags = tag_cache.get_all(netscape_bookmark.tag_names)
for tag in tags:
relationships.append(BookmarkToTagRelationShip(bookmark=bookmark, tag=tag))
# Insert all bookmark -> tag associations at once, should ignore errors if association already exists
BookmarkToTagRelationShip.objects.bulk_create(relationships, ignore_conflicts=True)
def _copy_bookmark_data(
netscape_bookmark: NetscapeBookmark, bookmark: Bookmark, options: ImportOptions
):
bookmark.url = netscape_bookmark.href
bookmark.url_normalized = normalize_url(bookmark.url)
if netscape_bookmark.date_added:
bookmark.date_added = parse_timestamp(netscape_bookmark.date_added)
else:
bookmark.date_added = timezone.now()
if netscape_bookmark.date_modified:
bookmark.date_modified = parse_timestamp(netscape_bookmark.date_modified)
else:
bookmark.date_modified = bookmark.date_added
bookmark.unread = netscape_bookmark.to_read
if netscape_bookmark.title:
bookmark.title = netscape_bookmark.title
if netscape_bookmark.description:
bookmark.description = netscape_bookmark.description
if netscape_bookmark.notes:
bookmark.notes = netscape_bookmark.notes
if options.map_private_flag and not netscape_bookmark.private:
bookmark.shared = True
if netscape_bookmark.archived:
bookmark.is_archived = True
================================================
FILE: bookmarks/services/monolith.py
================================================
import gzip
import os
import shutil
import subprocess
from django.conf import settings
class MonolithError(Exception):
pass
# Monolith isn't used at the moment, as the local snapshot implementation
# switched to single-file after the prototype. Keeping this around in case
# it turns out to be useful in the future.
def create_snapshot(url: str, filepath: str):
monolith_path = settings.LD_MONOLITH_PATH
monolith_options = settings.LD_MONOLITH_OPTIONS
temp_filepath = filepath + ".tmp"
try:
command = f"{monolith_path} '{url}' {monolith_options} -o {temp_filepath}"
subprocess.run(command, check=True, shell=True)
with (
open(temp_filepath, "rb") as raw_file,
gzip.open(filepath, "wb") as gz_file,
):
shutil.copyfileobj(raw_file, gz_file)
os.remove(temp_filepath)
except subprocess.CalledProcessError as error:
raise MonolithError(f"Failed to create snapshot: {error.stderr}") from error
================================================
FILE: bookmarks/services/parser.py
================================================
import contextlib
from dataclasses import dataclass
from html.parser import HTMLParser
from bookmarks.models import parse_tag_string
@dataclass
class NetscapeBookmark:
href: str
title: str
description: str
notes: str
date_added: str
date_modified: str
tag_names: list[str]
to_read: bool
private: bool
archived: bool
class BookmarkParser(HTMLParser):
def __init__(self):
super().__init__()
self.bookmarks = []
self.current_tag = None
self.bookmark = None
self.href = ""
self.add_date = ""
self.last_modified = ""
self.tags = ""
self.title = ""
self.description = ""
self.notes = ""
self.toread = ""
self.private = ""
def handle_starttag(self, tag: str, attrs: list):
name = "handle_start_" + tag.lower()
if name in dir(self):
getattr(self, name)({k.lower(): v for k, v in attrs})
self.current_tag = tag
def handle_endtag(self, tag: str):
name = "handle_end_" + tag.lower()
if name in dir(self):
getattr(self, name)()
self.current_tag = None
def handle_data(self, data):
name = f"handle_{self.current_tag}_data"
if name in dir(self):
getattr(self, name)(data)
def handle_end_dl(self):
self.add_bookmark()
def handle_start_dt(self, attrs: dict[str, str]):
self.add_bookmark()
def handle_start_a(self, attrs: dict[str, str]):
vars(self).update(attrs)
tag_names = parse_tag_string(self.tags)
archived = "linkding:bookmarks.archived" in self.tags
with contextlib.suppress(ValueError):
tag_names.remove("linkding:bookmarks.archived")
self.bookmark = NetscapeBookmark(
href=self.href,
title="",
description="",
notes="",
date_added=self.add_date,
date_modified=self.last_modified,
tag_names=tag_names,
to_read=self.toread == "1",
# Mark as private by default, also when attribute is not specified
private=self.private != "0",
archived=archived,
)
def handle_a_data(self, data):
self.title = data.strip()
def handle_dd_data(self, data):
desc = data.strip()
if "[linkding-notes]" in desc:
self.notes = desc.split("[linkding-notes]")[1].split("[/linkding-notes]")[0]
self.description = desc.split("[linkding-notes]")[0]
def add_bookmark(self):
if self.bookmark:
self.bookmark.title = self.title
self.bookmark.description = self.description
self.bookmark.notes = self.notes
self.bookmarks.append(self.bookmark)
self.bookmark = None
self.href = ""
self.add_date = ""
self.last_modified = ""
self.tags = ""
self.title = ""
self.description = ""
self.notes = ""
self.toread = ""
self.private = ""
def parse(html: str) -> list[NetscapeBookmark]:
parser = BookmarkParser()
parser.feed(html)
return parser.bookmarks
================================================
FILE: bookmarks/services/preview_image_loader.py
================================================
import hashlib
import logging
import mimetypes
import os.path
from pathlib import Path
import requests
from django.conf import settings
from bookmarks.services import website_loader
logger = logging.getLogger(__name__)
def _ensure_preview_folder():
Path(settings.LD_PREVIEW_FOLDER).mkdir(parents=True, exist_ok=True)
def _url_to_filename(preview_image: str) -> str:
return hashlib.md5(preview_image.encode()).hexdigest()
def _get_image_path(preview_image_file: str) -> Path:
return Path(os.path.join(settings.LD_PREVIEW_FOLDER, preview_image_file))
def load_preview_image(url: str) -> str | None:
_ensure_preview_folder()
metadata = website_loader.load_website_metadata(url)
if not metadata.preview_image:
logger.debug(f"Could not find preview image in metadata: {url}")
return None
image_url = metadata.preview_image
logger.debug(f"Loading preview image: {image_url}")
with requests.get(image_url, stream=True) as response:
if response.status_code < 200 or response.status_code >= 300:
logger.debug(
f"Bad response status code for preview image: {image_url} status_code={response.status_code}"
)
return None
if "Content-Length" not in response.headers:
logger.debug(f"Empty Content-Length for preview image: {image_url}")
return None
content_length = int(response.headers["Content-Length"])
if content_length > settings.LD_PREVIEW_MAX_SIZE:
logger.debug(
f"Content-Length exceeds LD_PREVIEW_MAX_SIZE: {image_url} length={content_length}"
)
return None
if "Content-Type" not in response.headers:
logger.debug(f"Empty Content-Type for preview image: {image_url}")
return None
content_type = response.headers["Content-Type"].split(";", 1)[0]
file_extension = mimetypes.guess_extension(content_type)
if file_extension not in settings.LD_PREVIEW_ALLOWED_EXTENSIONS:
logger.debug(
f"Unsupported Content-Type for preview image: {image_url} content_type={content_type}"
)
return None
preview_image_hash = _url_to_filename(url)
preview_image_file = f"{preview_image_hash}{file_extension}"
preview_image_path = _get_image_path(preview_image_file)
with open(preview_image_path, "wb") as file:
downloaded = 0
for chunk in response.iter_content(chunk_size=8192):
downloaded += len(chunk)
if downloaded > content_length:
logger.debug(
f"Content-Length mismatch for preview image: {image_url} length={content_length} downloaded={downloaded}"
)
file.close()
preview_image_path.unlink()
return None
file.write(chunk)
logger.debug(f"Saved preview image as: {preview_image_path}")
return preview_image_file
================================================
FILE: bookmarks/services/search_query_parser.py
================================================
from dataclasses import dataclass
from enum import Enum
from bookmarks.models import UserProfile
class TokenType(Enum):
TERM = "TERM"
TAG = "TAG"
SPECIAL_KEYWORD = "SPECIAL_KEYWORD"
AND = "AND"
OR = "OR"
NOT = "NOT"
LPAREN = "LPAREN"
RPAREN = "RPAREN"
EOF = "EOF"
@dataclass
class Token:
type: TokenType
value: str
position: int
class SearchQueryTokenizer:
def __init__(self, query: str):
self.query = query.strip()
self.position = 0
self.current_char = self.query[0] if self.query else None
def advance(self):
"""Move to the next character in the query."""
self.position += 1
if self.position >= len(self.query):
self.current_char = None
else:
self.current_char = self.query[self.position]
def skip_whitespace(self):
"""Skip whitespace characters."""
while self.current_char and self.current_char.isspace():
self.advance()
def read_term(self) -> str:
"""Read a search term (sequence of non-whitespace, non-special characters)."""
term = ""
while (
self.current_char
and not self.current_char.isspace()
and self.current_char not in "()\"'#!"
):
term += self.current_char
self.advance()
return term
def read_quoted_string(self, quote_char: str) -> str:
"""Read a quoted string, handling escaped quotes."""
content = ""
self.advance() # skip opening quote
while self.current_char and self.current_char != quote_char:
if self.current_char == "\\":
# Handle escaped characters
self.advance()
if self.current_char:
if self.current_char == "n":
content += "\n"
elif self.current_char == "t":
content += "\t"
elif self.current_char == "r":
content += "\r"
elif self.current_char == "\\":
content += "\\"
elif self.current_char == quote_char:
content += quote_char
else:
# For any other escaped character, just include it as-is
content += self.current_char
self.advance()
else:
content += self.current_char
self.advance()
if self.current_char == quote_char:
self.advance() # skip closing quote
else:
# Unclosed quote - we could raise an error here, but let's be lenient
# and treat it as if the quote was closed at the end
pass
return content
def read_tag(self) -> str:
"""Read a tag (starts with # and continues until whitespace or special chars)."""
tag = ""
self.advance() # skip the # character
while (
self.current_char
and not self.current_char.isspace()
and self.current_char not in "()\"'"
):
tag += self.current_char
self.advance()
return tag
def read_special_keyword(self) -> str:
"""Read a special keyword (starts with ! and continues until whitespace or special chars)."""
keyword = ""
self.advance() # skip the ! character
while (
self.current_char
and not self.current_char.isspace()
and self.current_char not in "()\"'"
):
keyword += self.current_char
self.advance()
return keyword
def tokenize(self) -> list[Token]:
"""Convert the query string into a list of tokens."""
tokens = []
while self.current_char:
self.skip_whitespace()
if not self.current_char:
break
start_pos = self.position
if self.current_char == "(":
tokens.append(Token(TokenType.LPAREN, "(", start_pos))
self.advance()
elif self.current_char == ")":
tokens.append(Token(TokenType.RPAREN, ")", start_pos))
self.advance()
elif self.current_char in "\"'":
# Read a quoted string - always treated as a term
quote_char = self.current_char
term = self.read_quoted_string(quote_char)
tokens.append(Token(TokenType.TERM, term, start_pos))
elif self.current_char == "#":
# Read a tag
tag = self.read_tag()
# Only add the tag token if it has content
if tag:
tokens.append(Token(TokenType.TAG, tag, start_pos))
elif self.current_char == "!":
# Read a special keyword
keyword = self.read_special_keyword()
# Only add the keyword token if it has content
if keyword:
tokens.append(Token(TokenType.SPECIAL_KEYWORD, keyword, start_pos))
else:
# Read a term and check if it's an operator
term = self.read_term()
term_lower = term.lower()
if term_lower == "and":
tokens.append(Token(TokenType.AND, term, start_pos))
elif term_lower == "or":
tokens.append(Token(TokenType.OR, term, start_pos))
elif term_lower == "not":
tokens.append(Token(TokenType.NOT, term, start_pos))
else:
tokens.append(Token(TokenType.TERM, term, start_pos))
tokens.append(Token(TokenType.EOF, "", len(self.query)))
return tokens
class SearchExpression:
pass
@dataclass
class TermExpression(SearchExpression):
term: str
@dataclass
class TagExpression(SearchExpression):
tag: str
@dataclass
class SpecialKeywordExpression(SearchExpression):
keyword: str
@dataclass
class AndExpression(SearchExpression):
left: SearchExpression
right: SearchExpression
@dataclass
class OrExpression(SearchExpression):
left: SearchExpression
right: SearchExpression
@dataclass
class NotExpression(SearchExpression):
operand: SearchExpression
class SearchQueryParseError(Exception):
def __init__(self, message: str, position: int):
self.message = message
self.position = position
super().__init__(f"{message} at position {position}")
class SearchQueryParser:
def __init__(self, tokens: list[Token]):
self.tokens = tokens
self.position = 0
self.current_token = tokens[0] if tokens else Token(TokenType.EOF, "", 0)
def advance(self):
"""Move to the next token."""
if self.position < len(self.tokens) - 1:
self.position += 1
self.current_token = self.tokens[self.position]
def consume(self, expected_type: TokenType) -> Token:
"""Consume a token of the expected type or raise an error."""
if self.current_token.type == expected_type:
token = self.current_token
self.advance()
return token
else:
raise SearchQueryParseError(
f"Expected {expected_type.value}, got {self.current_token.type.value}",
self.current_token.position,
)
def parse(self) -> SearchExpression | None:
"""Parse the tokens into an AST."""
if not self.tokens or (
len(self.tokens) == 1 and self.tokens[0].type == TokenType.EOF
):
return None
expr = self.parse_or_expression()
if self.current_token.type != TokenType.EOF:
raise SearchQueryParseError(
f"Unexpected token {self.current_token.type.value}",
self.current_token.position,
)
return expr
def parse_or_expression(self) -> SearchExpression:
"""Parse OR expressions (lowest precedence)."""
left = self.parse_and_expression()
while self.current_token.type == TokenType.OR:
self.advance() # consume OR
right = self.parse_and_expression()
left = OrExpression(left, right)
return left
def parse_and_expression(self) -> SearchExpression:
"""Parse AND expressions (medium precedence), including implicit AND."""
left = self.parse_not_expression()
while self.current_token.type == TokenType.AND or self.current_token.type in [
TokenType.TERM,
TokenType.TAG,
TokenType.SPECIAL_KEYWORD,
TokenType.LPAREN,
TokenType.NOT,
]:
if self.current_token.type == TokenType.AND:
self.advance() # consume explicit AND
# else: implicit AND (don't advance token)
right = self.parse_not_expression()
left = AndExpression(left, right)
return left
def parse_not_expression(self) -> SearchExpression:
"""Parse NOT expressions (high precedence)."""
if self.current_token.type == TokenType.NOT:
self.advance() # consume NOT
operand = self.parse_not_expression() # right associative
return NotExpression(operand)
return self.parse_primary_expression()
def parse_primary_expression(self) -> SearchExpression:
"""Parse primary expressions (terms, tags, special keywords, and parenthesized expressions)."""
if self.current_token.type == TokenType.TERM:
term = self.current_token.value
self.advance()
return TermExpression(term)
elif self.current_token.type == TokenType.TAG:
tag = self.current_token.value
self.advance()
return TagExpression(tag)
elif self.current_token.type == TokenType.SPECIAL_KEYWORD:
keyword = self.current_token.value
self.advance()
return SpecialKeywordExpression(keyword)
elif self.current_token.type == TokenType.LPAREN:
self.advance() # consume (
expr = self.parse_or_expression()
self.consume(TokenType.RPAREN) # consume )
return expr
else:
raise SearchQueryParseError(
f"Unexpected token {self.current_token.type.value}",
self.current_token.position,
)
def parse_search_query(query: str) -> SearchExpression | None:
if not query or not query.strip():
return None
tokenizer = SearchQueryTokenizer(query)
tokens = tokenizer.tokenize()
parser = SearchQueryParser(tokens)
return parser.parse()
def _needs_parentheses(expr: SearchExpression, parent_type: type) -> bool:
if isinstance(expr, OrExpression) and parent_type == AndExpression:
return True
# AndExpression or OrExpression needs parentheses when inside NotExpression
return (
isinstance(expr, (AndExpression, OrExpression)) and parent_type == NotExpression
)
def _is_simple_expression(expr: SearchExpression) -> bool:
"""Check if an expression is simple (term, tag, or keyword)."""
return isinstance(expr, (TermExpression, TagExpression, SpecialKeywordExpression))
def _expression_to_string(expr: SearchExpression, parent_type: type = None) -> str:
if isinstance(expr, TermExpression):
# Quote terms if they contain spaces or special characters
if " " in expr.term or any(c in expr.term for c in ["(", ")", '"', "'"]):
# Escape any quotes in the term
escaped = expr.term.replace("\\", "\\\\").replace('"', '\\"')
return f'"{escaped}"'
return expr.term
elif isinstance(expr, TagExpression):
return f"#{expr.tag}"
elif isinstance(expr, SpecialKeywordExpression):
return f"!{expr.keyword}"
elif isinstance(expr, NotExpression):
# Don't pass parent type to children
operand_str = _expression_to_string(expr.operand, None)
# Add parentheses if the operand is a binary operation
if isinstance(expr.operand, (AndExpression, OrExpression)):
return f"not ({operand_str})"
return f"not {operand_str}"
elif isinstance(expr, AndExpression):
# Don't pass parent type to children - they'll add their own parens only if needed
left_str = _expression_to_string(expr.left, None)
right_str = _expression_to_string(expr.right, None)
# Add parentheses to children if needed for precedence
if _needs_parentheses(expr.left, AndExpression):
left_str = f"({left_str})"
if _needs_parentheses(expr.right, AndExpression):
right_str = f"({right_str})"
result = f"{left_str} {right_str}"
# Add outer parentheses if needed based on parent context
if parent_type and _needs_parentheses(expr, parent_type):
result = f"({result})"
return result
elif isinstance(expr, OrExpression):
# Don't pass parent type to children
left_str = _expression_to_string(expr.left, None)
right_str = _expression_to_string(expr.right, None)
# OrExpression children don't need parentheses unless they're also OR (handled by recursion)
result = f"{left_str} or {right_str}"
# Add outer parentheses if needed based on parent context
if parent_type and _needs_parentheses(expr, parent_type):
result = f"({result})"
return result
else:
raise ValueError(f"Unknown expression type: {type(expr)}")
def expression_to_string(expr: SearchExpression | None) -> str:
if expr is None:
return ""
return _expression_to_string(expr)
def _strip_tag_from_expression(
expr: SearchExpression | None, tag_name: str, enable_lax_search: bool = False
) -> SearchExpression | None:
if expr is None:
return None
if isinstance(expr, TagExpression):
# Remove this tag if it matches
if expr.tag.lower() == tag_name.lower():
return None
return expr
elif isinstance(expr, TermExpression):
# In lax search mode, also remove terms that match the tag name
if enable_lax_search and expr.term.lower() == tag_name.lower():
return None
return expr
elif isinstance(expr, SpecialKeywordExpression):
# Keep special keywords as-is
return expr
elif isinstance(expr, NotExpression):
# Recursively filter the operand
filtered_operand = _strip_tag_from_expression(
expr.operand, tag_name, enable_lax_search
)
if filtered_operand is None:
# If the operand is removed, the whole NOT expression should be removed
return None
return NotExpression(filtered_operand)
elif isinstance(expr, AndExpression):
# Recursively filter both sides
left = _strip_tag_from_expression(expr.left, tag_name, enable_lax_search)
right = _strip_tag_from_expression(expr.right, tag_name, enable_lax_search)
# If both sides are removed, remove the AND expression
if left is None and right is None:
return None
# If one side is removed, return the other side
elif left is None:
return right
elif right is None:
return left
else:
return AndExpression(left, right)
elif isinstance(expr, OrExpression):
# Recursively filter both sides
left = _strip_tag_from_expression(expr.left, tag_name, enable_lax_search)
right = _strip_tag_from_expression(expr.right, tag_name, enable_lax_search)
# If both sides are removed, remove the OR expression
if left is None and right is None:
return None
# If one side is removed, return the other side
elif left is None:
return right
elif right is None:
return left
else:
return OrExpression(left, right)
else:
# Unknown expression type, return as-is
return expr
def strip_tag_from_query(
query: str, tag_name: str, user_profile: UserProfile | None = None
) -> str:
try:
ast = parse_search_query(query)
except SearchQueryParseError:
return query
if ast is None:
return ""
# Determine if lax search is enabled
enable_lax_search = False
if user_profile is not None:
enable_lax_search = user_profile.tag_search == UserProfile.TAG_SEARCH_LAX
# Strip the tag from the AST
filtered_ast = _strip_tag_from_expression(ast, tag_name, enable_lax_search)
# Convert back to a query string
return expression_to_string(filtered_ast)
def _extract_tag_names_from_expression(
expr: SearchExpression | None, enable_lax_search: bool = False
) -> list[str]:
if expr is None:
return []
if isinstance(expr, TagExpression):
return [expr.tag]
elif isinstance(expr, TermExpression):
# In lax search mode, terms are also considered tags
if enable_lax_search:
return [expr.term]
return []
elif isinstance(expr, SpecialKeywordExpression):
# Special keywords are not tags
return []
elif isinstance(expr, NotExpression):
# Recursively extract from the operand
return _extract_tag_names_from_expression(expr.operand, enable_lax_search)
elif isinstance(expr, (AndExpression, OrExpression)):
# Recursively extract from both sides and combine
left_tags = _extract_tag_names_from_expression(expr.left, enable_lax_search)
right_tags = _extract_tag_names_from_expression(expr.right, enable_lax_search)
return left_tags + right_tags
else:
# Unknown expression type
return []
def extract_tag_names_from_query(
query: str, user_profile: UserProfile | None = None
) -> list[str]:
try:
ast = parse_search_query(query)
except SearchQueryParseError:
return []
if ast is None:
return []
# Determine if lax search is enabled
enable_lax_search = False
if user_profile is not None:
enable_lax_search = user_profile.tag_search == UserProfile.TAG_SEARCH_LAX
# Extract tag names from the AST
tag_names = _extract_tag_names_from_expression(ast, enable_lax_search)
# Deduplicate (case-insensitive) and sort
seen = set()
unique_tags = []
for tag in tag_names:
tag_lower = tag.lower()
if tag_lower not in seen:
seen.add(tag_lower)
unique_tags.append(tag_lower)
return sorted(unique_tags)
================================================
FILE: bookmarks/services/singlefile.py
================================================
import logging
import os
import shlex
import signal
import subprocess
from django.conf import settings
class SingleFileError(Exception):
pass
logger = logging.getLogger(__name__)
def create_snapshot(url: str, filepath: str):
singlefile_path = settings.LD_SINGLEFILE_PATH
# parse options to list of arguments
ublock_options = shlex.split(settings.LD_SINGLEFILE_UBLOCK_OPTIONS)
custom_options = shlex.split(settings.LD_SINGLEFILE_OPTIONS)
# concat lists
args = [singlefile_path] + ublock_options + custom_options + [url, filepath]
try:
# Use start_new_session=True to create a new process group
process = subprocess.Popen(args, start_new_session=True)
process.wait(timeout=settings.LD_SINGLEFILE_TIMEOUT_SEC)
# check if the file was created
if not os.path.exists(filepath):
raise SingleFileError("Failed to create snapshot")
except subprocess.TimeoutExpired:
# First try to terminate properly
try:
logger.error(
"Timeout expired while creating snapshot. Terminating process..."
)
process.terminate()
process.wait(timeout=20)
raise SingleFileError("Timeout expired while creating snapshot") from None
except subprocess.TimeoutExpired:
# Kill the whole process group, which should also clean up any chromium
# processes spawned by single-file
logger.error("Timeout expired while terminating. Killing process...")
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
raise SingleFileError("Timeout expired while creating snapshot") from None
except subprocess.CalledProcessError as error:
raise SingleFileError(f"Failed to create snapshot: {error.stderr}") from error
================================================
FILE: bookmarks/services/tags.py
================================================
import logging
import operator
from django.contrib.auth.models import User
from django.utils import timezone
from bookmarks.models import Tag
from bookmarks.utils import unique
logger = logging.getLogger(__name__)
def get_or_create_tags(tag_names: list[str], user: User):
tags = [get_or_create_tag(tag_name, user) for tag_name in tag_names]
return unique(tags, operator.attrgetter("id"))
def get_or_create_tag(name: str, user: User):
try:
return Tag.objects.get(name__iexact=name, owner=user)
except Tag.DoesNotExist:
tag = Tag(name=name, owner=user)
tag.date_added = timezone.now()
tag.save()
return tag
except Tag.MultipleObjectsReturned:
# Legacy databases might contain duplicate tags with different capitalization
first_tag = Tag.objects.filter(name__iexact=name, owner=user).first()
message = (
f"Found multiple tags for the name '{name}' with different capitalization. "
f"Using the first tag with the name '{first_tag.name}'. "
"Since v.1.2 tags work case-insensitive, which means duplicates of the same name are not allowed anymore. "
"To solve this error remove the duplicate tag in admin."
)
logger.error(message)
return first_tag
================================================
FILE: bookmarks/services/tasks.py
================================================
import functools
import logging
import waybackpy
from django.conf import settings
from django.contrib.auth.models import User
from django.db.models import Q
from django.utils import timezone
from huey import crontab
from huey.contrib.djhuey import HUEY as huey
from huey.exceptions import TaskLockedException
from waybackpy.exceptions import TooManyRequestsError, WaybackError
from bookmarks.models import Bookmark, BookmarkAsset, UserProfile
from bookmarks.services import assets, favicon_loader, preview_image_loader
from bookmarks.services.website_loader import DEFAULT_USER_AGENT, load_website_metadata
logger = logging.getLogger(__name__)
# Create custom decorator for Huey tasks that implements exponential backoff
# Taken from: https://huey.readthedocs.io/en/latest/guide.html#tips-and-tricks
# Retry 1: 60
# Retry 2: 240
# Retry 3: 960
# Retry 4: 3840
# Retry 5: 15360
def task(retries=5, retry_delay=15, retry_backoff=4):
def deco(fn):
@functools.wraps(fn)
def inner(*args, **kwargs):
task = kwargs.pop("task")
try:
return fn(*args, **kwargs)
except TaskLockedException as exc:
# Task locks are currently only used as workaround to enforce
# running specific types of tasks (e.g. singlefile snapshots)
# sequentially. In that case don't reduce the number of retries.
task.retries = retries
raise exc
except Exception as exc:
task.retry_delay *= retry_backoff
raise exc
return huey.task(retries=retries, retry_delay=retry_delay, context=True)(inner)
return deco
def is_web_archive_integration_active(user: User) -> bool:
background_tasks_enabled = not settings.LD_DISABLE_BACKGROUND_TASKS
web_archive_integration_enabled = (
user.profile.web_archive_integration
== UserProfile.WEB_ARCHIVE_INTEGRATION_ENABLED
)
return background_tasks_enabled and web_archive_integration_enabled
def create_web_archive_snapshot(user: User, bookmark: Bookmark, force_update: bool):
if is_web_archive_integration_active(user):
_create_web_archive_snapshot_task(bookmark.id, force_update)
def _create_snapshot(bookmark: Bookmark):
logger.info(f"Create new snapshot for bookmark. url={bookmark.url}...")
archive = waybackpy.WaybackMachineSaveAPI(
bookmark.url, DEFAULT_USER_AGENT, max_tries=1
)
archive.save()
bookmark.web_archive_snapshot_url = archive.archive_url
bookmark.save(update_fields=["web_archive_snapshot_url"])
logger.info(f"Successfully created new snapshot for bookmark:. url={bookmark.url}")
@task()
def _create_web_archive_snapshot_task(bookmark_id: int, force_update: bool):
try:
bookmark = Bookmark.objects.get(id=bookmark_id)
except Bookmark.DoesNotExist:
return
# Skip if snapshot exists and update is not explicitly requested
if bookmark.web_archive_snapshot_url and not force_update:
return
# Create new snapshot
try:
_create_snapshot(bookmark)
return
except TooManyRequestsError:
logger.error(
f"Failed to create snapshot due to rate limiting. url={bookmark.url}"
)
except WaybackError as error:
logger.error(
f"Failed to create snapshot. url={bookmark.url}",
exc_info=error,
)
@task()
def _load_web_archive_snapshot_task(bookmark_id: int):
# Loading snapshots from CDX API has been removed, keeping the task function
# for now to prevent errors when huey tries to run the task
pass
@task()
def _schedule_bookmarks_without_snapshots_task(user_id: int):
# Loading snapshots from CDX API has been removed, keeping the task function
# for now to prevent errors when huey tries to run the task
pass
def is_favicon_feature_active(user: User) -> bool:
background_tasks_enabled = not settings.LD_DISABLE_BACKGROUND_TASKS
return background_tasks_enabled and user.profile.enable_favicons
def is_preview_feature_active(user: User) -> bool:
return (
user.profile.enable_preview_images and not settings.LD_DISABLE_BACKGROUND_TASKS
)
def load_favicon(user: User, bookmark: Bookmark):
if is_favicon_feature_active(user):
_load_favicon_task(bookmark.id)
@task()
def _load_favicon_task(bookmark_id: int):
try:
bookmark = Bookmark.objects.get(id=bookmark_id)
except Bookmark.DoesNotExist:
return
logger.info(f"Load favicon for bookmark. url={bookmark.url}")
new_favicon_file = favicon_loader.load_favicon(bookmark.url)
if new_favicon_file != bookmark.favicon_file:
bookmark.favicon_file = new_favicon_file
bookmark.save(update_fields=["favicon_file"])
logger.info(
f"Successfully updated favicon for bookmark. url={bookmark.url} icon={new_favicon_file}"
)
def schedule_bookmarks_without_favicons(user: User):
if is_favicon_feature_active(user):
_schedule_bookmarks_without_favicons_task(user.id)
@task()
def _schedule_bookmarks_without_favicons_task(user_id: int):
user = User.objects.get(id=user_id)
bookmarks = Bookmark.objects.filter(favicon_file__exact="", owner=user)
# TODO: Implement bulk task creation
for bookmark in bookmarks:
_load_favicon_task(bookmark.id)
pass
def schedule_refresh_favicons(user: User):
if is_favicon_feature_active(user) and settings.LD_ENABLE_REFRESH_FAVICONS:
_schedule_refresh_favicons_task(user.id)
@task()
def _schedule_refresh_favicons_task(user_id: int):
user = User.objects.get(id=user_id)
bookmarks = Bookmark.objects.filter(owner=user)
# TODO: Implement bulk task creation
for bookmark in bookmarks:
_load_favicon_task(bookmark.id)
def load_preview_image(user: User, bookmark: Bookmark):
if is_preview_feature_active(user):
_load_preview_image_task(bookmark.id)
@task()
def _load_preview_image_task(bookmark_id: int):
try:
bookmark = Bookmark.objects.get(id=bookmark_id)
except Bookmark.DoesNotExist:
return
logger.info(f"Load preview image for bookmark. url={bookmark.url}")
new_preview_image_file = preview_image_loader.load_preview_image(bookmark.url)
if new_preview_image_file != bookmark.preview_image_file:
bookmark.preview_image_file = new_preview_image_file or ""
bookmark.save(update_fields=["preview_image_file"])
logger.info(
f"Successfully updated preview image for bookmark. url={bookmark.url} preview_image_file={new_preview_image_file}"
)
def schedule_bookmarks_without_previews(user: User):
if is_preview_feature_active(user):
_schedule_bookmarks_without_previews_task(user.id)
@task()
def _schedule_bookmarks_without_previews_task(user_id: int):
user = User.objects.get(id=user_id)
bookmarks = Bookmark.objects.filter(
Q(preview_image_file__exact=""),
owner=user,
)
# TODO: Implement bulk task creation
for bookmark in bookmarks:
try:
_load_preview_image_task(bookmark.id)
except Exception as exc:
logging.exception(exc)
def refresh_metadata(bookmark: Bookmark):
if not settings.LD_DISABLE_BACKGROUND_TASKS:
_refresh_metadata_task(bookmark.id)
@task()
def _refresh_metadata_task(bookmark_id: int):
try:
bookmark = Bookmark.objects.get(id=bookmark_id)
except Bookmark.DoesNotExist:
return
logger.info(f"Refresh metadata for bookmark. url={bookmark.url}")
metadata = load_website_metadata(bookmark.url)
if metadata.title:
bookmark.title = metadata.title
if metadata.description:
bookmark.description = metadata.description
bookmark.date_modified = timezone.now()
bookmark.save()
logger.info(f"Successfully refreshed metadata for bookmark. url={bookmark.url}")
def is_html_snapshot_feature_active() -> bool:
return settings.LD_ENABLE_SNAPSHOTS and not settings.LD_DISABLE_BACKGROUND_TASKS
def create_html_snapshot(bookmark: Bookmark):
if not is_html_snapshot_feature_active():
return
asset = assets.create_snapshot_asset(bookmark)
asset.save()
def create_html_snapshots(bookmark_list: list[Bookmark]):
if not is_html_snapshot_feature_active():
return
assets_to_create = []
for bookmark in bookmark_list:
asset = assets.create_snapshot_asset(bookmark)
assets_to_create.append(asset)
BookmarkAsset.objects.bulk_create(assets_to_create)
# singe-file does not support running multiple instances in parallel, so we can
# not queue up multiple snapshot tasks at once. Instead, schedule a periodic
# task that grabs a number of pending assets and creates snapshots for them in
# sequence. The task uses a lock to ensure that a new task isn't scheduled
# before the previous one has finished.
@huey.periodic_task(crontab(minute="*"))
@huey.lock_task("schedule-html-snapshots-lock")
def _schedule_html_snapshots_task():
# Get five pending assets
assets = BookmarkAsset.objects.filter(status=BookmarkAsset.STATUS_PENDING).order_by(
"date_created"
)[:5]
for asset in assets:
_create_html_snapshot_task(asset.id)
def _create_html_snapshot_task(asset_id: int):
try:
asset = BookmarkAsset.objects.get(id=asset_id)
except BookmarkAsset.DoesNotExist:
return
logger.info(f"Create HTML snapshot for bookmark. url={asset.bookmark.url}")
try:
assets.create_snapshot(asset)
logger.info(
f"Successfully created HTML snapshot for bookmark. url={asset.bookmark.url}"
)
except Exception as error:
logger.error(
f"Failed to HTML snapshot for bookmark. url={asset.bookmark.url}",
exc_info=error,
)
def create_missing_html_snapshots(user: User) -> int:
if not is_html_snapshot_feature_active():
return 0
bookmarks_without_snapshots = Bookmark.objects.filter(owner=user).exclude(
bookmarkasset__asset_type=BookmarkAsset.TYPE_SNAPSHOT,
bookmarkasset__status__in=[
BookmarkAsset.STATUS_PENDING,
BookmarkAsset.STATUS_COMPLETE,
],
)
bookmarks_without_snapshots |= Bookmark.objects.filter(owner=user).exclude(
bookmarkasset__asset_type=BookmarkAsset.TYPE_SNAPSHOT
)
create_html_snapshots(list(bookmarks_without_snapshots))
return bookmarks_without_snapshots.count()
================================================
FILE: bookmarks/services/wayback.py
================================================
import datetime
from django.utils import timezone
def generate_fallback_webarchive_url(
url: str, timestamp: datetime.datetime
) -> str | None:
"""
Generate a URL to the web archive for the given URL and timestamp.
A snapshot for the specific timestamp might not exist, in which case the
web archive will show the closest snapshot to the given timestamp.
If there is no snapshot at all the URL will be invalid.
"""
if not url:
return None
if not timestamp:
timestamp = timezone.now()
return f"https://web.archive.org/web/{timestamp.strftime('%Y%m%d%H%M%S')}/{url}"
================================================
FILE: bookmarks/services/website_loader.py
================================================
import logging
from dataclasses import dataclass
from functools import lru_cache
from urllib.parse import urljoin
import requests
from bs4 import BeautifulSoup
from charset_normalizer import from_bytes
from django.utils import timezone
logger = logging.getLogger(__name__)
@dataclass
class WebsiteMetadata:
url: str
title: str | None
description: str | None
preview_image: str | None
def to_dict(self):
return {
"url": self.url,
"title": self.title,
"description": self.description,
"preview_image": self.preview_image,
}
def load_website_metadata(url: str, ignore_cache: bool = False):
if ignore_cache:
return _load_website_metadata(url)
return _load_website_metadata_cached(url)
# Caching metadata avoids scraping again when saving bookmarks, in case the
# metadata was already scraped to show preview values in the bookmark form
@lru_cache(maxsize=10)
def _load_website_metadata_cached(url: str):
return _load_website_metadata(url)
def _load_website_metadata(url: str):
title = None
description = None
preview_image = None
try:
start = timezone.now()
page_text = load_page(url)
end = timezone.now()
logger.debug(f"Load duration: {end - start}")
start = timezone.now()
soup = BeautifulSoup(page_text, "html.parser")
if soup.title and soup.title.string:
title = soup.title.string.strip()
description_tag = soup.find("meta", attrs={"name": "description"})
description = (
description_tag["content"].strip()
if description_tag and description_tag["content"]
else None
)
if not description:
description_tag = soup.find("meta", attrs={"property": "og:description"})
description = (
description_tag["content"].strip()
if description_tag and description_tag["content"]
else None
)
image_tag = soup.find("meta", attrs={"property": "og:image"})
preview_image = image_tag["content"].strip() if image_tag else None
if (
preview_image
and not preview_image.startswith("http://")
and not preview_image.startswith("https://")
):
preview_image = urljoin(url, preview_image)
end = timezone.now()
logger.debug(f"Parsing duration: {end - start}")
except Exception:
pass
return WebsiteMetadata(
url=url, title=title, description=description, preview_image=preview_image
)
CHUNK_SIZE = 50 * 1024
MAX_CONTENT_LIMIT = 5000 * 1024
def load_page(url: str):
headers = fake_request_headers()
size = 0
content = None
iteration = 0
# Use with to ensure request gets closed even if it's only read partially
with requests.get(url, timeout=10, headers=headers, stream=True) as r:
for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
size += len(chunk)
iteration = iteration + 1
content = chunk if content is None else content + chunk
logger.debug(f"Loaded chunk (iteration={iteration}, total={size / 1024})")
# Stop reading if we have parsed end of head tag
end_of_head = b""
if end_of_head in content:
logger.debug(f"Found closing head tag after {size} bytes")
content = content.split(end_of_head)[0] + end_of_head
break
# Stop reading if we exceed limit
if size > MAX_CONTENT_LIMIT:
logger.debug(f"Cancel reading document after {size} bytes")
break
if hasattr(r, "_content_consumed"):
logger.debug(f"Request consumed: {r._content_consumed}")
# Use charset_normalizer to determine encoding that best matches the response content
# Several sites seem to specify the response encoding incorrectly, so we ignore it and use custom logic instead
# This is different from Response.text which does respect the encoding specified in the response first,
# before trying to determine one
results = from_bytes(content or "")
return str(results.best())
DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36"
def fake_request_headers():
return {
"Accept": "text/html,application/xhtml+xml,application/xml",
"Accept-Encoding": "gzip, deflate",
"Dnt": "1",
"Upgrade-Insecure-Requests": "1",
"User-Agent": DEFAULT_USER_AGENT,
}
def detect_content_type(url: str, timeout: int = 10) -> str | None:
"""Make HEAD request to detect content type of URL. Returns None on failure."""
headers = fake_request_headers()
try:
response = requests.head(
url, headers=headers, timeout=timeout, allow_redirects=True
)
if response.status_code == 200:
return (
response.headers.get("Content-Type", "").split(";")[0].strip().lower()
)
except requests.RequestException:
pass
try:
with requests.get(
url, headers=headers, timeout=timeout, stream=True, allow_redirects=True
) as response:
if response.status_code == 200:
return (
response.headers.get("Content-Type", "")
.split(";")[0]
.strip()
.lower()
)
except requests.RequestException:
pass
return None
def is_pdf_content_type(content_type: str | None) -> bool:
"""Check if the content type indicates a PDF."""
if not content_type:
return False
return content_type in ("application/pdf", "application/x-pdf")
================================================
FILE: bookmarks/settings/__init__.py
================================================
# Use dev settings as default, use production if dev settings do not exist
# ruff: noqa
try:
from .dev import *
except:
from .prod import *
================================================
FILE: bookmarks/settings/base.py
================================================
"""
Django settings for linkding webapp.
Generated by 'django-admin startproject' using Django 2.2.2.
For more information on this file, see
https://docs.djangoproject.com/en/2.2/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/2.2/ref/settings/
"""
import json
import os
import shlex
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = "kgq$h3@!!vbb6*nzfz(dbze=*)zsroqa8gvc0#1gx$3cd8z99^"
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
ALLOWED_HOSTS = ["*"]
USE_X_FORWARDED_HOST = os.getenv("LD_USE_X_FORWARDED_HOST", False) in (
True,
"True",
"true",
"1",
)
# Application definition
INSTALLED_APPS = [
"bookmarks.apps.BookmarksConfig",
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"rest_framework",
"rest_framework.authtoken",
"huey.contrib.djhuey",
"mozilla_django_oidc",
]
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"bookmarks.middlewares.LinkdingMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
"django.middleware.locale.LocaleMiddleware",
]
ROOT_URLCONF = "bookmarks.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
"bookmarks.context_processors.toasts",
"bookmarks.context_processors.app_version",
],
},
},
]
DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
WSGI_APPLICATION = "bookmarks.wsgi.application"
# Password validation
# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
},
{
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
},
{
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
},
{
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
},
]
# Website context path.
LD_CONTEXT_PATH = os.getenv("LD_CONTEXT_PATH", "")
LOGIN_URL = "/" + LD_CONTEXT_PATH + "login"
LOGIN_REDIRECT_URL = "/" + LD_CONTEXT_PATH + "bookmarks"
LOGOUT_REDIRECT_URL = "/" + LD_CONTEXT_PATH + "login"
# Internationalization
# https://docs.djangoproject.com/en/2.2/topics/i18n/
LANGUAGE_CODE = "en-us"
TIME_ZONE = os.getenv("TZ", "UTC")
USE_I18N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.2/howto/static-files/
STATIC_URL = "/" + LD_CONTEXT_PATH + "static/"
# Collect static files in static folder
STATIC_ROOT = os.path.join(BASE_DIR, "static")
# REST framework
REST_FRAMEWORK = {
"DEFAULT_AUTHENTICATION_CLASSES": [
"bookmarks.api.auth.LinkdingTokenAuthentication",
"rest_framework.authentication.SessionAuthentication",
],
"DEFAULT_PERMISSION_CLASSES": ["rest_framework.permissions.IsAuthenticated"],
"DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination",
"PAGE_SIZE": 100,
}
# URL validation flag
LD_DISABLE_URL_VALIDATION = os.getenv("LD_DISABLE_URL_VALIDATION", False) in (
True,
"True",
"true",
"1",
)
# Background task enabled setting
LD_DISABLE_BACKGROUND_TASKS = os.getenv("LD_DISABLE_BACKGROUND_TASKS", False) in (
True,
"True",
"true",
"1",
)
# Huey task queue
HUEY = {
"huey_class": "huey.SqliteHuey",
"filename": os.path.join(BASE_DIR, "data", "tasks.sqlite3"),
"immediate": False,
"results": False,
"store_none": False,
"utc": True,
"consumer": {
"workers": 2,
"worker_type": "thread",
"initial_delay": 5,
"backoff": 1.15,
"max_delay": 10,
"scheduler_interval": 10,
"periodic": True,
"check_worker_health": True,
"health_check_interval": 10,
},
}
# Disable login form if configured
LD_DISABLE_LOGIN_FORM = os.getenv("LD_DISABLE_LOGIN_FORM", False) in (
True,
"True",
"true",
"1",
)
# Enable OICD support if configured
LD_ENABLE_OIDC = os.getenv("LD_ENABLE_OIDC", False) in (True, "True", "true", "1")
AUTHENTICATION_BACKENDS = ["django.contrib.auth.backends.ModelBackend"]
if LD_ENABLE_OIDC:
AUTHENTICATION_BACKENDS.append("mozilla_django_oidc.auth.OIDCAuthenticationBackend")
OIDC_USERNAME_ALGO = "bookmarks.utils.generate_username"
OIDC_OP_AUTHORIZATION_ENDPOINT = os.getenv("OIDC_OP_AUTHORIZATION_ENDPOINT")
OIDC_OP_TOKEN_ENDPOINT = os.getenv("OIDC_OP_TOKEN_ENDPOINT")
OIDC_OP_USER_ENDPOINT = os.getenv("OIDC_OP_USER_ENDPOINT")
OIDC_OP_JWKS_ENDPOINT = os.getenv("OIDC_OP_JWKS_ENDPOINT")
OIDC_RP_CLIENT_ID = os.getenv("OIDC_RP_CLIENT_ID")
OIDC_RP_CLIENT_SECRET = os.getenv("OIDC_RP_CLIENT_SECRET")
OIDC_RP_SIGN_ALGO = os.getenv("OIDC_RP_SIGN_ALGO", "RS256")
OIDC_RP_SCOPES = os.getenv("OIDC_RP_SCOPES", "openid email profile")
OIDC_USE_PKCE = os.getenv("OIDC_USE_PKCE", True) in (True, "True", "true", "1")
OIDC_VERIFY_SSL = os.getenv("OIDC_VERIFY_SSL", True) in (True, "True", "true", "1")
OIDC_USERNAME_CLAIM = os.getenv("OIDC_USERNAME_CLAIM", "email")
# Enable authentication proxy support if configured
LD_ENABLE_AUTH_PROXY = os.getenv("LD_ENABLE_AUTH_PROXY", False) in (
True,
"True",
"true",
"1",
)
LD_AUTH_PROXY_USERNAME_HEADER = os.getenv(
"LD_AUTH_PROXY_USERNAME_HEADER", "REMOTE_USER"
)
LD_AUTH_PROXY_LOGOUT_URL = os.getenv("LD_AUTH_PROXY_LOGOUT_URL", None)
if LD_ENABLE_AUTH_PROXY:
# Add middleware that automatically authenticates requests that have a known username
# in the LD_AUTH_PROXY_USERNAME_HEADER request header
MIDDLEWARE.append("bookmarks.middlewares.CustomRemoteUserMiddleware")
# Configure auth backend that does not require a password credential
AUTHENTICATION_BACKENDS = ["django.contrib.auth.backends.RemoteUserBackend"]
# Configure logout URL
if LD_AUTH_PROXY_LOGOUT_URL:
LOGOUT_REDIRECT_URL = LD_AUTH_PROXY_LOGOUT_URL
# CSRF trusted origins
trusted_origins = os.getenv("LD_CSRF_TRUSTED_ORIGINS", "")
if trusted_origins:
CSRF_TRUSTED_ORIGINS = trusted_origins.split(",")
# Database
# https://docs.djangoproject.com/en/2.2/ref/settings/#databases
LD_DB_ENGINE = os.getenv("LD_DB_ENGINE", "sqlite")
LD_DB_HOST = os.getenv("LD_DB_HOST", "localhost")
LD_DB_DATABASE = os.getenv("LD_DB_DATABASE", "linkding")
LD_DB_USER = os.getenv("LD_DB_USER", "linkding")
LD_DB_PASSWORD = os.getenv("LD_DB_PASSWORD", None)
LD_DB_PORT = os.getenv("LD_DB_PORT", None)
LD_DB_OPTIONS = json.loads(os.getenv("LD_DB_OPTIONS") or "{}")
if LD_DB_ENGINE == "postgres":
default_database = {
"ENGINE": "django.db.backends.postgresql_psycopg2",
"NAME": LD_DB_DATABASE,
"USER": LD_DB_USER,
"PASSWORD": LD_DB_PASSWORD,
"HOST": LD_DB_HOST,
"PORT": LD_DB_PORT,
"OPTIONS": LD_DB_OPTIONS,
}
else:
default_database = {
"ENGINE": "django.db.backends.sqlite3",
"NAME": os.path.join(BASE_DIR, "data", "db.sqlite3"),
"OPTIONS": LD_DB_OPTIONS,
# Creating a connection loads the ICU extension into the SQLite
# connection, and also loads an ICU collation. The latter causes a
# memory leak, so try to counter that by making connections indefinitely
# persistent.
"CONN_MAX_AGE": None,
}
DATABASES = {"default": default_database}
SQLITE_ICU_EXTENSION_PATH = "./libicu.so"
USE_SQLITE = default_database["ENGINE"] == "django.db.backends.sqlite3"
USE_SQLITE_ICU_EXTENSION = USE_SQLITE and os.path.exists(SQLITE_ICU_EXTENSION_PATH)
# Favicons
LD_DEFAULT_FAVICON_PROVIDER = "https://t1.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url={url}&size=32"
LD_FAVICON_PROVIDER = os.getenv("LD_FAVICON_PROVIDER", LD_DEFAULT_FAVICON_PROVIDER)
LD_FAVICON_FOLDER = os.path.join(BASE_DIR, "data", "favicons")
LD_ENABLE_REFRESH_FAVICONS = os.getenv("LD_ENABLE_REFRESH_FAVICONS", True) in (
True,
"True",
"true",
"1",
)
# Previews settings
LD_PREVIEW_FOLDER = os.path.join(BASE_DIR, "data", "previews")
LD_PREVIEW_MAX_SIZE = int(os.getenv("LD_PREVIEW_MAX_SIZE", 5242880))
LD_PREVIEW_ALLOWED_EXTENSIONS = [
".jpg",
".jpeg",
".png",
".gif",
".svg",
".webp",
]
# Asset / snapshot settings
LD_ASSET_FOLDER = os.path.join(BASE_DIR, "data", "assets")
LD_ENABLE_SNAPSHOTS = os.getenv("LD_ENABLE_SNAPSHOTS", False) in (
True,
"True",
"true",
"1",
)
LD_DISABLE_ASSET_UPLOAD = os.getenv("LD_DISABLE_ASSET_UPLOAD", False) in (
True,
"True",
"true",
"1",
)
LD_SINGLEFILE_PATH = os.getenv("LD_SINGLEFILE_PATH", "single-file")
LD_SINGLEFILE_UBLOCK_OPTIONS = os.getenv(
"LD_SINGLEFILE_UBLOCK_OPTIONS",
shlex.join(
[
'--browser-arg="--headless=new"',
'--browser-arg="--user-data-dir=./chromium-profile"',
'--browser-arg="--no-sandbox"',
'--browser-arg="--load-extension=uBOLite.chromium.mv3"',
]
),
)
LD_SINGLEFILE_OPTIONS = os.getenv("LD_SINGLEFILE_OPTIONS", "")
LD_SINGLEFILE_TIMEOUT_SEC = float(os.getenv("LD_SINGLEFILE_TIMEOUT_SEC", 120))
LD_SNAPSHOT_PDF_MAX_SIZE = int(os.getenv("LD_SNAPSHOT_PDF_MAX_SIZE", 15728640)) # 15MB
# Monolith isn't used at the moment, as the local snapshot implementation
# switched to single-file after the prototype. Keeping this around in case
# it turns out to be useful in the future.
LD_MONOLITH_PATH = os.getenv("LD_MONOLITH_PATH", "monolith")
LD_MONOLITH_OPTIONS = os.getenv("LD_MONOLITH_OPTIONS", "-a -v -s")
================================================
FILE: bookmarks/settings/custom.py
================================================
# Placeholder, can be mounted in a Docker container with a custom settings
================================================
FILE: bookmarks/settings/dev.py
================================================
"""
Development settings for linkding webapp
"""
# ruff: noqa
# Start from development settings
# noinspection PyUnresolvedReferences
from .base import *
# Turn on debug mode
DEBUG = True
# Enable debug toolbar
# INSTALLED_APPS.append("debug_toolbar")
# MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware")
INTERNAL_IPS = [
"127.0.0.1",
]
# Allow access through ngrok
CSRF_TRUSTED_ORIGINS = ["https://*.ngrok-free.app"]
STATICFILES_DIRS = [
# Resolve theme files from style source folder
os.path.join(BASE_DIR, "bookmarks", "styles"),
# Resolve downloaded files in dev environment
os.path.join(BASE_DIR, "data", "favicons"),
os.path.join(BASE_DIR, "data", "previews"),
]
# Enable debug logging
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"simple": {
"format": "{levelname} {asctime} {module}: {message}",
"style": "{",
},
},
"handlers": {"console": {"class": "logging.StreamHandler", "formatter": "simple"}},
"root": {
"handlers": ["console"],
"level": "WARNING",
},
"loggers": {
"django.db.backends": {
"level": "ERROR", # Set to DEBUG to log all SQL calls
"handlers": ["console"],
},
"bookmarks": { # Log importer debug output
"level": "DEBUG",
"handlers": ["console"],
"propagate": False,
},
"huey": { # Huey
"level": "INFO",
"handlers": ["console"],
"propagate": False,
},
},
}
# Import custom settings
# noinspection PyUnresolvedReferences
from .custom import *
================================================
FILE: bookmarks/settings/prod.py
================================================
"""
Production settings for linkding webapp
"""
# ruff: noqa
# Start from development settings
# noinspection PyUnresolvedReferences
import os
from django.core.management.utils import get_random_secret_key
from .base import *
# Turn of debug mode
DEBUG = False
# Try read secret key from file
try:
with open(os.path.join(BASE_DIR, "data", "secretkey.txt")) as f:
SECRET_KEY = f.read().strip()
except:
SECRET_KEY = get_random_secret_key()
# Set ALLOWED_HOSTS
# By default look in the HOST_NAME environment variable, if that is not set then allow all hosts
host_name = os.environ.get("HOST_NAME")
if host_name:
ALLOWED_HOSTS = [host_name]
else:
ALLOWED_HOSTS = ["*"]
# Logging
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"formatters": {
"simple": {
"format": "{asctime} {levelname} {message}",
"style": "{",
},
},
"handlers": {"console": {"class": "logging.StreamHandler", "formatter": "simple"}},
"root": {
"handlers": ["console"],
"level": "WARN",
},
"loggers": {
"bookmarks": {
"level": "INFO",
"handlers": ["console"],
"propagate": False,
},
"huey": {
"level": "INFO",
"handlers": ["console"],
"propagate": False,
},
},
}
# Import custom settings
# noinspection PyUnresolvedReferences
from .custom import *
================================================
FILE: bookmarks/signals.py
================================================
from django.conf import settings
from django.db.backends.signals import connection_created
from django.dispatch import receiver
@receiver(connection_created)
def extend_sqlite(connection=None, **kwargs):
# Load ICU extension into Sqlite connection to support case-insensitive
# comparisons with unicode characters
if connection.vendor == "sqlite" and settings.USE_SQLITE_ICU_EXTENSION:
connection.connection.enable_load_extension(True)
connection.connection.load_extension(
settings.SQLITE_ICU_EXTENSION_PATH.rstrip(".so")
)
with connection.cursor() as cursor:
# Load an ICU collation for case-insensitive ordering.
# The first param can be a specific locale, it seems that not
# providing one will use a default collation from the ICU project
# that works reasonably for multiple languages
cursor.execute("SELECT icu_load_collation('', 'ICU');")
================================================
FILE: bookmarks/static/live-reload.js
================================================
const RELOAD_URL = "/live_reload";
let eventSource = null;
let serverId = null;
function connect() {
console.debug("[live-reload] Connecting to", RELOAD_URL);
eventSource = new EventSource(RELOAD_URL);
eventSource.addEventListener("connected", (event) => {
const data = JSON.parse(event.data);
if (serverId && serverId !== data.server_id) {
console.log("[live-reload] Server restarted, reloading page");
window.location.reload();
return;
}
console.debug("[live-reload] Connected, server ID:", data.server_id);
serverId = data.server_id;
});
eventSource.addEventListener("file_change", (event) => {
const data = JSON.parse(event.data);
console.log("[live-reload] File changed:", data);
if (data.file_path.endsWith(".html") || data.file_path.endsWith(".css") || data.file_path.endsWith(".js")) {
console.log("[live-reload] Asset changed, reloading page");
window.location.reload();
}
});
eventSource.onerror = (error) => {
console.debug("[live-reload] Disconnected", error);
eventSource.close();
eventSource = null;
// Reconnect after a delay
setTimeout(connect, 1000);
};
}
connect();
================================================
FILE: bookmarks/static/robots.txt
================================================
User-agent: *
Disallow: /
================================================
FILE: bookmarks/static/vendor/Readability.js
================================================
/*
* Copyright (c) 2010 Arc90 Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* This code is heavily based on Arc90's readability.js (1.7.1) script
* available at: http://code.google.com/p/arc90labs-readability
*/
/**
* Public constructor.
* @param {HTMLDocument} doc The document to parse.
* @param {Object} options The options object.
*/
function Readability(doc, options) {
// In some older versions, people passed a URI as the first argument. Cope:
if (options && options.documentElement) {
doc = options;
options = arguments[2];
} else if (!doc || !doc.documentElement) {
throw new Error("First argument to Readability constructor should be a document object.");
}
options = options || {};
this._doc = doc;
this._docJSDOMParser = this._doc.firstChild.__JSDOMParser__;
this._articleTitle = null;
this._articleByline = null;
this._articleDir = null;
this._articleSiteName = null;
this._attempts = [];
// Configurable options
this._debug = !!options.debug;
this._maxElemsToParse = options.maxElemsToParse || this.DEFAULT_MAX_ELEMS_TO_PARSE;
this._nbTopCandidates = options.nbTopCandidates || this.DEFAULT_N_TOP_CANDIDATES;
this._charThreshold = options.charThreshold || this.DEFAULT_CHAR_THRESHOLD;
this._classesToPreserve = this.CLASSES_TO_PRESERVE.concat(options.classesToPreserve || []);
this._keepClasses = !!options.keepClasses;
this._serializer = options.serializer || function(el) {
return el.innerHTML;
};
this._disableJSONLD = !!options.disableJSONLD;
this._allowedVideoRegex = options.allowedVideoRegex || this.REGEXPS.videos;
// Start with all flags set
this._flags = this.FLAG_STRIP_UNLIKELYS |
this.FLAG_WEIGHT_CLASSES |
this.FLAG_CLEAN_CONDITIONALLY;
// Control whether log messages are sent to the console
if (this._debug) {
let logNode = function(node) {
if (node.nodeType == node.TEXT_NODE) {
return `${node.nodeName} ("${node.textContent}")`;
}
let attrPairs = Array.from(node.attributes || [], function(attr) {
return `${attr.name}="${attr.value}"`;
}).join(" ");
return `<${node.localName} ${attrPairs}>`;
};
this.log = function () {
if (typeof console !== "undefined") {
let args = Array.from(arguments, arg => {
if (arg && arg.nodeType == this.ELEMENT_NODE) {
return logNode(arg);
}
return arg;
});
args.unshift("Reader: (Readability)");
console.log.apply(console, args);
} else if (typeof dump !== "undefined") {
/* global dump */
var msg = Array.prototype.map.call(arguments, function(x) {
return (x && x.nodeName) ? logNode(x) : x;
}).join(" ");
dump("Reader: (Readability) " + msg + "\n");
}
};
} else {
this.log = function () {};
}
}
Readability.prototype = {
FLAG_STRIP_UNLIKELYS: 0x1,
FLAG_WEIGHT_CLASSES: 0x2,
FLAG_CLEAN_CONDITIONALLY: 0x4,
// https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeType
ELEMENT_NODE: 1,
TEXT_NODE: 3,
// Max number of nodes supported by this parser. Default: 0 (no limit)
DEFAULT_MAX_ELEMS_TO_PARSE: 0,
// The number of top candidates to consider when analysing how
// tight the competition is among candidates.
DEFAULT_N_TOP_CANDIDATES: 5,
// Element tags to score by default.
DEFAULT_TAGS_TO_SCORE: "section,h2,h3,h4,h5,h6,p,td,pre".toUpperCase().split(","),
// The default number of chars an article must have in order to return a result
DEFAULT_CHAR_THRESHOLD: 500,
// All of the regular expressions in use within readability.
// Defined up here so we don't instantiate them repeatedly in loops.
REGEXPS: {
// NOTE: These two regular expressions are duplicated in
// Readability-readerable.js. Please keep both copies in sync.
unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i,
positive: /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,
negative: /-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i,
extraneous: /print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,
byline: /byline|author|dateline|writtenby|p-author/i,
replaceFonts: /<(\/?)font[^>]*>/gi,
normalize: /\s{2,}/g,
videos: /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,
shareElements: /(\b|_)(share|sharedaddy)(\b|_)/i,
nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,
prevLink: /(prev|earl|old|new|<|«)/i,
tokenize: /\W+/g,
whitespace: /^\s*$/,
hasContent: /\S$/,
hashUrl: /^#.+/,
srcsetUrl: /(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))/g,
b64DataUrl: /^data:\s*([^\s;,]+)\s*;\s*base64\s*,/i,
// Commas as used in Latin, Sindhi, Chinese and various other scripts.
// see: https://en.wikipedia.org/wiki/Comma#Comma_variants
commas: /\u002C|\u060C|\uFE50|\uFE10|\uFE11|\u2E41|\u2E34|\u2E32|\uFF0C/g,
// See: https://schema.org/Article
jsonLdArticleTypes: /^Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|AskPublicNewsArticle|BackgroundNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|SatiricalArticle|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference$/
},
UNLIKELY_ROLES: [ "menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog" ],
DIV_TO_P_ELEMS: new Set([ "BLOCKQUOTE", "DL", "DIV", "IMG", "OL", "P", "PRE", "TABLE", "UL" ]),
ALTER_TO_DIV_EXCEPTIONS: ["DIV", "ARTICLE", "SECTION", "P"],
PRESENTATIONAL_ATTRIBUTES: [ "align", "background", "bgcolor", "border", "cellpadding", "cellspacing", "frame", "hspace", "rules", "style", "valign", "vspace" ],
DEPRECATED_SIZE_ATTRIBUTE_ELEMS: [ "TABLE", "TH", "TD", "HR", "PRE" ],
// The commented out elements qualify as phrasing content but tend to be
// removed by readability when put into paragraphs, so we ignore them here.
PHRASING_ELEMS: [
// "CANVAS", "IFRAME", "SVG", "VIDEO",
"ABBR", "AUDIO", "B", "BDO", "BR", "BUTTON", "CITE", "CODE", "DATA",
"DATALIST", "DFN", "EM", "EMBED", "I", "IMG", "INPUT", "KBD", "LABEL",
"MARK", "MATH", "METER", "NOSCRIPT", "OBJECT", "OUTPUT", "PROGRESS", "Q",
"RUBY", "SAMP", "SCRIPT", "SELECT", "SMALL", "SPAN", "STRONG", "SUB",
"SUP", "TEXTAREA", "TIME", "VAR", "WBR"
],
// These are the classes that readability sets itself.
CLASSES_TO_PRESERVE: [ "page" ],
// These are the list of HTML entities that need to be escaped.
HTML_ESCAPE_MAP: {
"lt": "<",
"gt": ">",
"amp": "&",
"quot": '"',
"apos": "'",
},
/**
* Run any post-process modifications to article content as necessary.
*
* @param Element
* @return void
**/
_postProcessContent: function(articleContent) {
// Readability cannot open relative uris so we convert them to absolute uris.
this._fixRelativeUris(articleContent);
this._simplifyNestedElements(articleContent);
if (!this._keepClasses) {
// Remove classes.
this._cleanClasses(articleContent);
}
},
/**
* Iterates over a NodeList, calls `filterFn` for each node and removes node
* if function returned `true`.
*
* If function is not passed, removes all the nodes in node list.
*
* @param NodeList nodeList The nodes to operate on
* @param Function filterFn the function to use as a filter
* @return void
*/
_removeNodes: function(nodeList, filterFn) {
// Avoid ever operating on live node lists.
if (this._docJSDOMParser && nodeList._isLiveNodeList) {
throw new Error("Do not pass live node lists to _removeNodes");
}
for (var i = nodeList.length - 1; i >= 0; i--) {
var node = nodeList[i];
var parentNode = node.parentNode;
if (parentNode) {
if (!filterFn || filterFn.call(this, node, i, nodeList)) {
parentNode.removeChild(node);
}
}
}
},
/**
* Iterates over a NodeList, and calls _setNodeTag for each node.
*
* @param NodeList nodeList The nodes to operate on
* @param String newTagName the new tag name to use
* @return void
*/
_replaceNodeTags: function(nodeList, newTagName) {
// Avoid ever operating on live node lists.
if (this._docJSDOMParser && nodeList._isLiveNodeList) {
throw new Error("Do not pass live node lists to _replaceNodeTags");
}
for (const node of nodeList) {
this._setNodeTag(node, newTagName);
}
},
/**
* Iterate over a NodeList, which doesn't natively fully implement the Array
* interface.
*
* For convenience, the current object context is applied to the provided
* iterate function.
*
* @param NodeList nodeList The NodeList.
* @param Function fn The iterate function.
* @return void
*/
_forEachNode: function(nodeList, fn) {
Array.prototype.forEach.call(nodeList, fn, this);
},
/**
* Iterate over a NodeList, and return the first node that passes
* the supplied test function
*
* For convenience, the current object context is applied to the provided
* test function.
*
* @param NodeList nodeList The NodeList.
* @param Function fn The test function.
* @return void
*/
_findNode: function(nodeList, fn) {
return Array.prototype.find.call(nodeList, fn, this);
},
/**
* Iterate over a NodeList, return true if any of the provided iterate
* function calls returns true, false otherwise.
*
* For convenience, the current object context is applied to the
* provided iterate function.
*
* @param NodeList nodeList The NodeList.
* @param Function fn The iterate function.
* @return Boolean
*/
_someNode: function(nodeList, fn) {
return Array.prototype.some.call(nodeList, fn, this);
},
/**
* Iterate over a NodeList, return true if all of the provided iterate
* function calls return true, false otherwise.
*
* For convenience, the current object context is applied to the
* provided iterate function.
*
* @param NodeList nodeList The NodeList.
* @param Function fn The iterate function.
* @return Boolean
*/
_everyNode: function(nodeList, fn) {
return Array.prototype.every.call(nodeList, fn, this);
},
/**
* Concat all nodelists passed as arguments.
*
* @return ...NodeList
* @return Array
*/
_concatNodeLists: function() {
var slice = Array.prototype.slice;
var args = slice.call(arguments);
var nodeLists = args.map(function(list) {
return slice.call(list);
});
return Array.prototype.concat.apply([], nodeLists);
},
_getAllNodesWithTag: function(node, tagNames) {
if (node.querySelectorAll) {
return node.querySelectorAll(tagNames.join(","));
}
return [].concat.apply([], tagNames.map(function(tag) {
var collection = node.getElementsByTagName(tag);
return Array.isArray(collection) ? collection : Array.from(collection);
}));
},
/**
* Removes the class="" attribute from every element in the given
* subtree, except those that match CLASSES_TO_PRESERVE and
* the classesToPreserve array from the options object.
*
* @param Element
* @return void
*/
_cleanClasses: function(node) {
var classesToPreserve = this._classesToPreserve;
var className = (node.getAttribute("class") || "")
.split(/\s+/)
.filter(function(cls) {
return classesToPreserve.indexOf(cls) != -1;
})
.join(" ");
if (className) {
node.setAttribute("class", className);
} else {
node.removeAttribute("class");
}
for (node = node.firstElementChild; node; node = node.nextElementSibling) {
this._cleanClasses(node);
}
},
/**
* Converts each and .
* Whitespace between abc block.
var replaced = false;
// If we find a later).
while ((next = this._nextNode(next)) && (next.tagName == "BR")) {
replaced = true;
var brSibling = next.nextSibling;
next.parentNode.removeChild(next);
next = brSibling;
}
// If we removed a . Add
// all sibling nodes as children of the until we hit another .
if (next.tagName == "BR") {
var nextElem = this._nextNode(next.nextSibling);
if (nextElem && nextElem.tagName == "BR")
break;
}
if (!this._isPhrasingContent(next))
break;
// Otherwise, make this node a child of the new .
var sibling = next.nextSibling;
p.appendChild(next);
next = sibling;
}
while (p.lastChild && this._isWhitespace(p.lastChild)) {
p.removeChild(p.lastChild);
}
if (p.parentNode.tagName === "P")
this._setNodeTag(p.parentNode, "DIV");
}
});
},
_setNodeTag: function (node, tag) {
this.log("_setNodeTag", node, tag);
if (this._docJSDOMParser) {
node.localName = tag.toLowerCase();
node.tagName = tag.toUpperCase();
return node;
}
var replacement = node.ownerDocument.createElement(tag);
while (node.firstChild) {
replacement.appendChild(node.firstChild);
}
node.parentNode.replaceChild(replacement, node);
if (node.readability)
replacement.readability = node.readability;
for (var i = 0; i < node.attributes.length; i++) {
try {
replacement.setAttribute(node.attributes[i].name, node.attributes[i].value);
} catch (ex) {
/* it's possible for setAttribute() to throw if the attribute name
* isn't a valid XML Name. Such attributes can however be parsed from
* source in HTML docs, see https://github.com/whatwg/html/issues/4275,
* so we can hit them here and then throw. We don't care about such
* attributes so we ignore them.
*/
}
}
return replacement;
},
/**
* Prepare the article node for display. Clean out any inline styles,
* iframes, forms, strip extraneous tags, etc.
*
* @param Element
* @return void
**/
_prepArticle: function(articleContent) {
this._cleanStyles(articleContent);
// Check for data tables before we continue, to avoid removing items in
// those tables, which will often be isolated even though they're
// visually linked to other content-ful elements (text, images, etc.).
this._markDataTables(articleContent);
this._fixLazyImages(articleContent);
// Clean out junk from the article content
this._cleanConditionally(articleContent, "form");
this._cleanConditionally(articleContent, "fieldset");
this._clean(articleContent, "object");
this._clean(articleContent, "embed");
this._clean(articleContent, "footer");
this._clean(articleContent, "link");
this._clean(articleContent, "aside");
// Clean out elements with little content that have "share" in their id/class combinations from final top candidates,
// which means we don't remove the top candidates even they have "share".
var shareElementThreshold = this.DEFAULT_CHAR_THRESHOLD;
this._forEachNode(articleContent.children, function (topCandidate) {
this._cleanMatchedNodes(topCandidate, function (node, matchString) {
return this.REGEXPS.shareElements.test(matchString) && node.textContent.length < shareElementThreshold;
});
});
this._clean(articleContent, "iframe");
this._clean(articleContent, "input");
this._clean(articleContent, "textarea");
this._clean(articleContent, "select");
this._clean(articleContent, "button");
this._cleanHeaders(articleContent);
// Do these last as the previous stuff may have removed junk
// that will affect these
this._cleanConditionally(articleContent, "table");
this._cleanConditionally(articleContent, "ul");
this._cleanConditionally(articleContent, "div");
// replace H1 with H2 as H1 should be only title that is displayed separately
this._replaceNodeTags(this._getAllNodesWithTag(articleContent, ["h1"]), "h2");
// Remove extra paragraphs
this._removeNodes(this._getAllNodesWithTag(articleContent, ["p"]), function (paragraph) {
var imgCount = paragraph.getElementsByTagName("img").length;
var embedCount = paragraph.getElementsByTagName("embed").length;
var objectCount = paragraph.getElementsByTagName("object").length;
// At this point, nasty iframes have been removed, only remain embedded video ones.
var iframeCount = paragraph.getElementsByTagName("iframe").length;
var totalCount = imgCount + embedCount + objectCount + iframeCount;
return totalCount === 0 && !this._getInnerText(paragraph, false);
});
this._forEachNode(this._getAllNodesWithTag(articleContent, ["br"]), function(br) {
var next = this._nextNode(br.nextSibling);
if (next && next.tagName == "P")
br.parentNode.removeChild(br);
});
// Remove single-cell tables
this._forEachNode(this._getAllNodesWithTag(articleContent, ["table"]), function(table) {
var tbody = this._hasSingleTagInsideElement(table, "TBODY") ? table.firstElementChild : table;
if (this._hasSingleTagInsideElement(tbody, "TR")) {
var row = tbody.firstElementChild;
if (this._hasSingleTagInsideElement(row, "TD")) {
var cell = row.firstElementChild;
cell = this._setNodeTag(cell, this._everyNode(cell.childNodes, this._isPhrasingContent) ? "P" : "DIV");
table.parentNode.replaceChild(cell, table);
}
}
});
},
/**
* Initialize a node with the readability object. Also checks the
* className/id for special names to add to its score.
*
* @param Element
* @return void
**/
_initializeNode: function(node) {
node.readability = {"contentScore": 0};
switch (node.tagName) {
case "DIV":
node.readability.contentScore += 5;
break;
case "PRE":
case "TD":
case "BLOCKQUOTE":
node.readability.contentScore += 3;
break;
case "ADDRESS":
case "OL":
case "UL":
case "DL":
case "DD":
case "DT":
case "LI":
case "FORM":
node.readability.contentScore -= 3;
break;
case "H1":
case "H2":
case "H3":
case "H4":
case "H5":
case "H6":
case "TH":
node.readability.contentScore -= 5;
break;
}
node.readability.contentScore += this._getClassWeight(node);
},
_removeAndGetNext: function(node) {
var nextNode = this._getNextNode(node, true);
node.parentNode.removeChild(node);
return nextNode;
},
/**
* Traverse the DOM from node to node, starting at the node passed in.
* Pass true for the second parameter to indicate this node itself
* (and its kids) are going away, and we want the next node over.
*
* Calling this in a loop will traverse the DOM depth-first.
*/
_getNextNode: function(node, ignoreSelfAndKids) {
// First check for kids if those aren't being ignored
if (!ignoreSelfAndKids && node.firstElementChild) {
return node.firstElementChild;
}
// Then for siblings...
if (node.nextElementSibling) {
return node.nextElementSibling;
}
// And finally, move up the parent chain *and* find a sibling
// (because this is depth-first traversal, we will have already
// seen the parent nodes themselves).
do {
node = node.parentNode;
} while (node && !node.nextElementSibling);
return node && node.nextElementSibling;
},
// compares second text to first one
// 1 = same text, 0 = completely different text
// works the way that it splits both texts into words and then finds words that are unique in second text
// the result is given by the lower length of unique parts
_textSimilarity: function(textA, textB) {
var tokensA = textA.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean);
var tokensB = textB.toLowerCase().split(this.REGEXPS.tokenize).filter(Boolean);
if (!tokensA.length || !tokensB.length) {
return 0;
}
var uniqTokensB = tokensB.filter(token => !tokensA.includes(token));
var distanceB = uniqTokensB.join(" ").length / tokensB.join(" ").length;
return 1 - distanceB;
},
_checkByline: function(node, matchString) {
if (this._articleByline) {
return false;
}
if (node.getAttribute !== undefined) {
var rel = node.getAttribute("rel");
var itemprop = node.getAttribute("itemprop");
}
if ((rel === "author" || (itemprop && itemprop.indexOf("author") !== -1) || this.REGEXPS.byline.test(matchString)) && this._isValidByline(node.textContent)) {
this._articleByline = node.textContent.trim();
return true;
}
return false;
},
_getNodeAncestors: function(node, maxDepth) {
maxDepth = maxDepth || 0;
var i = 0, ancestors = [];
while (node.parentNode) {
ancestors.push(node.parentNode);
if (maxDepth && ++i === maxDepth)
break;
node = node.parentNode;
}
return ancestors;
},
/***
* grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
* most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
*
* @param page a document to run upon. Needs to be a full document, complete with body.
* @return Element
**/
_grabArticle: function (page) {
this.log("**** grabArticle ****");
var doc = this._doc;
var isPaging = page !== null;
page = page ? page : this._doc.body;
// We can't grab an article if we don't have a page!
if (!page) {
this.log("No body found in document. Abort.");
return null;
}
var pageCacheHtml = page.innerHTML;
while (true) {
this.log("Starting grabArticle loop");
var stripUnlikelyCandidates = this._flagIsActive(this.FLAG_STRIP_UNLIKELYS);
// First, node prepping. Trash nodes that look cruddy (like ones with the
// class name "comment", etc), and turn divs into P tags where they have been
// used inappropriately (as in, where they contain no other block level elements.)
var elementsToScore = [];
var node = this._doc.documentElement;
let shouldRemoveTitleHeader = true;
while (node) {
if (node.tagName === "HTML") {
this._articleLang = node.getAttribute("lang");
}
var matchString = node.className + " " + node.id;
if (!this._isProbablyVisible(node)) {
this.log("Removing hidden node - " + matchString);
node = this._removeAndGetNext(node);
continue;
}
// User is not able to see elements applied with both "aria-modal = true" and "role = dialog"
if (node.getAttribute("aria-modal") == "true" && node.getAttribute("role") == "dialog") {
node = this._removeAndGetNext(node);
continue;
}
// Check to see if this node is a byline, and remove it if it is.
if (this._checkByline(node, matchString)) {
node = this._removeAndGetNext(node);
continue;
}
if (shouldRemoveTitleHeader && this._headerDuplicatesTitle(node)) {
this.log("Removing header: ", node.textContent.trim(), this._articleTitle.trim());
shouldRemoveTitleHeader = false;
node = this._removeAndGetNext(node);
continue;
}
// Remove unlikely candidates
if (stripUnlikelyCandidates) {
if (this.REGEXPS.unlikelyCandidates.test(matchString) &&
!this.REGEXPS.okMaybeItsACandidate.test(matchString) &&
!this._hasAncestorTag(node, "table") &&
!this._hasAncestorTag(node, "code") &&
node.tagName !== "BODY" &&
node.tagName !== "A") {
this.log("Removing unlikely candidate - " + matchString);
node = this._removeAndGetNext(node);
continue;
}
if (this.UNLIKELY_ROLES.includes(node.getAttribute("role"))) {
this.log("Removing content with role " + node.getAttribute("role") + " - " + matchString);
node = this._removeAndGetNext(node);
continue;
}
}
// Remove DIV, SECTION, and HEADER nodes without any content(e.g. text, image, video, or iframe).
if ((node.tagName === "DIV" || node.tagName === "SECTION" || node.tagName === "HEADER" ||
node.tagName === "H1" || node.tagName === "H2" || node.tagName === "H3" ||
node.tagName === "H4" || node.tagName === "H5" || node.tagName === "H6") &&
this._isElementWithoutContent(node)) {
node = this._removeAndGetNext(node);
continue;
}
if (this.DEFAULT_TAGS_TO_SCORE.indexOf(node.tagName) !== -1) {
elementsToScore.push(node);
}
// Turn all divs that don't have children block level elements into p's
if (node.tagName === "DIV") {
// Put phrasing content into paragraphs.
var p = null;
var childNode = node.firstChild;
while (childNode) {
var nextSibling = childNode.nextSibling;
if (this._isPhrasingContent(childNode)) {
if (p !== null) {
p.appendChild(childNode);
} else if (!this._isWhitespace(childNode)) {
p = doc.createElement("p");
node.replaceChild(p, childNode);
p.appendChild(childNode);
}
} else if (p !== null) {
while (p.lastChild && this._isWhitespace(p.lastChild)) {
p.removeChild(p.lastChild);
}
p = null;
}
childNode = nextSibling;
}
// Sites like http://mobile.slate.com encloses each paragraph with a DIV
// element. DIVs with only a P element inside and no text content can be
// safely converted into plain P elements to avoid confusing the scoring
// algorithm with DIVs with are, in practice, paragraphs.
if (this._hasSingleTagInsideElement(node, "P") && this._getLinkDensity(node) < 0.25) {
var newNode = node.children[0];
node.parentNode.replaceChild(newNode, node);
node = newNode;
elementsToScore.push(node);
} else if (!this._hasChildBlockElement(node)) {
node = this._setNodeTag(node, "P");
elementsToScore.push(node);
}
}
node = this._getNextNode(node);
}
/**
* Loop through all paragraphs, and assign a score to them based on how content-y they look.
* Then add their score to their parent node.
*
* A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
**/
var candidates = [];
this._forEachNode(elementsToScore, function(elementToScore) {
if (!elementToScore.parentNode || typeof(elementToScore.parentNode.tagName) === "undefined")
return;
// If this paragraph is less than 25 characters, don't even count it.
var innerText = this._getInnerText(elementToScore);
if (innerText.length < 25)
return;
// Exclude nodes with no ancestor.
var ancestors = this._getNodeAncestors(elementToScore, 5);
if (ancestors.length === 0)
return;
var contentScore = 0;
// Add a point for the paragraph itself as a base.
contentScore += 1;
// Add points for any commas within this paragraph.
contentScore += innerText.split(this.REGEXPS.commas).length;
// For every 100 characters in this paragraph, add another point. Up to 3 points.
contentScore += Math.min(Math.floor(innerText.length / 100), 3);
// Initialize and score ancestors.
this._forEachNode(ancestors, function(ancestor, level) {
if (!ancestor.tagName || !ancestor.parentNode || typeof(ancestor.parentNode.tagName) === "undefined")
return;
if (typeof(ancestor.readability) === "undefined") {
this._initializeNode(ancestor);
candidates.push(ancestor);
}
// Node score divider:
// - parent: 1 (no division)
// - grandparent: 2
// - great grandparent+: ancestor level * 3
if (level === 0)
var scoreDivider = 1;
else if (level === 1)
scoreDivider = 2;
else
scoreDivider = level * 3;
ancestor.readability.contentScore += contentScore / scoreDivider;
});
});
// After we've calculated scores, loop through all of the possible
// candidate nodes we found and find the one with the highest score.
var topCandidates = [];
for (var c = 0, cl = candidates.length; c < cl; c += 1) {
var candidate = candidates[c];
// Scale the final candidates score based on link density. Good content
// should have a relatively small link density (5% or less) and be mostly
// unaffected by this operation.
var candidateScore = candidate.readability.contentScore * (1 - this._getLinkDensity(candidate));
candidate.readability.contentScore = candidateScore;
this.log("Candidate:", candidate, "with score " + candidateScore);
for (var t = 0; t < this._nbTopCandidates; t++) {
var aTopCandidate = topCandidates[t];
if (!aTopCandidate || candidateScore > aTopCandidate.readability.contentScore) {
topCandidates.splice(t, 0, candidate);
if (topCandidates.length > this._nbTopCandidates)
topCandidates.pop();
break;
}
}
}
var topCandidate = topCandidates[0] || null;
var neededToCreateTopCandidate = false;
var parentOfTopCandidate;
// If we still have no top candidate, just use the body as a last resort.
// We also have to copy the body node so it is something we can modify.
if (topCandidate === null || topCandidate.tagName === "BODY") {
// Move all of the page's children into topCandidate
topCandidate = doc.createElement("DIV");
neededToCreateTopCandidate = true;
// Move everything (not just elements, also text nodes etc.) into the container
// so we even include text directly in the body:
while (page.firstChild) {
this.log("Moving child out:", page.firstChild);
topCandidate.appendChild(page.firstChild);
}
page.appendChild(topCandidate);
this._initializeNode(topCandidate);
} else if (topCandidate) {
// Find a better top candidate node if it contains (at least three) nodes which belong to `topCandidates` array
// and whose scores are quite closed with current `topCandidate` node.
var alternativeCandidateAncestors = [];
for (var i = 1; i < topCandidates.length; i++) {
if (topCandidates[i].readability.contentScore / topCandidate.readability.contentScore >= 0.75) {
alternativeCandidateAncestors.push(this._getNodeAncestors(topCandidates[i]));
}
}
var MINIMUM_TOPCANDIDATES = 3;
if (alternativeCandidateAncestors.length >= MINIMUM_TOPCANDIDATES) {
parentOfTopCandidate = topCandidate.parentNode;
while (parentOfTopCandidate.tagName !== "BODY") {
var listsContainingThisAncestor = 0;
for (var ancestorIndex = 0; ancestorIndex < alternativeCandidateAncestors.length && listsContainingThisAncestor < MINIMUM_TOPCANDIDATES; ancestorIndex++) {
listsContainingThisAncestor += Number(alternativeCandidateAncestors[ancestorIndex].includes(parentOfTopCandidate));
}
if (listsContainingThisAncestor >= MINIMUM_TOPCANDIDATES) {
topCandidate = parentOfTopCandidate;
break;
}
parentOfTopCandidate = parentOfTopCandidate.parentNode;
}
}
if (!topCandidate.readability) {
this._initializeNode(topCandidate);
}
// Because of our bonus system, parents of candidates might have scores
// themselves. They get half of the node. There won't be nodes with higher
// scores than our topCandidate, but if we see the score going *up* in the first
// few steps up the tree, that's a decent sign that there might be more content
// lurking in other places that we want to unify in. The sibling stuff
// below does some of that - but only if we've looked high enough up the DOM
// tree.
parentOfTopCandidate = topCandidate.parentNode;
var lastScore = topCandidate.readability.contentScore;
// The scores shouldn't get too low.
var scoreThreshold = lastScore / 3;
while (parentOfTopCandidate.tagName !== "BODY") {
if (!parentOfTopCandidate.readability) {
parentOfTopCandidate = parentOfTopCandidate.parentNode;
continue;
}
var parentScore = parentOfTopCandidate.readability.contentScore;
if (parentScore < scoreThreshold)
break;
if (parentScore > lastScore) {
// Alright! We found a better parent to use.
topCandidate = parentOfTopCandidate;
break;
}
lastScore = parentOfTopCandidate.readability.contentScore;
parentOfTopCandidate = parentOfTopCandidate.parentNode;
}
// If the top candidate is the only child, use parent instead. This will help sibling
// joining logic when adjacent content is actually located in parent's sibling node.
parentOfTopCandidate = topCandidate.parentNode;
while (parentOfTopCandidate.tagName != "BODY" && parentOfTopCandidate.children.length == 1) {
topCandidate = parentOfTopCandidate;
parentOfTopCandidate = topCandidate.parentNode;
}
if (!topCandidate.readability) {
this._initializeNode(topCandidate);
}
}
// Now that we have the top candidate, look through its siblings for content
// that might also be related. Things like preambles, content split by ads
// that we removed, etc.
var articleContent = doc.createElement("DIV");
if (isPaging)
articleContent.id = "readability-content";
var siblingScoreThreshold = Math.max(10, topCandidate.readability.contentScore * 0.2);
// Keep potential top candidate's parent node to try to get text direction of it later.
parentOfTopCandidate = topCandidate.parentNode;
var siblings = parentOfTopCandidate.children;
for (var s = 0, sl = siblings.length; s < sl; s++) {
var sibling = siblings[s];
var append = false;
this.log("Looking at sibling node:", sibling, sibling.readability ? ("with score " + sibling.readability.contentScore) : "");
this.log("Sibling has score", sibling.readability ? sibling.readability.contentScore : "Unknown");
if (sibling === topCandidate) {
append = true;
} else {
var contentBonus = 0;
// Give a bonus if sibling nodes and top candidates have the example same classname
if (sibling.className === topCandidate.className && topCandidate.className !== "")
contentBonus += topCandidate.readability.contentScore * 0.2;
if (sibling.readability &&
((sibling.readability.contentScore + contentBonus) >= siblingScoreThreshold)) {
append = true;
} else if (sibling.nodeName === "P") {
var linkDensity = this._getLinkDensity(sibling);
var nodeContent = this._getInnerText(sibling);
var nodeLength = nodeContent.length;
if (nodeLength > 80 && linkDensity < 0.25) {
append = true;
} else if (nodeLength < 80 && nodeLength > 0 && linkDensity === 0 &&
nodeContent.search(/\.( |$)/) !== -1) {
append = true;
}
}
}
if (append) {
this.log("Appending node:", sibling);
if (this.ALTER_TO_DIV_EXCEPTIONS.indexOf(sibling.nodeName) === -1) {
// We have a node that isn't a common block level element, like a form or td tag.
// Turn it into a div so it doesn't get filtered out later by accident.
this.log("Altering sibling:", sibling, "to div.");
sibling = this._setNodeTag(sibling, "DIV");
}
articleContent.appendChild(sibling);
// Fetch children again to make it compatible
// with DOM parsers without live collection support.
siblings = parentOfTopCandidate.children;
// siblings is a reference to the children array, and
// sibling is removed from the array when we call appendChild().
// As a result, we must revisit this index since the nodes
// have been shifted.
s -= 1;
sl -= 1;
}
}
if (this._debug)
this.log("Article content pre-prep: " + articleContent.innerHTML);
// So we have all of the content that we need. Now we clean it up for presentation.
this._prepArticle(articleContent);
if (this._debug)
this.log("Article content post-prep: " + articleContent.innerHTML);
if (neededToCreateTopCandidate) {
// We already created a fake div thing, and there wouldn't have been any siblings left
// for the previous loop, so there's no point trying to create a new div, and then
// move all the children over. Just assign IDs and class names here. No need to append
// because that already happened anyway.
topCandidate.id = "readability-page-1";
topCandidate.className = "page";
} else {
var div = doc.createElement("DIV");
div.id = "readability-page-1";
div.className = "page";
while (articleContent.firstChild) {
div.appendChild(articleContent.firstChild);
}
articleContent.appendChild(div);
}
if (this._debug)
this.log("Article content after paging: " + articleContent.innerHTML);
var parseSuccessful = true;
// Now that we've gone through the full algorithm, check to see if
// we got any meaningful content. If we didn't, we may need to re-run
// grabArticle with different flags set. This gives us a higher likelihood of
// finding the content, and the sieve approach gives us a higher likelihood of
// finding the -right- content.
var textLength = this._getInnerText(articleContent, true).length;
if (textLength < this._charThreshold) {
parseSuccessful = false;
page.innerHTML = pageCacheHtml;
if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) {
this._removeFlag(this.FLAG_STRIP_UNLIKELYS);
this._attempts.push({articleContent: articleContent, textLength: textLength});
} else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) {
this._removeFlag(this.FLAG_WEIGHT_CLASSES);
this._attempts.push({articleContent: articleContent, textLength: textLength});
} else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) {
this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY);
this._attempts.push({articleContent: articleContent, textLength: textLength});
} else {
this._attempts.push({articleContent: articleContent, textLength: textLength});
// No luck after removing flags, just return the longest text we found during the different loops
this._attempts.sort(function (a, b) {
return b.textLength - a.textLength;
});
// But first check if we actually have something
if (!this._attempts[0].textLength) {
return null;
}
articleContent = this._attempts[0].articleContent;
parseSuccessful = true;
}
}
if (parseSuccessful) {
// Find out text direction from ancestors of final top candidate.
var ancestors = [parentOfTopCandidate, topCandidate].concat(this._getNodeAncestors(parentOfTopCandidate));
this._someNode(ancestors, function(ancestor) {
if (!ancestor.tagName)
return false;
var articleDir = ancestor.getAttribute("dir");
if (articleDir) {
this._articleDir = articleDir;
return true;
}
return false;
});
return articleContent;
}
}
},
/**
* Check whether the input string could be a byline.
* This verifies that the input is a string, and that the length
* is less than 100 chars.
*
* @param possibleByline {string} - a string to check whether its a byline.
* @return Boolean - whether the input string is a byline.
*/
_isValidByline: function(byline) {
if (typeof byline == "string" || byline instanceof String) {
byline = byline.trim();
return (byline.length > 0) && (byline.length < 100);
}
return false;
},
/**
* Converts some of the common HTML entities in string to their corresponding characters.
*
* @param str {string} - a string to unescape.
* @return string without HTML entity.
*/
_unescapeHtmlEntities: function(str) {
if (!str) {
return str;
}
var htmlEscapeMap = this.HTML_ESCAPE_MAP;
return str.replace(/&(quot|amp|apos|lt|gt);/g, function(_, tag) {
return htmlEscapeMap[tag];
}).replace(/(?:x([0-9a-z]{1,4})|([0-9]{1,4}));/gi, function(_, hex, numStr) {
var num = parseInt(hex || numStr, hex ? 16 : 10);
return String.fromCharCode(num);
});
},
/**
* Try to extract metadata from JSON-LD object.
* For now, only Schema.org objects of type Article or its subtypes are supported.
* @return Object with any metadata that could be extracted (possibly none)
*/
_getJSONLD: function (doc) {
var scripts = this._getAllNodesWithTag(doc, ["script"]);
var metadata;
this._forEachNode(scripts, function(jsonLdElement) {
if (!metadata && jsonLdElement.getAttribute("type") === "application/ld+json") {
try {
// Strip CDATA markers if present
var content = jsonLdElement.textContent.replace(/^\s*\s*$/g, "");
var parsed = JSON.parse(content);
if (
!parsed["@context"] ||
!parsed["@context"].match(/^https?\:\/\/schema\.org$/)
) {
return;
}
if (!parsed["@type"] && Array.isArray(parsed["@graph"])) {
parsed = parsed["@graph"].find(function(it) {
return (it["@type"] || "").match(
this.REGEXPS.jsonLdArticleTypes
);
});
}
if (
!parsed ||
!parsed["@type"] ||
!parsed["@type"].match(this.REGEXPS.jsonLdArticleTypes)
) {
return;
}
metadata = {};
if (typeof parsed.name === "string" && typeof parsed.headline === "string" && parsed.name !== parsed.headline) {
// we have both name and headline element in the JSON-LD. They should both be the same but some websites like aktualne.cz
// put their own name into "name" and the article title to "headline" which confuses Readability. So we try to check if either
// "name" or "headline" closely matches the html title, and if so, use that one. If not, then we use "name" by default.
var title = this._getArticleTitle();
var nameMatches = this._textSimilarity(parsed.name, title) > 0.75;
var headlineMatches = this._textSimilarity(parsed.headline, title) > 0.75;
if (headlineMatches && !nameMatches) {
metadata.title = parsed.headline;
} else {
metadata.title = parsed.name;
}
} else if (typeof parsed.name === "string") {
metadata.title = parsed.name.trim();
} else if (typeof parsed.headline === "string") {
metadata.title = parsed.headline.trim();
}
if (parsed.author) {
if (typeof parsed.author.name === "string") {
metadata.byline = parsed.author.name.trim();
} else if (Array.isArray(parsed.author) && parsed.author[0] && typeof parsed.author[0].name === "string") {
metadata.byline = parsed.author
.filter(function(author) {
return author && typeof author.name === "string";
})
.map(function(author) {
return author.name.trim();
})
.join(", ");
}
}
if (typeof parsed.description === "string") {
metadata.excerpt = parsed.description.trim();
}
if (
parsed.publisher &&
typeof parsed.publisher.name === "string"
) {
metadata.siteName = parsed.publisher.name.trim();
}
if (typeof parsed.datePublished === "string") {
metadata.datePublished = parsed.datePublished.trim();
}
return;
} catch (err) {
this.log(err.message);
}
}
});
return metadata ? metadata : {};
},
/**
* Attempts to get excerpt and byline metadata for the article.
*
* @param {Object} jsonld — object containing any metadata that
* could be extracted from JSON-LD object.
*
* @return Object with optional "excerpt" and "byline" properties
*/
_getArticleMetadata: function(jsonld) {
var metadata = {};
var values = {};
var metaElements = this._doc.getElementsByTagName("meta");
// property is a space-separated list of values
var propertyPattern = /\s*(article|dc|dcterm|og|twitter)\s*:\s*(author|creator|description|published_time|title|site_name)\s*/gi;
// name is a single value
var namePattern = /^\s*(?:(dc|dcterm|og|twitter|weibo:(article|webpage))\s*[\.:]\s*)?(author|creator|description|title|site_name)\s*$/i;
// Find description tags.
this._forEachNode(metaElements, function(element) {
var elementName = element.getAttribute("name");
var elementProperty = element.getAttribute("property");
var content = element.getAttribute("content");
if (!content) {
return;
}
var matches = null;
var name = null;
if (elementProperty) {
matches = elementProperty.match(propertyPattern);
if (matches) {
// Convert to lowercase, and remove any whitespace
// so we can match below.
name = matches[0].toLowerCase().replace(/\s/g, "");
// multiple authors
values[name] = content.trim();
}
}
if (!matches && elementName && namePattern.test(elementName)) {
name = elementName;
if (content) {
// Convert to lowercase, remove any whitespace, and convert dots
// to colons so we can match below.
name = name.toLowerCase().replace(/\s/g, "").replace(/\./g, ":");
values[name] = content.trim();
}
}
});
// get title
metadata.title = jsonld.title ||
values["dc:title"] ||
values["dcterm:title"] ||
values["og:title"] ||
values["weibo:article:title"] ||
values["weibo:webpage:title"] ||
values["title"] ||
values["twitter:title"];
if (!metadata.title) {
metadata.title = this._getArticleTitle();
}
// get author
metadata.byline = jsonld.byline ||
values["dc:creator"] ||
values["dcterm:creator"] ||
values["author"];
// get description
metadata.excerpt = jsonld.excerpt ||
values["dc:description"] ||
values["dcterm:description"] ||
values["og:description"] ||
values["weibo:article:description"] ||
values["weibo:webpage:description"] ||
values["description"] ||
values["twitter:description"];
// get site name
metadata.siteName = jsonld.siteName ||
values["og:site_name"];
// get article published time
metadata.publishedTime = jsonld.datePublished ||
values["article:published_time"] || null;
// in many sites the meta value is escaped with HTML entities,
// so here we need to unescape it
metadata.title = this._unescapeHtmlEntities(metadata.title);
metadata.byline = this._unescapeHtmlEntities(metadata.byline);
metadata.excerpt = this._unescapeHtmlEntities(metadata.excerpt);
metadata.siteName = this._unescapeHtmlEntities(metadata.siteName);
metadata.publishedTime = this._unescapeHtmlEntities(metadata.publishedTime);
return metadata;
},
/**
* Check if node is image, or if node contains exactly only one image
* whether as a direct child or as its descendants.
*
* @param Element
**/
_isSingleImage: function(node) {
if (node.tagName === "IMG") {
return true;
}
if (node.children.length !== 1 || node.textContent.trim() !== "") {
return false;
}
return this._isSingleImage(node.children[0]);
},
/**
* Find all uri in the given element to an absolute URI,
* ignoring #ref URIs.
*
* @param Element
* @return void
*/
_fixRelativeUris: function(articleContent) {
var baseURI = this._doc.baseURI;
var documentURI = this._doc.documentURI;
function toAbsoluteURI(uri) {
// Leave hash links alone if the base URI matches the document URI:
if (baseURI == documentURI && uri.charAt(0) == "#") {
return uri;
}
// Otherwise, resolve against base URI:
try {
return new URL(uri, baseURI).href;
} catch (ex) {
// Something went wrong, just return the original:
}
return uri;
}
var links = this._getAllNodesWithTag(articleContent, ["a"]);
this._forEachNode(links, function(link) {
var href = link.getAttribute("href");
if (href) {
// Remove links with javascript: URIs, since
// they won't work after scripts have been removed from the page.
if (href.indexOf("javascript:") === 0) {
// if the link only contains simple text content, it can be converted to a text node
if (link.childNodes.length === 1 && link.childNodes[0].nodeType === this.TEXT_NODE) {
var text = this._doc.createTextNode(link.textContent);
link.parentNode.replaceChild(text, link);
} else {
// if the link has multiple children, they should all be preserved
var container = this._doc.createElement("span");
while (link.firstChild) {
container.appendChild(link.firstChild);
}
link.parentNode.replaceChild(container, link);
}
} else {
link.setAttribute("href", toAbsoluteURI(href));
}
}
});
var medias = this._getAllNodesWithTag(articleContent, [
"img", "picture", "figure", "video", "audio", "source"
]);
this._forEachNode(medias, function(media) {
var src = media.getAttribute("src");
var poster = media.getAttribute("poster");
var srcset = media.getAttribute("srcset");
if (src) {
media.setAttribute("src", toAbsoluteURI(src));
}
if (poster) {
media.setAttribute("poster", toAbsoluteURI(poster));
}
if (srcset) {
var newSrcset = srcset.replace(this.REGEXPS.srcsetUrl, function(_, p1, p2, p3) {
return toAbsoluteURI(p1) + (p2 || "") + p3;
});
media.setAttribute("srcset", newSrcset);
}
});
},
_simplifyNestedElements: function(articleContent) {
var node = articleContent;
while (node) {
if (node.parentNode && ["DIV", "SECTION"].includes(node.tagName) && !(node.id && node.id.startsWith("readability"))) {
if (this._isElementWithoutContent(node)) {
node = this._removeAndGetNext(node);
continue;
} else if (this._hasSingleTagInsideElement(node, "DIV") || this._hasSingleTagInsideElement(node, "SECTION")) {
var child = node.children[0];
for (var i = 0; i < node.attributes.length; i++) {
child.setAttribute(node.attributes[i].name, node.attributes[i].value);
}
node.parentNode.replaceChild(child, node);
node = child;
continue;
}
}
node = this._getNextNode(node);
}
},
/**
* Get the article title as an H1.
*
* @return string
**/
_getArticleTitle: function() {
var doc = this._doc;
var curTitle = "";
var origTitle = "";
try {
curTitle = origTitle = doc.title.trim();
// If they had an element with id "title" in their HTML
if (typeof curTitle !== "string")
curTitle = origTitle = this._getInnerText(doc.getElementsByTagName("title")[0]);
} catch (e) {/* ignore exceptions setting the title. */}
var titleHadHierarchicalSeparators = false;
function wordCount(str) {
return str.split(/\s+/).length;
}
// If there's a separator in the title, first remove the final part
if ((/ [\|\-\\\/>»] /).test(curTitle)) {
titleHadHierarchicalSeparators = / [\\\/>»] /.test(curTitle);
curTitle = origTitle.replace(/(.*)[\|\-\\\/>»] .*/gi, "$1");
// If the resulting title is too short (3 words or fewer), remove
// the first part instead:
if (wordCount(curTitle) < 3)
curTitle = origTitle.replace(/[^\|\-\\\/>»]*[\|\-\\\/>»](.*)/gi, "$1");
} else if (curTitle.indexOf(": ") !== -1) {
// Check if we have an heading containing this exact string, so we
// could assume it's the full title.
var headings = this._concatNodeLists(
doc.getElementsByTagName("h1"),
doc.getElementsByTagName("h2")
);
var trimmedTitle = curTitle.trim();
var match = this._someNode(headings, function(heading) {
return heading.textContent.trim() === trimmedTitle;
});
// If we don't, let's extract the title out of the original title string.
if (!match) {
curTitle = origTitle.substring(origTitle.lastIndexOf(":") + 1);
// If the title is now too short, try the first colon instead:
if (wordCount(curTitle) < 3) {
curTitle = origTitle.substring(origTitle.indexOf(":") + 1);
// But if we have too many words before the colon there's something weird
// with the titles and the H tags so let's just use the original title instead
} else if (wordCount(origTitle.substr(0, origTitle.indexOf(":"))) > 5) {
curTitle = origTitle;
}
}
} else if (curTitle.length > 150 || curTitle.length < 15) {
var hOnes = doc.getElementsByTagName("h1");
if (hOnes.length === 1)
curTitle = this._getInnerText(hOnes[0]);
}
curTitle = curTitle.trim().replace(this.REGEXPS.normalize, " ");
// If we now have 4 words or fewer as our title, and either no
// 'hierarchical' separators (\, /, > or ») were found in the original
// title or we decreased the number of words by more than 1 word, use
// the original title.
var curTitleWordCount = wordCount(curTitle);
if (curTitleWordCount <= 4 &&
(!titleHadHierarchicalSeparators ||
curTitleWordCount != wordCount(origTitle.replace(/[\|\-\\\/>»]+/g, "")) - 1)) {
curTitle = origTitle;
}
return curTitle;
},
/**
* Prepare the HTML document for readability to scrape it.
* This includes things like stripping javascript, CSS, and handling terrible markup.
*
* @return void
**/
_prepDocument: function() {
var doc = this._doc;
// Remove all style tags in head
this._removeNodes(this._getAllNodesWithTag(doc, ["style"]));
if (doc.body) {
this._replaceBrs(doc.body);
}
this._replaceNodeTags(this._getAllNodesWithTag(doc, ["font"]), "SPAN");
},
/**
* Finds the next node, starting from the given node, and ignoring
* whitespace in between. If the given node is an element, the same node is
* returned.
*/
_nextNode: function (node) {
var next = node;
while (next
&& (next.nodeType != this.ELEMENT_NODE)
&& this.REGEXPS.whitespace.test(next.textContent)) {
next = next.nextSibling;
}
return next;
},
/**
* Replaces 2 or more successive
elements with a single
elements are ignored. For example:
*
bar
abc
bar
elements have been found and replaced with a
//
chain, remove the
s until we hit another node
// or non-whitespace. This leaves behind the first
in the chain
// (which will be replaced with a
chain, replace the remaining
with a
// chain.
if (replaced) {
var p = this._doc.createElement("p");
br.parentNode.replaceChild(p, br);
next = p.nextSibling;
while (next) {
// If we've hit another
, we're done adding children to this