Master 1.30.0

.gitignore

@@ -194,6 +194,7 @@ typings/
 # ublock + chromium
 /uBlock0.chromium
 /chromium-profile
+/.direnv
 
 # KJ
 deploy.sh

CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## v1.30.0 (20/04/2024)
+
+### What's Changed
+* Add reader mode by @sissbruecker in https://github.com/sissbruecker/linkding/pull/703
+* Allow uploading custom files for bookmarks by @sissbruecker in https://github.com/sissbruecker/linkding/pull/713
+* Add option for marking bookmarks as unread by default by @ab623 in https://github.com/sissbruecker/linkding/pull/706
+* Make blocking cookie banners more reliable by @sissbruecker in https://github.com/sissbruecker/linkding/pull/699
+* Close bookmark details with escape by @sissbruecker in https://github.com/sissbruecker/linkding/pull/702
+* Show proper name for bookmark assets in admin by @ab623 in https://github.com/sissbruecker/linkding/pull/708
+* Bump sqlparse from 0.4.4 to 0.5.0 by @dependabot in https://github.com/sissbruecker/linkding/pull/704
+
+### New Contributors
+* @ab623 made their first contribution in https://github.com/sissbruecker/linkding/pull/706
+
+**Full Changelog**: https://github.com/sissbruecker/linkding/compare/v1.29.0...v1.30.0
+
+---
+
 ## v1.29.0 (14/04/2024)
 
 ### What's Changed

bookmarks/api/routes.py

@@ -99,7 +99,10 @@ def check(self, request):
         # Either return metadata from existing bookmark, or scrape from URL
         if bookmark:
             metadata = WebsiteMetadata(
-                url, bookmark.website_title, bookmark.website_description
+                url,
+                bookmark.website_title,
+                bookmark.website_description,
+                None,
             )
         else:
             metadata = website_loader.load_website_metadata(url)

bookmarks/migrations/0034_bookmark_preview_image_file_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.0.3 on 2024-05-10 07:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookmarks", "0033_userprofile_default_mark_unread"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="bookmark",
+            name="preview_image_file",
+            field=models.CharField(blank=True, max_length=512),
+        ),
+        migrations.AddField(
+            model_name="userprofile",
+            name="enable_preview_images",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookmarks/migrations/0035_userprofile_tag_grouping.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.0.3 on 2024-05-14 08:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookmarks", "0034_bookmark_preview_image_file_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="userprofile",
+            name="tag_grouping",
+            field=models.CharField(
+                choices=[("alphabetical", "Alphabetical"), ("disabled", "Disabled")],
+                default="alphabetical",
+                max_length=12,
+            ),
+        ),
+    ]

bookmarks/migrations/0036_userprofile_auto_tagging_rules.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.3 on 2024-05-17 07:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookmarks", "0035_userprofile_tag_grouping"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="userprofile",
+            name="auto_tagging_rules",
+            field=models.TextField(blank=True),
+        ),
+    ]

bookmarks/models.py

@@ -59,6 +59,7 @@ class Bookmark(models.Model):
     website_description = models.TextField(blank=True, null=True)
     web_archive_snapshot_url = models.CharField(max_length=2048, blank=True)
     favicon_file = models.CharField(max_length=512, blank=True)
+    preview_image_file = models.CharField(max_length=512, blank=True)
     unread = models.BooleanField(default=False)
     is_archived = models.BooleanField(default=False)
     shared = models.BooleanField(default=False)
@@ -351,6 +352,12 @@ class UserProfile(models.Model):
         (TAG_SEARCH_STRICT, "Strict"),
         (TAG_SEARCH_LAX, "Lax"),
     ]
+    TAG_GROUPING_ALPHABETICAL = "alphabetical"
+    TAG_GROUPING_DISABLED = "disabled"
+    TAG_GROUPING_CHOICES = [
+        (TAG_GROUPING_ALPHABETICAL, "Alphabetical"),
+        (TAG_GROUPING_DISABLED, "Disabled"),
+    ]
     user = models.OneToOneField(
         get_user_model(), related_name="profile", on_delete=models.CASCADE
     )
@@ -391,16 +398,24 @@ class UserProfile(models.Model):
         blank=False,
         default=TAG_SEARCH_STRICT,
     )
+    tag_grouping = models.CharField(
+        max_length=12,
+        choices=TAG_GROUPING_CHOICES,
+        blank=False,
+        default=TAG_GROUPING_ALPHABETICAL,
+    )
     enable_sharing = models.BooleanField(default=False, null=False)
     enable_public_sharing = models.BooleanField(default=False, null=False)
     enable_favicons = models.BooleanField(default=False, null=False)
+    enable_preview_images = models.BooleanField(default=False, null=False)
     display_url = models.BooleanField(default=False, null=False)
     display_view_bookmark_action = models.BooleanField(default=True, null=False)
     display_edit_bookmark_action = models.BooleanField(default=True, null=False)
     display_archive_bookmark_action = models.BooleanField(default=True, null=False)
     display_remove_bookmark_action = models.BooleanField(default=True, null=False)
     permanent_notes = models.BooleanField(default=False, null=False)
     custom_css = models.TextField(blank=True, null=False)
+    auto_tagging_rules = models.TextField(blank=True, null=False)
     search_preferences = models.JSONField(default=dict, null=False)
     enable_automatic_html_snapshots = models.BooleanField(default=True, null=False)
     default_mark_unread = models.BooleanField(default=False, null=False)
@@ -417,9 +432,11 @@ class Meta:
             "bookmark_link_target",
             "web_archive_integration",
             "tag_search",
+            "tag_grouping",
             "enable_sharing",
             "enable_public_sharing",
             "enable_favicons",
+            "enable_preview_images",
             "enable_automatic_html_snapshots",
             "display_url",
             "display_view_bookmark_action",
@@ -429,6 +446,7 @@ class Meta:
             "permanent_notes",
             "default_mark_unread",
             "custom_css",
+            "auto_tagging_rules",
         ]
 
 

bookmarks/services/auto_tagging.py

@@ -0,0 +1,70 @@
+from urllib.parse import urlparse, parse_qs
+import re
+import idna
+
+
+def get_tags(script: str, url: str):
+    parsed_url = urlparse(url.lower())
+    result = set()
+
+    for line in script.lower().split("\n"):
+        if "#" in line:
+            i = line.index("#")
+            line = line[:i]
+
+        parts = line.split()
+        if len(parts) < 2:
+            continue
+
+        domain_pattern = re.sub("^https?://", "", parts[0])
+        path_pattern = None
+        qs_pattern = None
+
+        if "/" in domain_pattern:
+            i = domain_pattern.index("/")
+            path_pattern = domain_pattern[i:]
+            domain_pattern = domain_pattern[:i]
+
+        if path_pattern and "?" in path_pattern:
+            i = path_pattern.index("?")
+            qs_pattern = path_pattern[i + 1 :]
+            path_pattern = path_pattern[:i]
+
+        if not _domains_matches(domain_pattern, parsed_url.netloc):
+            continue
+
+        if path_pattern and not _path_matches(path_pattern, parsed_url.path):
+            continue
+
+        if qs_pattern and not _qs_matches(qs_pattern, parsed_url.query):
+            continue
+
+        for tag in parts[1:]:
+            result.add(tag)
+
+    return result
+
+
+def _path_matches(expected_path: str, actual_path: str) -> bool:
+    return actual_path.startswith(expected_path)
+
+
+def _domains_matches(expected_domain: str, actual_domain: str) -> bool:
+    expected_domain = idna.encode(expected_domain)
+    actual_domain = idna.encode(actual_domain)
+
+    return actual_domain.endswith(expected_domain)
+
+
+def _qs_matches(expected_qs: str, actual_qs: str) -> bool:
+    expected_qs = parse_qs(expected_qs, keep_blank_values=True)
+    actual_qs = parse_qs(actual_qs, keep_blank_values=True)
+
+    for key in expected_qs:
+        if key not in actual_qs:
+            return False
+        for value in expected_qs[key]:
+            if value != "" and value not in actual_qs[key]:
+                return False
+
+    return True

bookmarks/services/bookmarks.py

@@ -10,6 +10,7 @@
 from bookmarks.models import Bookmark, BookmarkAsset, parse_tag_string
 from bookmarks.services import tasks
 from bookmarks.services import website_loader
+from bookmarks.services import auto_tagging
 from bookmarks.services.tags import get_or_create_tags
 
 logger = logging.getLogger(__name__)
@@ -40,6 +41,8 @@ def create_bookmark(bookmark: Bookmark, tag_string: str, current_user: User):
     tasks.create_web_archive_snapshot(current_user, bookmark, False)
     # Load favicon
     tasks.load_favicon(current_user, bookmark)
+    # Load preview image
+    tasks.load_preview_image(current_user, bookmark)
     # Create HTML snapshot
     if current_user.profile.enable_automatic_html_snapshots:
         tasks.create_html_snapshot(bookmark)
@@ -58,6 +61,8 @@ def update_bookmark(bookmark: Bookmark, tag_string, current_user: User):
     bookmark.save()
     # Update favicon
     tasks.load_favicon(current_user, bookmark)
+    # Update preview image
+    tasks.load_preview_image(current_user, bookmark)
 
     if has_url_changed:
         # Update web archive snapshot, if URL changed
@@ -238,6 +243,15 @@ def _update_website_metadata(bookmark: Bookmark):
 
 def _update_bookmark_tags(bookmark: Bookmark, tag_string: str, user: User):
     tag_names = parse_tag_string(tag_string)
+
+    if user.profile.auto_tagging_rules:
+        auto_tag_names = auto_tagging.get_tags(
+            user.profile.auto_tagging_rules, bookmark.url
+        )
+        for auto_tag_name in auto_tag_names:
+            if auto_tag_name not in tag_names:
+                tag_names.append(auto_tag_name)
+
     tags = get_or_create_tags(tag_names, user)
     bookmark.tags.set(tags)
 

bookmarks/services/importer.py

@@ -83,6 +83,8 @@ def import_netscape_html(
     tasks.schedule_bookmarks_without_snapshots(user)
     # Load favicons for newly imported bookmarks
     tasks.schedule_bookmarks_without_favicons(user)
+    # Load previews for newly imported bookmarks
+    tasks.schedule_bookmarks_without_previews(user)
 
     end = timezone.now()
     logger.debug(f"Import duration: {end - import_start}")

bookmarks/services/preview_image_loader.py

@@ -0,0 +1,88 @@
+import logging
+import mimetypes
+import os.path
+import hashlib
+from pathlib import Path
+
+import requests
+from django.conf import settings
+from bookmarks.services import website_loader
+
+logger = logging.getLogger(__name__)
+
+
+def _ensure_preview_folder():
+    Path(settings.LD_PREVIEW_FOLDER).mkdir(parents=True, exist_ok=True)
+
+
+def _url_to_filename(preview_image: str) -> str:
+    return hashlib.md5(preview_image.encode()).hexdigest()
+
+
+def _get_image_path(preview_image_file: str) -> Path:
+    return Path(os.path.join(settings.LD_PREVIEW_FOLDER, preview_image_file))
+
+
+def load_preview_image(url: str) -> str | None:
+    _ensure_preview_folder()
+
+    metadata = website_loader.load_website_metadata(url)
+    if not metadata.preview_image:
+        logger.debug(f"Could not find preview image in metadata: {url}")
+        return None
+
+    image_url = metadata.preview_image
+
+    logger.debug(f"Loading preview image: {image_url}")
+    with requests.get(image_url, stream=True) as response:
+        if response.status_code < 200 or response.status_code >= 300:
+            logger.debug(
+                f"Bad response status code for preview image: {image_url} status_code={response.status_code}"
+            )
+            return None
+
+        if "Content-Length" not in response.headers:
+            logger.debug(f"Empty Content-Length for preview image: {image_url}")
+            return None
+
+        content_length = int(response.headers["Content-Length"])
+        if content_length > settings.LD_PREVIEW_MAX_SIZE:
+            logger.debug(
+                f"Content-Length exceeds LD_PREVIEW_MAX_SIZE: {image_url} length={content_length}"
+            )
+            return None
+
+        if "Content-Type" not in response.headers:
+            logger.debug(f"Empty Content-Type for preview image: {image_url}")
+            return None
+
+        content_type = response.headers["Content-Type"].split(";", 1)[0]
+        file_extension = mimetypes.guess_extension(content_type)
+
+        if file_extension not in settings.LD_PREVIEW_ALLOWED_EXTENSIONS:
+            logger.debug(
+                f"Unsupported Content-Type for preview image: {image_url} content_type={content_type}"
+            )
+            return None
+
+        preview_image_hash = _url_to_filename(url)
+        preview_image_file = f"{preview_image_hash}{file_extension}"
+        preview_image_path = _get_image_path(preview_image_file)
+
+        with open(preview_image_path, "wb") as file:
+            downloaded = 0
+            for chunk in response.iter_content(chunk_size=8192):
+                downloaded += len(chunk)
+                if downloaded > content_length:
+                    logger.debug(
+                        f"Content-Length mismatch for preview image: {image_url} length={content_length} downloaded={downloaded}"
+                    )
+                    file.close()
+                    preview_image_path.unlink()
+                    return None
+
+                file.write(chunk)
+
+    logger.debug(f"Saved preview image as: {preview_image_path}")
+
+    return preview_image_file