local only setting for /discover

2024-07-15 16:49:21 -04:00 · 2024-07-15 16:49:21 -04:00 · 85c5c0b96a
commit 85c5c0b96a
parent 3fbc3624bd
6 changed files with 49 additions and 23 deletions
--- a/boofilsic/settings.py
+++ b/boofilsic/settings.py
@ -85,6 +85,8 @@ env = environ.FileAwareEnv(
    NEODB_MIN_MARKS_FOR_DISCOVER=(int, 1),
    # if True, only show title language with NEODB_PREFERRED_LANGUAGES
    NEODB_FILTER_LANGUAGE_FOR_DISCOVER=(bool, False),
+    # if True, only show items marked by local users rather than entire network
+    NEODB_FILTER_LOCAL_ONLY_FOR_DISCOVER=(bool, False),
    # Disable cron jobs, * for all
    NEODB_DISABLE_CRON_JOBS=(list, []),
    # federated search peers
@ -218,6 +220,7 @@ DISABLE_DEFAULT_RELAY = env("NEODB_DISABLE_DEFAULT_RELAY", default=DEBUG)
 MIN_MARKS_FOR_DISCOVER = env("NEODB_MIN_MARKS_FOR_DISCOVER")

 FILTER_LANGUAGE_FOR_DISCOVER = env("NEODB_FILTER_LANGUAGE_FOR_DISCOVER")
+FILTER_LOCAL_ONLY_FOR_DISCOVER = env("NEODB_FILTER_LOCAL_ONLY_FOR_DISCOVER")

 MASTODON_ALLOWED_SITES = env("NEODB_LOGIN_MASTODON_WHITELIST")

--- a/catalog/jobs/discover.py
+++ b/catalog/jobs/discover.py
@ -39,6 +39,8 @@ class DiscoverGenerator(BaseJob):
            .filter(created_time__gt=timezone.now() - timedelta(days=days))
            .exclude(item_id__in=exisiting_ids)
        )
+        if settings.FILTER_LOCAL_ONLY_FOR_DISCOVER:
+            qs = qs.filter(local=True)
        if settings.FILTER_LANGUAGE_FOR_DISCOVER:
            q = None
            for loc in PREFERRED_LOCALES:
@ -58,10 +60,13 @@ class DiscoverGenerator(BaseJob):
        return item_ids

    def get_popular_commented_podcast_ids(self, days, exisiting_ids):
+        qs = Comment.objects.filter(q_item_in_category(ItemCategory.Podcast)).filter(
+            created_time__gt=timezone.now() - timedelta(days=days)
+        )
+        if settings.FILTER_LOCAL_ONLY_FOR_DISCOVER:
+            qs = qs.filter(local=True)
        return list(
-            Comment.objects.filter(q_item_in_category(ItemCategory.Podcast))
-            .filter(created_time__gt=timezone.now() - timedelta(days=days))
-            .annotate(p=F("item__podcastepisode__program"))
+            qs.annotate(p=F("item__podcastepisode__program"))
            .filter(p__isnull=False)
            .exclude(p__in=exisiting_ids)
            .values("p")
@ -80,6 +85,7 @@ class DiscoverGenerator(BaseJob):

    def run(self):
        logger.info("Discover data update start.")
+        local = settings.FILTER_LOCAL_ONLY_FOR_DISCOVER
        gallery_categories = [
            ItemCategory.Book,
            ItemCategory.Movie,
@ -143,42 +149,49 @@ class DiscoverGenerator(BaseJob):
                    }
                )
        trends.sort(key=lambda x: int(x["history"][0]["accounts"]), reverse=True)
-        collection_ids = (
+
+        collections = (
            Collection.objects.filter(visibility=0)
            .annotate(num=Count("interactions"))
            .filter(num__gte=MIN_MARKS)
            .order_by("-edited_time")
-            .values_list("pk", flat=True)[:40]
        )
-        tags = TagManager.popular_tags(days=14)[:40]
+        if local:
+            collections = collections.filter(local=True)
+        collection_ids = collections.values_list("pk", flat=True)[:40]
+
+        tags = TagManager.popular_tags(days=14, local_only=local)[:40]
        excluding_identities = Takahe.get_no_discover_identities()
-        post_ids = (
-            set(
-                Takahe.get_popular_posts(
-                    28, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities
-                ).values_list("pk", flat=True)[:5]
-            )
-            | set(
-                Takahe.get_popular_posts(
-                    14, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities
-                ).values_list("pk", flat=True)[:5]
-            )
-            | set(
-                Takahe.get_popular_posts(
-                    7, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities
-                ).values_list("pk", flat=True)[:10]
-            )
-            | set(
-                Takahe.get_popular_posts(1, 0, excluding_identities).values_list(
-                    "pk", flat=True
-                )[:3]
-            )
-            | set(
+
+        reviews = (
            Review.objects.filter(visibility=0)
            .exclude(owner_id__in=excluding_identities)
            .order_by("-created_time")
-                .values_list("posts", flat=True)[:5]
        )
+        if local:
+            reviews = reviews.filter(local=True)
+        post_ids = (
+            set(
+                Takahe.get_popular_posts(
+                    28, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities, local
+                ).values_list("pk", flat=True)[:5]
+            )
+            | set(
+                Takahe.get_popular_posts(
+                    14, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities, local
+                ).values_list("pk", flat=True)[:5]
+            )
+            | set(
+                Takahe.get_popular_posts(
+                    7, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities, local
+                ).values_list("pk", flat=True)[:10]
+            )
+            | set(
+                Takahe.get_popular_posts(1, 0, excluding_identities, local).values_list(
+                    "pk", flat=True
+                )[:3]
+            )
+            | set(reviews.values_list("posts", flat=True)[:5])
        )
        cache.set("public_gallery", gallery_list, timeout=None)
        cache.set("trends_links", trends, timeout=None)
--- a/compose.yml
+++ b/compose.yml
@ -35,6 +35,7 @@ x-shared:
      NEODB_SEARCH_PEERS:
      NEODB_MIN_MARKS_FOR_DISCOVER:
      NEODB_FILTER_LANGUAGE_FOR_DISCOVER:
+      NEODB_FILTER_LOCAL_ONLY_FOR_DISCOVER:
      NEODB_SENTRY_DSN:
      TAKAHE_SENTRY_DSN:
      NEODB_SENTRY_SAMPLE_RATE:
--- a/docs/configuration.md
+++ b/docs/configuration.md
@ -24,8 +24,9 @@ if you are doing debug or development:
 - `NEODB_SITE_INTRO`
 - `NEODB_SITE_HEAD`
 - `NEODB_SITE_DESCRIPTION`
- - `NEODB_PREFERRED_LANGUAGES` - preferred languages when importing titles from 3rd party sites like TMDB and Steam, comma-separated list of ISO-639-1 two-letter codes, `en,zh` by default.
- - `NEODB_FILTER_LANGUAGE_FOR_DISCOVER` - when set to `True`, `/discover/` will only show items with languages match one of `NEODB_PREFERRED_LANGUAGES`. `False` by default.
+ - `NEODB_PREFERRED_LANGUAGES` - preferred languages when importing titles from 3rd party sites like TMDB and Steam, comma-separated list of ISO-639-1 two-letter codes, `en,zh` by default. It can includes languages with no UI translations yet, e.g. if set to `ja,en,zh`, NeoDB scraper will fetch catalog metadata in three languages if they are available from third party sites, Japanese users (= whose browser language set to ja-JP) will see English UI with Japanese metadata.
+ - `NEODB_FILTER_LANGUAGE_FOR_DISCOVER` - `False` by default; when set to `True`, `/discover/` will only show items with languages match one of `NEODB_PREFERRED_LANGUAGES`.
+ - `NEODB_FILTER_LOCAL_ONLY_FOR_DISCOVER` - `False` by default; when set to `True`, only show items marked by local users rather than entire network in `/discover/`
 - `NEODB_SITE_LINKS` - a list of title and links to show in the footer, comma separated, e.g. `Feedback=https://discord.gg/8KweCuApaK,ToS=/pages/rules/`
 - `NEODB_INVITE_ONLY` - `False` by default, set to `True` to require invite code(generated by `neodb-manage invite --create`) to register
 - `NEODB_ENABLE_LOCAL_ONLY` - `False` by default, set to `True` to allow user to post marks as "local public"
--- a/journal/models/tag.py
+++ b/journal/models/tag.py
@ -146,17 +146,19 @@ class TagManager:
        return tags

    @staticmethod
-    def popular_tags(days: int = 30):
+    def popular_tags(days: int = 30, local_only: bool = False):
        t = timezone.now() - timedelta(days=days)
-        titles = (
+        tags = (
            TagMember.objects.filter(created_time__gt=t)
            .filter(parent__visibility=0)
            .annotate(title=F("parent__title"))
            .values("title")
            .annotate(total=Count("parent_id", distinct=True))
            .order_by("-total")
-            .values_list("title", flat=True)
        )
+        if local_only:
+            tags = tags.filter(local=True)
+        titles = tags.values_list("title", flat=True)
        return titles

    def get_item_tags(self, item: Item):
--- a/takahe/utils.py
+++ b/takahe/utils.py
@ -837,11 +837,14 @@ class Takahe:

    @staticmethod
    def get_popular_posts(
-        days: int = 30, min_interaction: int = 1, exclude_identities: list[int] = []
+        days: int = 30,
+        min_interaction: int = 1,
+        exclude_identities: list[int] = [],
+        local_only=False,
    ):
        since = timezone.now() - timedelta(days=days)
        domains = Takahe.get_neodb_peers() + [settings.SITE_DOMAIN]
-        return (
+        qs = (
            Post.objects.exclude(state__in=["deleted", "deleted_fanned_out"])
            .exclude(author_id__in=exclude_identities)
            .filter(
@ -853,6 +856,9 @@ class Takahe:
            .filter(num_interactions__gte=min_interaction)
            .order_by("-num_interactions", "-published")
        )
+        if local_only:
+            qs = qs.filter(local=True)
+        return qs

    @staticmethod
    def get_recent_posts(author_pk: int, viewer_pk: int | None = None):