local only setting for /discover

boofilsic/settings.py

@@ -85,6 +85,8 @@ env = environ.FileAwareEnv(
     NEODB_MIN_MARKS_FOR_DISCOVER=(int, 1),
     # if True, only show title language with NEODB_PREFERRED_LANGUAGES
     NEODB_FILTER_LANGUAGE_FOR_DISCOVER=(bool, False),
+    # if True, only show items marked by local users rather than entire network
+    NEODB_FILTER_LOCAL_ONLY_FOR_DISCOVER=(bool, False),
     # Disable cron jobs, * for all
     NEODB_DISABLE_CRON_JOBS=(list, []),
     # federated search peers
@@ -218,6 +220,7 @@ DISABLE_DEFAULT_RELAY = env("NEODB_DISABLE_DEFAULT_RELAY", default=DEBUG)
 MIN_MARKS_FOR_DISCOVER = env("NEODB_MIN_MARKS_FOR_DISCOVER")
 
 FILTER_LANGUAGE_FOR_DISCOVER = env("NEODB_FILTER_LANGUAGE_FOR_DISCOVER")
+FILTER_LOCAL_ONLY_FOR_DISCOVER = env("NEODB_FILTER_LOCAL_ONLY_FOR_DISCOVER")
 
 MASTODON_ALLOWED_SITES = env("NEODB_LOGIN_MASTODON_WHITELIST")
 

catalog/jobs/discover.py

@@ -39,6 +39,8 @@ class DiscoverGenerator(BaseJob):
             .filter(created_time__gt=timezone.now() - timedelta(days=days))
             .exclude(item_id__in=exisiting_ids)
         )
+        if settings.FILTER_LOCAL_ONLY_FOR_DISCOVER:
+            qs = qs.filter(local=True)
         if settings.FILTER_LANGUAGE_FOR_DISCOVER:
             q = None
             for loc in PREFERRED_LOCALES:
@@ -58,10 +60,13 @@ class DiscoverGenerator(BaseJob):
         return item_ids
 
     def get_popular_commented_podcast_ids(self, days, exisiting_ids):
+        qs = Comment.objects.filter(q_item_in_category(ItemCategory.Podcast)).filter(
+            created_time__gt=timezone.now() - timedelta(days=days)
+        )
+        if settings.FILTER_LOCAL_ONLY_FOR_DISCOVER:
+            qs = qs.filter(local=True)
         return list(
-            Comment.objects.filter(q_item_in_category(ItemCategory.Podcast))
+            qs.annotate(p=F("item__podcastepisode__program"))
-            .filter(created_time__gt=timezone.now() - timedelta(days=days))
-            .annotate(p=F("item__podcastepisode__program"))
             .filter(p__isnull=False)
             .exclude(p__in=exisiting_ids)
             .values("p")
@@ -80,6 +85,7 @@ class DiscoverGenerator(BaseJob):
 
     def run(self):
         logger.info("Discover data update start.")
+        local = settings.FILTER_LOCAL_ONLY_FOR_DISCOVER
         gallery_categories = [
             ItemCategory.Book,
             ItemCategory.Movie,
@@ -143,42 +149,49 @@ class DiscoverGenerator(BaseJob):
                     }
                 )
         trends.sort(key=lambda x: int(x["history"][0]["accounts"]), reverse=True)
-        collection_ids = (
+
+        collections = (
             Collection.objects.filter(visibility=0)
             .annotate(num=Count("interactions"))
             .filter(num__gte=MIN_MARKS)
             .order_by("-edited_time")
-            .values_list("pk", flat=True)[:40]
         )
-        tags = TagManager.popular_tags(days=14)[:40]
+        if local:
+            collections = collections.filter(local=True)
+        collection_ids = collections.values_list("pk", flat=True)[:40]
+
+        tags = TagManager.popular_tags(days=14, local_only=local)[:40]
         excluding_identities = Takahe.get_no_discover_identities()
+
+        reviews = (
+            Review.objects.filter(visibility=0)
+            .exclude(owner_id__in=excluding_identities)
+            .order_by("-created_time")
+        )
+        if local:
+            reviews = reviews.filter(local=True)
         post_ids = (
             set(
                 Takahe.get_popular_posts(
-                    28, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities
+                    28, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities, local
                 ).values_list("pk", flat=True)[:5]
             )
             | set(
                 Takahe.get_popular_posts(
-                    14, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities
+                    14, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities, local
                 ).values_list("pk", flat=True)[:5]
             )
             | set(
                 Takahe.get_popular_posts(
-                    7, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities
+                    7, settings.MIN_MARKS_FOR_DISCOVER, excluding_identities, local
                 ).values_list("pk", flat=True)[:10]
             )
             | set(
-                Takahe.get_popular_posts(1, 0, excluding_identities).values_list(
+                Takahe.get_popular_posts(1, 0, excluding_identities, local).values_list(
                     "pk", flat=True
                 )[:3]
             )
-            | set(
+            | set(reviews.values_list("posts", flat=True)[:5])
-                Review.objects.filter(visibility=0)
-                .exclude(owner_id__in=excluding_identities)
-                .order_by("-created_time")
-                .values_list("posts", flat=True)[:5]
-            )
         )
         cache.set("public_gallery", gallery_list, timeout=None)
         cache.set("trends_links", trends, timeout=None)

compose.yml

@@ -35,6 +35,7 @@ x-shared:
       NEODB_SEARCH_PEERS:
       NEODB_MIN_MARKS_FOR_DISCOVER:
       NEODB_FILTER_LANGUAGE_FOR_DISCOVER:
+      NEODB_FILTER_LOCAL_ONLY_FOR_DISCOVER:
       NEODB_SENTRY_DSN:
       TAKAHE_SENTRY_DSN:
       NEODB_SENTRY_SAMPLE_RATE:

docs/configuration.md

@@ -24,8 +24,9 @@ if you are doing debug or development:
  - `NEODB_SITE_INTRO`
  - `NEODB_SITE_HEAD`
  - `NEODB_SITE_DESCRIPTION`
- - `NEODB_PREFERRED_LANGUAGES` - preferred languages when importing titles from 3rd party sites like TMDB and Steam, comma-separated list of ISO-639-1 two-letter codes, `en,zh` by default.
+ - `NEODB_PREFERRED_LANGUAGES` - preferred languages when importing titles from 3rd party sites like TMDB and Steam, comma-separated list of ISO-639-1 two-letter codes, `en,zh` by default. It can includes languages with no UI translations yet, e.g. if set to `ja,en,zh`, NeoDB scraper will fetch catalog metadata in three languages if they are available from third party sites, Japanese users (= whose browser language set to ja-JP) will see English UI with Japanese metadata.
- - `NEODB_FILTER_LANGUAGE_FOR_DISCOVER` - when set to `True`, `/discover/` will only show items with languages match one of `NEODB_PREFERRED_LANGUAGES`. `False` by default.
+ - `NEODB_FILTER_LANGUAGE_FOR_DISCOVER` - `False` by default; when set to `True`, `/discover/` will only show items with languages match one of `NEODB_PREFERRED_LANGUAGES`.
+ - `NEODB_FILTER_LOCAL_ONLY_FOR_DISCOVER` - `False` by default; when set to `True`, only show items marked by local users rather than entire network in `/discover/`
  - `NEODB_SITE_LINKS` - a list of title and links to show in the footer, comma separated, e.g. `Feedback=https://discord.gg/8KweCuApaK,ToS=/pages/rules/`
  - `NEODB_INVITE_ONLY` - `False` by default, set to `True` to require invite code(generated by `neodb-manage invite --create`) to register
  - `NEODB_ENABLE_LOCAL_ONLY` - `False` by default, set to `True` to allow user to post marks as "local public"

journal/models/tag.py

@@ -146,17 +146,19 @@ class TagManager:
         return tags
 
     @staticmethod
-    def popular_tags(days: int = 30):
+    def popular_tags(days: int = 30, local_only: bool = False):
         t = timezone.now() - timedelta(days=days)
-        titles = (
+        tags = (
             TagMember.objects.filter(created_time__gt=t)
             .filter(parent__visibility=0)
             .annotate(title=F("parent__title"))
             .values("title")
             .annotate(total=Count("parent_id", distinct=True))
             .order_by("-total")
-            .values_list("title", flat=True)
         )
+        if local_only:
+            tags = tags.filter(local=True)
+        titles = tags.values_list("title", flat=True)
         return titles
 
     def get_item_tags(self, item: Item):

takahe/utils.py

@@ -837,11 +837,14 @@ class Takahe:
 
     @staticmethod
     def get_popular_posts(
-        days: int = 30, min_interaction: int = 1, exclude_identities: list[int] = []
+        days: int = 30,
+        min_interaction: int = 1,
+        exclude_identities: list[int] = [],
+        local_only=False,
     ):
         since = timezone.now() - timedelta(days=days)
         domains = Takahe.get_neodb_peers() + [settings.SITE_DOMAIN]
-        return (
+        qs = (
             Post.objects.exclude(state__in=["deleted", "deleted_fanned_out"])
             .exclude(author_id__in=exclude_identities)
             .filter(
@@ -853,6 +856,9 @@ class Takahe:
             .filter(num_interactions__gte=min_interaction)
             .order_by("-num_interactions", "-published")
         )
+        if local_only:
+            qs = qs.filter(local=True)
+        return qs
 
     @staticmethod
     def get_recent_posts(author_pk: int, viewer_pk: int | None = None):