takahe integration

.github/workflows/django.yml

@@ -1,4 +1,4 @@
-name: all tests
+name: tests
 
 on:
   push:
@@ -6,8 +6,7 @@ on:
     branches: [ "main" ]
 
 jobs:
-  build:
+  django:
-
     runs-on: ubuntu-latest
     services:
       redis:
@@ -15,20 +14,25 @@ jobs:
         ports:
           - 6379:6379
       db:
-        image: postgres:12.13-alpine
+        image: postgres
         env:
-          POSTGRES_USER: postgres
+          POSTGRES_USER: testuser
-          POSTGRES_PASSWORD: admin123
+          POSTGRES_PASSWORD: testpass
-          POSTGRES_DB: test
+          POSTGRES_DB: test_neodb
         ports:
           - 5432:5432
-        options: --mount type=tmpfs,destination=/var/lib/postgresql/data --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      db2:
-
+        image: postgres
+        env:
+          POSTGRES_USER: testuser
+          POSTGRES_PASSWORD: testpass
+          POSTGRES_DB: test_neodb_takahe
+        ports:
+          - 15432:5432
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ['3.10', '3.11']
+        python-version: ['3.11']
-
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}

boofilsic/settings.py

@@ -1,6 +1,11 @@
 import os
 
+# import django_stubs_ext
+
+# django_stubs_ext.monkeypatch()
+
 NEODB_VERSION = "0.8"
+DATABASE_ROUTERS = ["takahe.db_routes.TakaheRouter"]
 
 PROJECT_ROOT = os.path.abspath(os.path.dirname(__name__))
 
@@ -65,6 +70,7 @@ INSTALLED_APPS += [
     "journal.apps.JournalConfig",
     "social.apps.SocialConfig",
     "developer.apps.DeveloperConfig",
+    "takahe.apps.TakaheConfig",
     "legacy.apps.LegacyConfig",
 ]
 
@@ -110,6 +116,8 @@ TEMPLATES = [
 
 WSGI_APPLICATION = "boofilsic.wsgi.application"
 
+SESSION_COOKIE_NAME = "neodbsid"
+
 CACHES = {
     "default": {
         "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
@@ -131,7 +139,25 @@ DATABASES = {
             "client_encoding": "UTF8",
             # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT,
         },
-    }
+        "TEST": {
+            "DEPENDENCIES": ["takahe"],
+        },
+    },
+    "takahe": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": os.environ.get("TAKAHE_DB_NAME", "test_neodb_takahe"),
+        "USER": os.environ.get("TAKAHE_DB_USER", "testuser"),
+        "PASSWORD": os.environ.get("TAKAHE_DB_PASSWORD", "testpass"),
+        "HOST": os.environ.get("TAKAHE_DB_HOST", "127.0.0.1"),
+        "PORT": os.environ.get("TAKAHE_DB_PORT", 15432),
+        "OPTIONS": {
+            "client_encoding": "UTF8",
+            # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT,
+        },
+        "TEST": {
+            "DEPENDENCIES": [],
+        },
+    },
 }
 
 # Customized auth backend, glue OAuth2 and Django User model together
@@ -189,6 +215,8 @@ AUTH_USER_MODEL = "users.User"
 
 SILENCED_SYSTEM_CHECKS = [
     "admin.E404",  # Required by django-user-messages
+    "models.W035",  # Required by takahe: identical table name in different database
+    "fields.W344",  # Required by takahe: identical table name in different database
 ]
 
 MEDIA_URL = "/media/"
@@ -358,6 +386,7 @@ SEARCH_BACKEND = None
 if os.environ.get("NEODB_TYPESENSE_ENABLE", ""):
     SEARCH_BACKEND = "TYPESENSE"
 
+TYPESENSE_INDEX_NAME = "catalog"
 TYPESENSE_CONNECTION = {
     "api_key": os.environ.get("NEODB_TYPESENSE_KEY", "insecure"),
     "nodes": [
@@ -371,6 +400,7 @@ TYPESENSE_CONNECTION = {
 }
 
 
+DOWNLOADER_CACHE_TIMEOUT = 300
 DOWNLOADER_RETRIES = 3
 DOWNLOADER_SAVEDIR = None
 DISABLE_MODEL_SIGNAL = False  # disable index and social feeds during importing/etc

catalog/book/models.py

@@ -166,7 +166,7 @@ class Edition(Item):
         """add Work from resource.metadata['work'] if not yet"""
         links = resource.required_resources + resource.related_resources
         for w in links:
-            if w["model"] == "Work":
+            if w.get("model") == "Work":
                 work = Work.objects.filter(
                     primary_lookup_id_type=w["id_type"],
                     primary_lookup_id_value=w["id_value"],

catalog/common/__init__.py

@@ -24,6 +24,7 @@ __all__ = (
     "use_local_response",
     "RetryDownloader",
     "BasicDownloader",
+    "CachedDownloader",
     "ProxiedDownloader",
     "BasicImageDownloader",
     "ProxiedImageDownloader",

catalog/common/downloaders.py

@@ -10,6 +10,7 @@ from urllib.parse import quote
 import filetype
 import requests
 from django.conf import settings
+from django.core.cache import cache
 from lxml import html
 from PIL import Image
 from requests import Response
@@ -153,7 +154,6 @@ class BasicDownloader:
     def _download(self, url) -> Tuple[DownloaderResponse | MockResponse, int]:
         try:
             if not _mock_mode:
-                # TODO cache = get/set from redis
                 resp = requests.get(
                     url, headers=self.headers, timeout=self.get_timeout()
                 )
@@ -256,6 +256,19 @@ class RetryDownloader(BasicDownloader):
         raise DownloadError(self, "max out of retries")
 
 
+class CachedDownloader(BasicDownloader):
+    def download(self):
+        cache_key = "dl:" + self.url
+        resp = cache.get(cache_key)
+        if resp:
+            self.response_type = RESPONSE_OK
+        else:
+            resp = super().download()
+            if self.response_type == RESPONSE_OK:
+                cache.set(cache_key, resp, timeout=settings.DOWNLOADER_CACHE_TIMEOUT)
+        return resp
+
+
 class ImageDownloaderMixin:
     def __init__(self, url, referer=None):
         self.extention = None

catalog/common/models.py

@@ -13,7 +13,7 @@ from django.db import connection, models
 from django.utils import timezone
 from django.utils.baseconv import base62
 from django.utils.translation import gettext_lazy as _
-from ninja import Schema
+from ninja import Field, Schema
 from polymorphic.models import PolymorphicModel
 
 from catalog.common import jsondata
@@ -46,6 +46,7 @@ class SiteName(models.TextChoices):
     RSS = "rss", _("RSS")
     Discogs = "discogs", _("Discogs")
     AppleMusic = "apple_music", _("苹果音乐")
+    Fediverse = "fedi", _("联邦实例")
 
 
 class IdType(models.TextChoices):
@@ -90,6 +91,7 @@ class IdType(models.TextChoices):
     Bangumi = "bangumi", _("Bangumi")
     ApplePodcast = "apple_podcast", _("苹果播客")
     AppleMusic = "apple_music", _("苹果音乐")
+    Fediverse = "fedi", _("联邦实例")
 
 
 IdealIdTypes = [
@@ -225,6 +227,8 @@ class ExternalResourceSchema(Schema):
 
 
 class BaseSchema(Schema):
+    id: str = Field(alias="absolute_url")
+    type: str = Field(alias="ap_object_type")
     uuid: str
     url: str
     api_url: str
@@ -250,7 +254,7 @@ class Item(SoftDeleteMixin, PolymorphicModel):
     url_path = "item"  # subclass must specify this
     type = None  # subclass must specify this
     parent_class = None  # subclass may specify this to allow create child item
-    category: ItemCategory | None = None  # subclass must specify this
+    category: ItemCategory  # subclass must specify this
     demonstrative: "_StrOrPromise | None" = None  # subclass must specify this
     uid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True)
     title = models.CharField(_("标题"), max_length=1000, default="")
@@ -345,6 +349,25 @@ class Item(SoftDeleteMixin, PolymorphicModel):
     def parent_uuid(self):
         return self.parent_item.uuid if self.parent_item else None
 
+    @classmethod
+    def get_ap_object_type(cls):
+        return cls.__name__
+
+    @property
+    def ap_object_type(self):
+        return self.get_ap_object_type()
+
+    @property
+    def ap_object_ref(self):
+        o = {
+            "type": self.get_ap_object_type(),
+            "url": self.absolute_url,
+            "name": self.title,
+        }
+        if self.has_cover():
+            o["image"] = self.cover_image_url
+        return o
+
     def log_action(self, changes):
         LogEntry.objects.log_create(
             self, action=LogEntry.Action.UPDATE, changes=changes
@@ -561,10 +584,13 @@ class ExternalResource(models.Model):
     edited_time = models.DateTimeField(auto_now=True)
     required_resources = jsondata.ArrayField(
         models.CharField(), null=False, blank=False, default=list
-    )
+    )  # links required to generate Item from this resource, e.g. parent TVShow of TVSeason
     related_resources = jsondata.ArrayField(
         models.CharField(), null=False, blank=False, default=list
-    )
+    )  # links related to this resource which may be fetched later, e.g. sub TVSeason of TVShow
+    prematched_resources = jsondata.ArrayField(
+        models.CharField(), null=False, blank=False, default=list
+    )  # links to help match an existing Item from this resource
 
     class Meta:
         unique_together = [["id_type", "id_value"]]
@@ -585,13 +611,24 @@ class ExternalResource(models.Model):
         return SiteManager.get_site_cls_by_id_type(self.id_type)
 
     @property
-    def site_name(self):
+    def site_name(self) -> SiteName:
         try:
-            return self.get_site().SITE_NAME
+            site = self.get_site()
+            return site.SITE_NAME if site else SiteName.Unknown
         except:
             _logger.warning(f"Unknown site for {self}")
             return SiteName.Unknown
 
+    @property
+    def site_label(self):
+        if self.id_type == IdType.Fediverse:
+            from takahe.utils import Takahe
+
+            domain = self.id_value.split("://")[1].split("/")[0]
+            n = Takahe.get_node_name_for_domain(domain)
+            return n or domain
+        return self.site_name.label
+
     def update_content(self, resource_content):
         self.other_lookup_ids = resource_content.lookup_ids
         self.metadata = resource_content.metadata
@@ -615,7 +652,16 @@ class ExternalResource(models.Model):
         d = {k: v for k, v in d.items() if bool(v)}
         return d
 
-    def get_preferred_model(self) -> type[Item] | None:
+    def get_lookup_ids(self, default_model):
+        lookup_ids = self.get_all_lookup_ids()
+        model = self.get_item_model(default_model)
+        bt, bv = model.get_best_lookup_id(lookup_ids)
+        ids = [(t, v) for t, v in lookup_ids.items() if t and v and t != bt]
+        if bt and bv:
+            ids = [(bt, bv)] + ids
+        return ids
+
+    def get_item_model(self, default_model: type[Item]) -> type[Item]:
         model = self.metadata.get("preferred_model")
         if model:
             m = ContentType.objects.filter(
@@ -625,7 +671,7 @@ class ExternalResource(models.Model):
                 return cast(Item, m).model_class()
             else:
                 raise ValueError(f"preferred model {model} does not exist")
-        return None
+        return default_model
 
 
 _CONTENT_TYPE_LIST = None

catalog/common/sites.py

@@ -39,7 +39,7 @@ class AbstractSite:
     Abstract class to represent a site
     """
 
-    SITE_NAME: SiteName | None = None
+    SITE_NAME: SiteName
     ID_TYPE: IdType | None = None
     WIKI_PROPERTY_ID: str | None = "P0undefined0"
     DEFAULT_MODEL: Type[Item] | None = None
@@ -104,18 +104,29 @@ class AbstractSite:
         return content.xpath(query)[0].strip()
 
     @classmethod
-    def get_model_for_resource(cls, resource):
+    def match_existing_item_for_resource(
-        model = resource.get_preferred_model()
+        cls, resource: ExternalResource
-        return model or cls.DEFAULT_MODEL
+    ) -> Item | None:
+        """
+        try match an existing Item for a given ExternalResource
 
-    @classmethod
+        order of matching:
-    def match_existing_item_for_resource(cls, resource) -> Item | None:
+        1. look for other ExternalResource by url in prematched_resources, if found, return the item
-        model = cls.get_model_for_resource(resource)
+        2. look for Item by primary_lookup_id_type and primary_lookup_id_value
+
+        """
+        for resource_link in resource.prematched_resources:  # type: ignore
+            url = resource_link.get("url")
+            if url:
+                matched_resource = ExternalResource.objects.filter(url=url).first()
+                if matched_resource and matched_resource.item:
+                    return matched_resource.item
+        model = resource.get_item_model(cls.DEFAULT_MODEL)
         if not model:
             return None
-        t, v = model.get_best_lookup_id(resource.get_all_lookup_ids())
+        ids = resource.get_lookup_ids(cls.DEFAULT_MODEL)
-        matched = None
+        for t, v in ids:
-        if t is not None:
+            matched = None
             matched = model.objects.filter(
                 primary_lookup_id_type=t,
                 primary_lookup_id_value=v,
@@ -143,14 +154,15 @@ class AbstractSite:
                 matched.primary_lookup_id_type = t
                 matched.primary_lookup_id_value = v
                 matched.save()
-        return matched
+            if matched:
+                return matched
 
     @classmethod
     def match_or_create_item_for_resource(cls, resource):
         previous_item = resource.item
         resource.item = cls.match_existing_item_for_resource(resource) or previous_item
         if resource.item is None:
-            model = cls.get_model_for_resource(resource)
+            model = resource.get_item_model(cls.DEFAULT_MODEL)
             if not model:
                 return None
             t, v = model.get_best_lookup_id(resource.get_all_lookup_ids())
@@ -243,7 +255,7 @@ class AbstractSite:
                         )
                     else:
                         _logger.error(f"unable to get site for {linked_url}")
-            if p.related_resources:
+            if p.related_resources or p.prematched_resources:
                 django_rq.get_queue("crawl").enqueue(crawl_related_resources_task, p.pk)
             if p.item:
                 p.item.update_linked_items_from_external_resource(p)
@@ -318,7 +330,7 @@ def crawl_related_resources_task(resource_pk):
     if not resource:
         _logger.warn(f"crawl resource not found {resource_pk}")
         return
-    links = resource.related_resources
+    links = (resource.related_resources or []) + (resource.prematched_resources or [])  # type: ignore
     for w in links:  # type: ignore
         try:
             item = None

catalog/common/utils.py

@@ -36,4 +36,4 @@ def piece_cover_path(item, filename):
         + "."
         + filename.split(".")[-1]
     )
-    return f"user/{item.owner_id}/{fn}"
+    return f"user/{item.owner_id or '_'}/{fn}"

catalog/management/commands/cat.py

@@ -31,10 +31,17 @@ class Command(BaseCommand):
         self.stdout.write(f"Fetching from {site}")
         if options["save"]:
             resource = site.get_resource_ready(ignore_existing_content=options["force"])
-            pprint.pp(resource.metadata)
+            if resource:
-            pprint.pp(site.get_item())
+                pprint.pp(resource.metadata)
-            pprint.pp(site.get_item().cover)
+            else:
-            pprint.pp(site.get_item().metadata)
+                self.stdout.write(self.style.ERROR(f"Unable to get resource for {url}"))
+            item = site.get_item()
+            if item:
+                pprint.pp(item.cover)
+                pprint.pp(item.metadata)
+                pprint.pp(item.absolute_url)
+            else:
+                self.stdout.write(self.style.ERROR(f"Unable to get item for {url}"))
         else:
             resource = site.scrape()
             pprint.pp(resource.metadata)

catalog/management/commands/crawl.py

@@ -29,16 +29,19 @@ class Command(BaseCommand):
             logger.info(f"Navigating {url}")
             content = ProxiedDownloader(url).download().html()
             urls = content.xpath("//a/@href")
-            for _u in urls:
+            for _u in urls:  # type:ignore
                 u = urljoin(url, _u)
                 if u not in history and u not in queue:
                     if len([p for p in item_patterns if re.match(p, u)]) > 0:
                         site = SiteManager.get_site_by_url(u)
-                        u = site.url
+                        if site:
-                        if u not in history:
+                            u = site.url
-                            history.append(u)
+                            if u not in history:
-                            logger.info(f"Fetching {u}")
+                                history.append(u)
-                            site.get_resource_ready()
+                                logger.info(f"Fetching {u}")
+                                site.get_resource_ready()
+                        else:
+                            logger.warning(f"unable to parse {u}")
                     elif pattern and u.find(pattern) >= 0:
                         queue.append(u)
         logger.info("Crawl finished.")

catalog/management/commands/discover.py

@@ -7,7 +7,7 @@ from django.utils import timezone
 from loguru import logger
 
 from catalog.models import *
-from journal.models import Comment, ShelfMember, query_item_category
+from journal.models import Comment, ShelfMember, q_item_in_category
 
 MAX_ITEMS_PER_PERIOD = 12
 MIN_MARKS = 2
@@ -28,7 +28,7 @@ class Command(BaseCommand):
     def get_popular_marked_item_ids(self, category, days, exisiting_ids):
         item_ids = [
             m["item_id"]
-            for m in ShelfMember.objects.filter(query_item_category(category))
+            for m in ShelfMember.objects.filter(q_item_in_category(category))
             .filter(created_time__gt=timezone.now() - timedelta(days=days))
             .exclude(item_id__in=exisiting_ids)
             .values("item_id")
@@ -40,7 +40,7 @@ class Command(BaseCommand):
 
     def get_popular_commented_podcast_ids(self, days, exisiting_ids):
         return list(
-            Comment.objects.filter(query_item_category(ItemCategory.Podcast))
+            Comment.objects.filter(q_item_in_category(ItemCategory.Podcast))
             .filter(created_time__gt=timezone.now() - timedelta(days=days))
             .annotate(p=F("item__podcastepisode__program"))
             .filter(p__isnull=False)

catalog/management/commands/index.py

@@ -1,6 +1,7 @@
 import pprint
 from datetime import timedelta
 from time import sleep
+from typing import TYPE_CHECKING
 
 from django.conf import settings
 from django.core.management.base import BaseCommand
@@ -8,7 +9,8 @@ from django.core.paginator import Paginator
 from django.utils import timezone
 from tqdm import tqdm
 
-from catalog.models import *
+from catalog.models import Item
+from catalog.search.typesense import Indexer
 
 BATCH_SIZE = 1000
 

catalog/migrations/0011_alter_externalresource_id_type_and_more.py

@@ -0,0 +1,117 @@
+# Generated by Django 4.2.3 on 2023-08-06 02:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("catalog", "0011_remove_item_last_editor"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="externalresource",
+            name="id_type",
+            field=models.CharField(
+                choices=[
+                    ("wikidata", "维基数据"),
+                    ("isbn10", "ISBN10"),
+                    ("isbn", "ISBN"),
+                    ("asin", "ASIN"),
+                    ("issn", "ISSN"),
+                    ("cubn", "统一书号"),
+                    ("isrc", "ISRC"),
+                    ("gtin", "GTIN UPC EAN码"),
+                    ("rss", "RSS Feed URL"),
+                    ("imdb", "IMDb"),
+                    ("tmdb_tv", "TMDB剧集"),
+                    ("tmdb_tvseason", "TMDB剧集"),
+                    ("tmdb_tvepisode", "TMDB剧集"),
+                    ("tmdb_movie", "TMDB电影"),
+                    ("goodreads", "Goodreads"),
+                    ("goodreads_work", "Goodreads著作"),
+                    ("googlebooks", "谷歌图书"),
+                    ("doubanbook", "豆瓣读书"),
+                    ("doubanbook_work", "豆瓣读书著作"),
+                    ("doubanmovie", "豆瓣电影"),
+                    ("doubanmusic", "豆瓣音乐"),
+                    ("doubangame", "豆瓣游戏"),
+                    ("doubandrama", "豆瓣舞台剧"),
+                    ("doubandrama_version", "豆瓣舞台剧版本"),
+                    ("bookstw", "博客来图书"),
+                    ("bandcamp", "Bandcamp"),
+                    ("spotify_album", "Spotify专辑"),
+                    ("spotify_show", "Spotify播客"),
+                    ("discogs_release", "Discogs Release"),
+                    ("discogs_master", "Discogs Master"),
+                    ("musicbrainz", "MusicBrainz ID"),
+                    ("doubanbook_author", "豆瓣读书作者"),
+                    ("doubanmovie_celebrity", "豆瓣电影影人"),
+                    ("goodreads_author", "Goodreads作者"),
+                    ("spotify_artist", "Spotify艺术家"),
+                    ("tmdb_person", "TMDB影人"),
+                    ("igdb", "IGDB游戏"),
+                    ("steam", "Steam游戏"),
+                    ("bangumi", "Bangumi"),
+                    ("apple_podcast", "苹果播客"),
+                    ("apple_music", "苹果音乐"),
+                    ("fedi", "联邦实例"),
+                ],
+                max_length=50,
+                verbose_name="IdType of the source site",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="itemlookupid",
+            name="id_type",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("wikidata", "维基数据"),
+                    ("isbn10", "ISBN10"),
+                    ("isbn", "ISBN"),
+                    ("asin", "ASIN"),
+                    ("issn", "ISSN"),
+                    ("cubn", "统一书号"),
+                    ("isrc", "ISRC"),
+                    ("gtin", "GTIN UPC EAN码"),
+                    ("rss", "RSS Feed URL"),
+                    ("imdb", "IMDb"),
+                    ("tmdb_tv", "TMDB剧集"),
+                    ("tmdb_tvseason", "TMDB剧集"),
+                    ("tmdb_tvepisode", "TMDB剧集"),
+                    ("tmdb_movie", "TMDB电影"),
+                    ("goodreads", "Goodreads"),
+                    ("goodreads_work", "Goodreads著作"),
+                    ("googlebooks", "谷歌图书"),
+                    ("doubanbook", "豆瓣读书"),
+                    ("doubanbook_work", "豆瓣读书著作"),
+                    ("doubanmovie", "豆瓣电影"),
+                    ("doubanmusic", "豆瓣音乐"),
+                    ("doubangame", "豆瓣游戏"),
+                    ("doubandrama", "豆瓣舞台剧"),
+                    ("doubandrama_version", "豆瓣舞台剧版本"),
+                    ("bookstw", "博客来图书"),
+                    ("bandcamp", "Bandcamp"),
+                    ("spotify_album", "Spotify专辑"),
+                    ("spotify_show", "Spotify播客"),
+                    ("discogs_release", "Discogs Release"),
+                    ("discogs_master", "Discogs Master"),
+                    ("musicbrainz", "MusicBrainz ID"),
+                    ("doubanbook_author", "豆瓣读书作者"),
+                    ("doubanmovie_celebrity", "豆瓣电影影人"),
+                    ("goodreads_author", "Goodreads作者"),
+                    ("spotify_artist", "Spotify艺术家"),
+                    ("tmdb_person", "TMDB影人"),
+                    ("igdb", "IGDB游戏"),
+                    ("steam", "Steam游戏"),
+                    ("bangumi", "Bangumi"),
+                    ("apple_podcast", "苹果播客"),
+                    ("apple_music", "苹果音乐"),
+                    ("fedi", "联邦实例"),
+                ],
+                max_length=50,
+                verbose_name="源网站",
+            ),
+        ),
+    ]

catalog/search/external.py

@@ -23,7 +23,8 @@ class SearchResultItem:
             "all": [
                 {
                     "url": source_url,
-                    "site_name": {"label": source_site, "value": source_site},
+                    "site_name": source_site,
+                    "site_label": source_site,
                 }
             ]
         }

catalog/search/typesense.py

@@ -14,7 +14,6 @@ from typesense.exceptions import ObjectNotFound
 
 from catalog.models import Item
 
-INDEX_NAME = "catalog"
 SEARCHABLE_ATTRIBUTES = [
     "title",
     "orig_title",
@@ -125,7 +124,7 @@ class Indexer:
     def instance(cls) -> Collection:
         if cls._instance is None:
             cls._instance = typesense.Client(settings.TYPESENSE_CONNECTION).collections[
-                INDEX_NAME
+                settings.TYPESENSE_INDEX_NAME
             ]
         return cls._instance  # type: ignore
 
@@ -178,7 +177,7 @@ class Indexer:
             {"name": ".*", "optional": True, "locale": "zh", "type": "auto"},
         ]
         return {
-            "name": INDEX_NAME,
+            "name": settings.TYPESENSE_INDEX_NAME,
             "fields": fields,
             # "default_sorting_field": "rating_count",
         }

catalog/search/views.py

@@ -130,9 +130,14 @@ def search(request):
         )
 
     if keywords.find("://") > 0:
+        host = keywords.split("://")[1].split("/")[0]
+        if host == settings.SITE_INFO["site_domain"]:
+            return redirect(keywords)
         site = SiteManager.get_site_by_url(keywords)
         if site:
             return fetch(request, keywords, False, site)
+        if request.GET.get("r"):
+            return redirect(keywords)
 
     items, num_pages, _, dup_items = query_index(keywords, categories, tag, p)
     return render(

catalog/sites/__init__.py

@@ -9,13 +9,14 @@ from .douban_drama import DoubanDrama
 from .douban_game import DoubanGame
 from .douban_movie import DoubanMovie
 from .douban_music import DoubanMusic
+from .fedi import FediverseInstance
 from .goodreads import Goodreads
 from .google_books import GoogleBooks
 from .igdb import IGDB
 from .imdb import IMDB
-
-# from .apple_podcast import ApplePodcast
 from .rss import RSS
 from .spotify import Spotify
 from .steam import Steam
 from .tmdb import TMDB_Movie
+
+# from .apple_podcast import ApplePodcast

catalog/sites/fedi.py

@@ -0,0 +1,101 @@
+import re
+
+from django.core.validators import URLValidator
+from loguru import logger
+
+from catalog.common import *
+from catalog.models import *
+
+
+@SiteManager.register
+class FediverseInstance(AbstractSite):
+    SITE_NAME = SiteName.Fediverse
+    ID_TYPE = IdType.Fediverse
+    URL_PATTERNS = []
+    WIKI_PROPERTY_ID = ""
+    DEFAULT_MODEL = None
+    id_type_mapping = {
+        "isbn": IdType.ISBN,
+        "imdb": IdType.IMDB,
+        "barcode": IdType.GTIN,
+    }
+    supported_types = {
+        "Book": Edition,
+        "Movie": Movie,
+        "TVShow": TVShow,
+        "TVSeason": TVSeason,
+        "TVEpisode": TVEpisode,
+        "Album": Album,
+        "Game": Game,
+        "Podcast": Podcast,
+        "Performance": Performance,
+        "PerformanceProduction": PerformanceProduction,
+    }
+    request_header = {"User-Agent": "NeoDB/0.5", "Accept": "application/activity+json"}
+
+    @classmethod
+    def id_to_url(cls, id_value):
+        return id_value
+
+    @classmethod
+    def url_to_id(cls, url: str):
+        u = url.split("://", 1)[1].split("/", 1)
+        return "https://" + u[0].lower() + "/" + u[1]
+
+    @classmethod
+    def validate_url_fallback(cls, url):
+        val = URLValidator()
+        try:
+            val(url)
+            if (
+                url.split("://", 1)[1].split("/", 1)[0].lower()
+                == settings.SITE_INFO["site_domain"]
+            ):
+                # disallow local instance URLs
+                return False
+            return cls.get_json_from_url(url) is not None
+        except Exception:
+            return False
+
+    @classmethod
+    def get_json_from_url(cls, url):
+        j = CachedDownloader(url, headers=cls.request_header).download().json()
+        if j.get("type") not in cls.supported_types.keys():
+            raise ValueError("Not a supported format or type")
+        if j.get("id") != url:
+            logger.warning(f"ID mismatch: {j.get('id')} != {url}")
+        return j
+
+    def scrape(self):
+        data = self.get_json_from_url(self.url)
+        img_url = data.get("cover_image_url")
+        raw_img, img_ext = (
+            BasicImageDownloader.download_image(img_url, None, headers={})
+            if img_url
+            else (None, None)
+        )
+        ids = {}
+        data["preferred_model"] = data.get("type")
+        data["prematched_resources"] = []
+        for ext in data.get("external_resources", []):
+            site = SiteManager.get_site_by_url(ext.get("url"))
+            if site and site.ID_TYPE != self.ID_TYPE:
+                ids[site.ID_TYPE] = site.id_value
+                data["prematched_resources"].append(
+                    {
+                        "model": data["preferred_model"],
+                        "id_type": site.ID_TYPE,
+                        "id_value": site.id_value,
+                        "url": site.url,
+                    }
+                )
+        # for k, v in self.id_type_mapping.items():
+        #     if data.get(k):
+        #         ids[v] = data.get(k)
+        d = ResourceContent(
+            metadata=data,
+            cover_image=raw_img,
+            cover_image_extention=img_ext,
+            lookup_ids=ids,
+        )
+        return d

catalog/sites/rss.py

@@ -33,7 +33,8 @@ class RSS(AbstractSite):
     def parse_feed_from_url(url):
         if not url:
             return None
-        feed = cache.get(url)
+        cache_key = f"rss:{url}"
+        feed = cache.get(cache_key)
         if feed:
             return feed
         if get_mock_mode():
@@ -50,7 +51,7 @@ class RSS(AbstractSite):
                     feed,
                     open(settings.DOWNLOADER_SAVEDIR + "/" + get_mock_file(url), "wb"),
                 )
-        cache.set(url, feed, timeout=300)
+        cache.set(cache_key, feed, timeout=settings.DOWNLOADER_CACHE_TIMEOUT)
         return feed
 
     @classmethod

catalog/templates/_item_card.html

@@ -7,7 +7,7 @@
         {% if not hide_category %}<span class="category">[{{ item.category.label }}]</span>{% endif %}
         <span class="site-list">
           {% for res in item.external_resources.all %}
-            <a href="{{ res.url }}" class="{{ res.site_name }}">{{ res.site_name.label }}</a>
+            <a href="{{ res.url }}" class="{{ res.site_name }}">{{ res.site_label }}</a>
           {% endfor %}
         </span>
       </small>

catalog/templates/_item_card_metadata_base.html

@@ -15,7 +15,7 @@
       {% if not hide_category %}<span class="category">[{{ item.category.label }}]</span>{% endif %}
       <span class="site-list">
         {% for res in item.external_resources.all %}
-          <a href="{{ res.url }}" class="{{ res.site_name.value }}">{{ res.site_name.label }}</a>
+          <a href="{{ res.url }}" class="{{ res.site_name }}">{{ res.site_label }}</a>
         {% endfor %}
       </span>
     </small>

catalog/templates/_item_comments.html

@@ -53,7 +53,7 @@
         <span>
           <a target="_blank"
              rel="noopener"
-             {% if comment.metadata.shared_link %} href="{{ comment.metadata.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if comment.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
+             {% if comment.shared_link %} href="{{ comment.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if comment.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
         </span>
       </span>
       <span>

catalog/templates/_item_comments_by_episode.html

@@ -58,7 +58,7 @@
           <span>
             <a target="_blank"
                rel="noopener"
-               {% if comment.metadata.shared_link %} href="{{ comment.metadata.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if comment.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
+               {% if comment.shared_link %} href="{{ comment.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if comment.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
           </span>
         </span>
         <span>

catalog/templates/_item_reviews.html

@@ -18,7 +18,7 @@
         <span>
           <a target="_blank"
              rel="noopener"
-             {% if review.metadata.shared_link %} href="{{ review.metadata.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if review.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
+             {% if review.shared_link %} href="{{ review.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if review.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
         </span>
       </span>
       <span>

catalog/templates/_item_user_pieces.html

@@ -66,7 +66,7 @@
       <span>
         <a target="_blank"
            rel="noopener"
-           {% if mark.comment.metadata.shared_link %} href="{{ mark.comment.metadata.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if mark.comment.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
+           {% if mark.comment.shared_link %} href="{{ mark.comment.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if mark.comment.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
       </span>
       {% comment %} <span class="timestamp">{{ mark.comment.created_time|date }}</span> {% endcomment %}
     </span>
@@ -89,7 +89,7 @@
       <span>
         <a target="_blank"
            rel="noopener"
-           {% if comment.metadata.shared_link %} href="{{ comment.metadata.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if comment.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
+           {% if comment.shared_link %} href="{{ comment.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if comment.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
       </span>
       {% comment %} <span class="timestamp">{{ comment.created_time|date }}</span> {% endcomment %}
     </span>
@@ -127,7 +127,7 @@
       <span>
         <a target="_blank"
            rel="noopener"
-           {% if mark.review.metadata.shared_link %} href="{{ mark.review.metadata.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if mark.review.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
+           {% if mark.review.shared_link %} href="{{ mark.review.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if mark.review.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
       </span>
       <span class="timestamp">{{ mark.review.created_time|date }}</span>
     </span>

catalog/templates/_sidebar_edit.html

@@ -52,7 +52,7 @@
     {% for res in item.external_resources.all %}
       <details>
         <summary>
-          {% trans '源网站' %}: <a href="{{ res.url }}">{{ res.site_name.label }}</a>
+          {% trans '源网站' %}: <a href="{{ res.url }}">{{ res.site_label }}</a>
         </summary>
         <div class="grid">
           <form method="post"

catalog/templates/item_base.html

@@ -43,7 +43,7 @@
         </h1>
         <span class="site-list">
           {% for res in item.external_resources.all %}
-            <a href="{{ res.url }}" class="{{ res.site_name }}">{{ res.site_name.label }}</a>
+            <a href="{{ res.url }}" class="{{ res.site_name }}">{{ res.site_label }}</a>
           {% endfor %}
         </span>
       </div>

catalog/templates/item_mark_list.html

@@ -43,7 +43,7 @@
                 <span>
                   <a target="_blank"
                      rel="noopener"
-                     {% if mark.metadata.shared_link %} href="{{ mark.metadata.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if mark.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
+                     {% if mark.shared_link %} href="{{ mark.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if mark.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
                 </span>
                 <span class="timestamp">{{ mark.created_time|date }}</span>
               </div>

catalog/templates/item_review_list.html

@@ -31,7 +31,7 @@
               <span>
                 <a target="_blank"
                    rel="noopener"
-                   {% if review.metadata.shared_link %} href="{{ review.metadata.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if review.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
+                   {% if review.shared_link %} href="{{ review.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if review.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
               </span>
               <span>
                 {% liked_piece review as liked %}

catalog/urls.py

@@ -129,8 +129,9 @@ urlpatterns = [
         mark_list,
         name="mark_list",
     ),
-    path("search/", search, name="search"),
+    path("search", search, name="search"),
-    path("search/external/", external_search, name="external_search"),
+    path("search/", search, name="search_legacy"),
+    path("search/external", external_search, name="external_search"),
     path("fetch_refresh/<str:job_id>", fetch_refresh, name="fetch_refresh"),
     path("refetch", refetch, name="refetch"),
     path("unlink", unlink, name="unlink"),

catalog/views.py

@@ -19,9 +19,9 @@ from journal.models import (
     ShelfMember,
     ShelfType,
     ShelfTypeNames,
-    query_following,
+    q_item_in_category,
-    query_item_category,
+    q_piece_in_home_feed_of_user,
-    query_visible,
+    q_piece_visible_to_user,
 )
 
 from .forms import *
@@ -74,6 +74,8 @@ def retrieve(request, item_path, item_uuid):
     item_url = f"/{item_path}/{item_uuid}"
     if item.url != item_url:
         return redirect(item.url)
+    if request.headers.get("Accept", "").endswith("json"):
+        return redirect(item.api_url)
     skipcheck = request.GET.get("skipcheck", False) and request.user.is_authenticated
     if not skipcheck and item.merged_to_item:
         return redirect(item.merged_to_item.url)
@@ -91,16 +93,16 @@ def retrieve(request, item_path, item_uuid):
     child_item_comments = []
     shelf_types = [(n[1], n[2]) for n in iter(ShelfTypeNames) if n[0] == item.category]
     if request.user.is_authenticated:
-        visible = query_visible(request.user)
+        visible = q_piece_visible_to_user(request.user)
-        mark = Mark(request.user, item)
+        mark = Mark(request.user.identity, item)
         child_item_comments = Comment.objects.filter(
-            owner=request.user, item__in=item.child_items.all()
+            owner=request.user.identity, item__in=item.child_items.all()
         )
         review = mark.review
-        my_collections = item.collections.all().filter(owner=request.user)
+        my_collections = item.collections.all().filter(owner=request.user.identity)
         collection_list = (
             item.collections.all()
-            .exclude(owner=request.user)
+            .exclude(owner=request.user.identity)
             .filter(visible)
             .annotate(like_counts=Count("likes"))
             .order_by("-like_counts")
@@ -145,9 +147,9 @@ def mark_list(request, item_path, item_uuid, following_only=False):
         raise Http404()
     queryset = ShelfMember.objects.filter(item=item).order_by("-created_time")
     if following_only:
-        queryset = queryset.filter(query_following(request.user))
+        queryset = queryset.filter(q_piece_in_home_feed_of_user(request.user))
     else:
-        queryset = queryset.filter(query_visible(request.user))
+        queryset = queryset.filter(q_piece_visible_to_user(request.user))
     paginator = Paginator(queryset, NUM_REVIEWS_ON_LIST_PAGE)
     page_number = request.GET.get("page", default=1)
     marks = paginator.get_page(page_number)
@@ -169,7 +171,7 @@ def review_list(request, item_path, item_uuid):
     if not item:
         raise Http404()
     queryset = Review.objects.filter(item=item).order_by("-created_time")
-    queryset = queryset.filter(query_visible(request.user))
+    queryset = queryset.filter(q_piece_visible_to_user(request.user))
     paginator = Paginator(queryset, NUM_REVIEWS_ON_LIST_PAGE)
     page_number = request.GET.get("page", default=1)
     reviews = paginator.get_page(page_number)
@@ -192,7 +194,7 @@ def comments(request, item_path, item_uuid):
         raise Http404()
     ids = item.child_item_ids + [item.id]
     queryset = Comment.objects.filter(item_id__in=ids).order_by("-created_time")
-    queryset = queryset.filter(query_visible(request.user))
+    queryset = queryset.filter(q_piece_visible_to_user(request.user))
     before_time = request.GET.get("last")
     if before_time:
         queryset = queryset.filter(created_time__lte=before_time)
@@ -218,7 +220,7 @@ def comments_by_episode(request, item_path, item_uuid):
     else:
         ids = item.child_item_ids
     queryset = Comment.objects.filter(item_id__in=ids).order_by("-created_time")
-    queryset = queryset.filter(query_visible(request.user))
+    queryset = queryset.filter(q_piece_visible_to_user(request.user))
     before_time = request.GET.get("last")
     if before_time:
         queryset = queryset.filter(created_time__lte=before_time)
@@ -240,7 +242,7 @@ def reviews(request, item_path, item_uuid):
         raise Http404()
     ids = item.child_item_ids + [item.id]
     queryset = Review.objects.filter(item_id__in=ids).order_by("-created_time")
-    queryset = queryset.filter(query_visible(request.user))
+    queryset = queryset.filter(q_piece_visible_to_user(request.user))
     before_time = request.GET.get("last")
     if before_time:
         queryset = queryset.filter(created_time__lte=before_time)

common/static/scss/_sitelabel.scss

@@ -71,6 +71,12 @@
     font-weight: lighter;
   }
 
+  .fedi {
+    background: var(--pico-primary);
+    color: white;
+    font-weight: lighter;
+  }
+
   .tmdb {
     background: linear-gradient(90deg, #91CCA3, #1FB4E2);
     color: white;

common/templates/_sidebar.html

@@ -51,7 +51,7 @@
                        target="_blank"
                        rel="noopener"
                        onclick="window.open(this.href); return false;">
-                      <span class="handler">@{{ user.handler }}</span>
+                      <span class="handler">{{ user.handler }}</span>
                     </a>
                   </div>
                 </hgroup>

common/templatetags/mastodon.py

@@ -3,6 +3,8 @@ from django.conf import settings
 from django.template.defaultfilters import stringfilter
 from django.utils.translation import gettext_lazy as _
 
+from users.models import APIdentity, User
+
 register = template.Library()
 
 
@@ -13,9 +15,10 @@ def mastodon(domain):
 
 
 @register.simple_tag(takes_context=True)
-def current_user_relationship(context, user):
+def current_user_relationship(context, user: "User"):
     current_user = context["request"].user
     r = {
+        "requesting": False,
         "following": False,
         "unfollowable": False,
         "muting": False,
@@ -24,21 +27,23 @@ def current_user_relationship(context, user):
         "status": "",
     }
     if current_user and current_user.is_authenticated and current_user != user:
-        if current_user.is_blocking(user) or user.is_blocking(current_user):
+        current_identity = context["request"].user.identity
+        target_identity = user.identity
+        if current_identity.is_blocking(
+            target_identity
+        ) or current_identity.is_blocked_by(target_identity):
             r["rejecting"] = True
         else:
-            r["muting"] = current_user.is_muting(user)
+            r["muting"] = current_identity.is_muting(target_identity)
-            if user in current_user.local_muting.all():
+            r["unmutable"] = r["muting"]
-                r["unmutable"] = current_user
+            r["following"] = current_identity.is_following(target_identity)
-            if current_user.is_following(user):
+            r["unfollowable"] = r["following"]
-                r["following"] = True
+            if r["following"]:
-                if user in current_user.local_following.all():
+                if current_identity.is_followed_by(target_identity):
-                    r["unfollowable"] = True
-                if current_user.is_followed_by(user):
                     r["status"] = _("互相关注")
                 else:
                     r["status"] = _("已关注")
             else:
-                if current_user.is_followed_by(user):
+                if current_identity.is_followed_by(target_identity):
                     r["status"] = _("被ta关注")
     return r

common/urls.py

@@ -1,4 +1,4 @@
-from django.urls import path
+from django.urls import path, re_path
 
 from .views import *
 
@@ -7,4 +7,5 @@ urlpatterns = [
     path("", home),
     path("home/", home, name="home"),
     path("me/", me, name="me"),
+    re_path("^~neodb~(?P<uri>.+)", ap_redirect),
 ]

common/utils.py

@@ -1,9 +1,22 @@
 import uuid
+from typing import TYPE_CHECKING
 
-from django.http import Http404
+from django.http import Http404, HttpRequest
 from django.utils import timezone
 from django.utils.baseconv import base62
 
+if TYPE_CHECKING:
+    from users.models import APIdentity, User
+
+
+class AuthedHttpRequest(HttpRequest):
+    """
+    A subclass of HttpRequest for type-checking only
+    """
+
+    user: "User"
+    target_identity: "APIdentity"
+
 
 class PageLinksGenerator:
     # TODO inherit django paginator

common/views.py

@@ -6,7 +6,7 @@ from django.urls import reverse
 
 @login_required
 def me(request):
-    return redirect(request.user.url)
+    return redirect(request.user.identity.url)
 
 
 def home(request):
@@ -22,6 +22,10 @@ def home(request):
         return redirect(reverse("catalog:discover"))
 
 
+def ap_redirect(request, uri):
+    return redirect(uri)
+
+
 def error_400(request, exception=None):
     return render(
         request,

doc/install.md

@@ -33,8 +33,8 @@ Install PostgreSQL, Redis and Python (3.10 or above) if not yet
 ### 1.1 Database
 Setup database
 ```
-CREATE DATABASE neodb ENCODING 'UTF8' LC_COLLATE='en_US.UTF-8' LC_CTYPE='en_US.UTF-8' TEMPLATE template0;
 CREATE ROLE neodb with LOGIN ENCRYPTED PASSWORD 'abadface';
+CREATE DATABASE neodb ENCODING 'UTF8' LC_COLLATE='en_US.UTF-8' LC_CTYPE='en_US.UTF-8' TEMPLATE template0;
 GRANT ALL ON DATABASE neodb TO neodb;
 ```
 

journal/api.py

@@ -10,8 +10,9 @@ from oauth2_provider.decorators import protected_resource
 
 from catalog.common.models import *
 from common.api import *
+from mastodon.api import share_review
 
-from .models import *
+from .models import Mark, Review, ShelfType, TagManager, q_item_in_category
 
 
 class MarkSchema(Schema):
@@ -84,9 +85,9 @@ def mark_item(request, item_uuid: str, mark: MarkInSchema):
     item = Item.get_by_url(item_uuid)
     if not item:
         return 404, {"message": "Item not found"}
-    m = Mark(request.user, item)
+    m = Mark(request.user.identity, item)
     try:
-        TagManager.tag_item_by_user(item, request.user, mark.tags, mark.visibility)
+        TagManager.tag_item(item, request.user, mark.tags, mark.visibility)
         m.update(
             mark.shelf_type,
             mark.comment_text,
@@ -114,7 +115,7 @@ def delete_mark(request, item_uuid: str):
     m = Mark(request.user, item)
     m.delete()
     # skip tag deletion for now to be consistent with web behavior
-    # TagManager.tag_item_by_user(item, request.user, [], 0)
+    # TagManager.tag_item(item, request.user, [], 0)
     return 200, {"message": "OK"}
 
 
@@ -144,9 +145,9 @@ def list_reviews(request, category: AvailableItemCategory | None = None):
 
     `category` is optional, reviews for all categories will be returned if not specified.
     """
-    queryset = Review.objects.filter(owner=request.user)
+    queryset = Review.objects.filter(owner=request.user.identity)
     if category:
-        queryset = queryset.filter(query_item_category(category))
+        queryset = queryset.filter(q_item_in_category(category))
     return queryset.prefetch_related("item")
 
 
@@ -161,7 +162,7 @@ def get_review_by_item(request, item_uuid: str):
     item = Item.get_by_url(item_uuid)
     if not item:
         return 404, {"message": "Item not found"}
-    review = Review.objects.filter(owner=request.user, item=item).first()
+    review = Review.objects.filter(owner=request.user.identity, item=item).first()
     if not review:
         return 404, {"message": "Review not found"}
     return review
@@ -182,15 +183,17 @@ def review_item(request, item_uuid: str, review: ReviewInSchema):
     item = Item.get_by_url(item_uuid)
     if not item:
         return 404, {"message": "Item not found"}
-    Review.review_item_by_user(
+    Review.update_item_review(
         item,
         request.user,
         review.title,
         review.body,
         review.visibility,
         created_time=review.created_time,
-        share_to_mastodon=review.post_to_fediverse,
     )
+    if review.post_to_fediverse and request.user.mastodon_username:
+        share_review(review)
+
     return 200, {"message": "OK"}
 
 
@@ -205,7 +208,7 @@ def delete_review(request, item_uuid: str):
     item = Item.get_by_url(item_uuid)
     if not item:
         return 404, {"message": "Item not found"}
-    Review.review_item_by_user(item, request.user, None, None)
+    Review.update_item_review(item, request.user, None, None)
     return 200, {"message": "OK"}
 
 

journal/exporters/doufen.py

@@ -47,9 +47,7 @@ def export_marks_task(user):
     ]:
         ws = wb.create_sheet(title=label)
         shelf = user.shelf_manager.get_shelf(status)
-        q = query_item_category(ItemCategory.Movie) | query_item_category(
+        q = q_item_in_category(ItemCategory.Movie) | q_item_in_category(ItemCategory.TV)
-            ItemCategory.TV
-        )
         marks = shelf.members.all().filter(q).order_by("created_time")
         ws.append(heading)
         for mm in marks:
@@ -95,7 +93,7 @@ def export_marks_task(user):
     ]:
         ws = wb.create_sheet(title=label)
         shelf = user.shelf_manager.get_shelf(status)
-        q = query_item_category(ItemCategory.Music)
+        q = q_item_in_category(ItemCategory.Music)
         marks = shelf.members.all().filter(q).order_by("created_time")
         ws.append(heading)
         for mm in marks:
@@ -135,7 +133,7 @@ def export_marks_task(user):
     ]:
         ws = wb.create_sheet(title=label)
         shelf = user.shelf_manager.get_shelf(status)
-        q = query_item_category(ItemCategory.Book)
+        q = q_item_in_category(ItemCategory.Book)
         marks = shelf.members.all().filter(q).order_by("created_time")
         ws.append(heading)
         for mm in marks:
@@ -177,7 +175,7 @@ def export_marks_task(user):
     ]:
         ws = wb.create_sheet(title=label)
         shelf = user.shelf_manager.get_shelf(status)
-        q = query_item_category(ItemCategory.Game)
+        q = q_item_in_category(ItemCategory.Game)
         marks = shelf.members.all().filter(q).order_by("created_time")
         ws.append(heading)
         for mm in marks:
@@ -219,7 +217,7 @@ def export_marks_task(user):
     ]:
         ws = wb.create_sheet(title=label)
         shelf = user.shelf_manager.get_shelf(status)
-        q = query_item_category(ItemCategory.Podcast)
+        q = q_item_in_category(ItemCategory.Podcast)
         marks = shelf.members.all().filter(q).order_by("created_time")
         ws.append(heading)
         for mm in marks:
@@ -267,7 +265,7 @@ def export_marks_task(user):
         (ItemCategory.Podcast, "播客评论"),
     ]:
         ws = wb.create_sheet(title=label)
-        q = query_item_category(category)
+        q = q_item_in_category(category)
         reviews = Review.objects.filter(owner=user).filter(q).order_by("created_time")
         ws.append(review_heading)
         for review in reviews:

journal/importers/douban.py

@@ -261,7 +261,7 @@ class DoubanImporter:
         )
         print("+", end="", flush=True)
         if tags:
-            TagManager.tag_item_by_user(item, self.user, tags)
+            TagManager.tag_item(item, self.user, tags)
         return 1
 
     def import_review_sheet(self, worksheet, sheet_name):

journal/management/commands/journal.py

@@ -1,9 +1,10 @@
-import pprint
-
 from django.core.management.base import BaseCommand
 
+from catalog.models import Item
 from journal.importers.douban import DoubanImporter
 from journal.models import *
+from journal.models.common import Content
+from journal.models.itemlist import ListMember
 from users.models import User
 
 

journal/migrations/0014_alter_piece_options_piece_local_piece_post_id_and_more.py

@@ -0,0 +1,50 @@
+# Generated by Django 4.2.3 on 2023-08-06 02:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        (
+            "journal",
+            "0014_remove_reply_piece_ptr_remove_reply_reply_to_content_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="piece",
+            options={},
+        ),
+        migrations.AddField(
+            model_name="piece",
+            name="local",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name="piece",
+            name="post_id",
+            field=models.BigIntegerField(default=None, null=True),
+        ),
+        migrations.AddField(
+            model_name="comment",
+            name="remote_id",
+            field=models.CharField(default=None, max_length=200, null=True),
+        ),
+        migrations.AddField(
+            model_name="rating",
+            name="remote_id",
+            field=models.CharField(default=None, max_length=200, null=True),
+        ),
+        migrations.AddField(
+            model_name="review",
+            name="remote_id",
+            field=models.CharField(default=None, max_length=200, null=True),
+        ),
+        migrations.AddIndex(
+            model_name="piece",
+            index=models.Index(
+                fields=["post_id"], name="journal_pie_post_id_6a74ff_idx"
+            ),
+        ),
+    ]

journal/migrations/0015_alter_collection_featured_by_users_and_more.py

@@ -0,0 +1,111 @@
+# Generated by Django 4.2.4 on 2023-08-09 13:26
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("users", "0012_apidentity"),
+        ("journal", "0014_alter_piece_options_piece_local_piece_post_id_and_more"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="collection",
+            name="featured_by_users",
+        ),
+        migrations.AddField(
+            model_name="collection",
+            name="featured_by",
+            field=models.ManyToManyField(
+                related_name="featured_collections",
+                through="journal.FeaturedCollection",
+                to="users.apidentity",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="collection",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="users.apidentity"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="collectionmember",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="users.apidentity"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="comment",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="users.apidentity"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="featuredcollection",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE, to="users.apidentity"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="like",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="users.apidentity"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="rating",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="users.apidentity"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="review",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="users.apidentity"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="shelf",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="users.apidentity"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="shelflogentry",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="users.apidentity"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="shelfmember",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="users.apidentity"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="tag",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="users.apidentity"
+            ),
+        ),
+        migrations.AlterField(
+            model_name="tagmember",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.PROTECT, to="users.apidentity"
+            ),
+        ),
+    ]

journal/models/__init__.py

@@ -4,11 +4,11 @@ from .common import (
     Piece,
     UserOwnedObjectMixin,
     VisibilityType,
-    max_visiblity_to,
+    max_visiblity_to_user,
-    q_visible_to,
+    q_item_in_category,
-    query_following,
+    q_owned_piece_visible_to_user,
-    query_item_category,
+    q_piece_in_home_feed_of_user,
-    query_visible,
+    q_piece_visible_to_user,
 )
 from .like import Like
 from .mark import Mark

journal/models/collection.py

@@ -1,14 +1,14 @@
 import re
 from functools import cached_property
 
-from django.db import connection, models
+from django.db import models
 from django.utils.translation import gettext_lazy as _
 
 from catalog.collection.models import Collection as CatalogCollection
 from catalog.common import jsondata
 from catalog.common.utils import DEFAULT_ITEM_COVER, piece_cover_path
 from catalog.models import Item
-from users.models import User
+from users.models import APIdentity
 
 from .common import Piece
 from .itemlist import List, ListMember
@@ -42,8 +42,8 @@ class Collection(List):
     collaborative = models.PositiveSmallIntegerField(
         default=0
     )  # 0: Editable by owner only / 1: Editable by bi-direction followers
-    featured_by_users = models.ManyToManyField(
+    featured_by = models.ManyToManyField(
-        to=User, related_name="featured_collections", through="FeaturedCollection"
+        to=APIdentity, related_name="featured_collections", through="FeaturedCollection"
     )
 
     @property
@@ -56,25 +56,25 @@ class Collection(List):
         html = render_md(self.brief)
         return _RE_HTML_TAG.sub(" ", html)
 
-    def featured_by_user_since(self, user):
+    def featured_since(self, owner: APIdentity):
-        f = FeaturedCollection.objects.filter(target=self, owner=user).first()
+        f = FeaturedCollection.objects.filter(target=self, owner=owner).first()
         return f.created_time if f else None
 
-    def get_stats_for_user(self, user):
+    def get_stats(self, owner: APIdentity):
         items = list(self.members.all().values_list("item_id", flat=True))
         stats = {"total": len(items)}
-        for st, shelf in user.shelf_manager.shelf_list.items():
+        for st, shelf in owner.shelf_manager.shelf_list.items():
             stats[st] = shelf.members.all().filter(item_id__in=items).count()
         stats["percentage"] = (
             round(stats["complete"] * 100 / stats["total"]) if stats["total"] else 0
         )
         return stats
 
-    def get_progress_for_user(self, user):
+    def get_progress(self, owner: APIdentity):
         items = list(self.members.all().values_list("item_id", flat=True))
         if len(items) == 0:
             return 0
-        shelf = user.shelf_manager.shelf_list["complete"]
+        shelf = owner.shelf_manager.shelf_list["complete"]
         return round(
             shelf.members.all().filter(item_id__in=items).count() * 100 / len(items)
         )
@@ -94,7 +94,7 @@ class Collection(List):
 
 
 class FeaturedCollection(Piece):
-    owner = models.ForeignKey(User, on_delete=models.CASCADE)
+    owner = models.ForeignKey(APIdentity, on_delete=models.CASCADE)
     target = models.ForeignKey(Collection, on_delete=models.CASCADE)
     created_time = models.DateTimeField(auto_now_add=True)
     edited_time = models.DateTimeField(auto_now=True)
@@ -108,4 +108,4 @@ class FeaturedCollection(Piece):
 
     @cached_property
     def progress(self):
-        return self.target.get_progress_for_user(self.owner)
+        return self.target.get_progress(self.owner)

journal/models/comment.py

@@ -1,10 +1,11 @@
+from datetime import datetime
 from functools import cached_property
 
 from django.db import models
 from django.utils import timezone
 
 from catalog.models import Item
-from users.models import User
+from users.models import APIdentity
 
 from .common import Content
 from .rating import Rating
@@ -14,13 +15,44 @@ from .renderers import render_text
 class Comment(Content):
     text = models.TextField(blank=False, null=False)
 
+    @property
+    def ap_object(self):
+        return {
+            "id": self.absolute_url,
+            "type": "Comment",
+            "content": self.text,
+            "published": self.created_time.isoformat(),
+            "updated": self.edited_time.isoformat(),
+            "attributedTo": self.owner.actor_uri,
+            "relatedWith": self.item.absolute_url,
+            "url": self.absolute_url,
+        }
+
+    @classmethod
+    def update_by_ap_object(cls, owner, item, obj, post_id, visibility):
+        content = obj.get("content", "").strip() if obj else ""
+        if not content:
+            cls.objects.filter(owner=owner, item=item).delete()
+            return
+        d = {
+            "text": content,
+            "local": False,
+            "remote_id": obj["id"],
+            "post_id": post_id,
+            "visibility": visibility,
+            "created_time": datetime.fromisoformat(obj["published"]),
+            "edited_time": datetime.fromisoformat(obj["updated"]),
+        }
+        p, _ = cls.objects.update_or_create(owner=owner, item=item, defaults=d)
+        return p
+
     @property
     def html(self):
         return render_text(self.text)
 
     @cached_property
     def rating_grade(self):
-        return Rating.get_item_rating_by_user(self.item, self.owner)
+        return Rating.get_item_rating(self.item, self.owner)
 
     @cached_property
     def mark(self):
@@ -38,17 +70,17 @@ class Comment(Content):
             return self.item.url
 
     @staticmethod
-    def comment_item_by_user(
+    def comment_item(
-        item: Item, user: User, text: str | None, visibility=0, created_time=None
+        item: Item, owner: APIdentity, text: str | None, visibility=0, created_time=None
     ):
-        comment = Comment.objects.filter(owner=user, item=item).first()
+        comment = Comment.objects.filter(owner=owner, item=item).first()
         if not text:
             if comment is not None:
                 comment.delete()
                 comment = None
         elif comment is None:
             comment = Comment.objects.create(
-                owner=user,
+                owner=owner,
                 item=item,
                 text=text,
                 visibility=visibility,

journal/models/common.py

@@ -1,30 +1,20 @@
 import re
 import uuid
-from functools import cached_property
 
-import django.dispatch
 from django.conf import settings
-from django.contrib.contenttypes.models import ContentType
-from django.core.exceptions import PermissionDenied
-from django.core.validators import MaxValueValidator, MinValueValidator, RegexValidator
 from django.db import connection, models
 from django.db.models import Avg, Count, Q
 from django.utils import timezone
 from django.utils.baseconv import base62
 from django.utils.translation import gettext_lazy as _
-from markdownx.models import MarkdownxField
 from polymorphic.models import PolymorphicModel
 
-from catalog.collection.models import Collection as CatalogCollection
+from catalog.common.models import AvailableItemCategory, Item, ItemCategory
-from catalog.common import jsondata
-from catalog.common.models import Item, ItemCategory
-from catalog.common.utils import DEFAULT_ITEM_COVER, piece_cover_path
 from catalog.models import *
-from mastodon.api import share_review
+from takahe.utils import Takahe
-from users.models import User
+from users.models import APIdentity, User
 
 from .mixins import UserOwnedObjectMixin
-from .renderers import render_md, render_text
 
 _logger = logging.getLogger(__name__)
 
@@ -35,46 +25,57 @@ class VisibilityType(models.IntegerChoices):
     Private = 2, _("仅自己")
 
 
-def q_visible_to(viewer, owner):
+def q_owned_piece_visible_to_user(viewing_user: User, owner: APIdentity):
+    if (
+        not viewing_user
+        or not viewing_user.is_authenticated
+        or not viewing_user.identity
+    ):
+        return Q(visibility=0)
+    viewer = viewing_user.identity
     if viewer == owner:
         return Q()
     # elif viewer.is_blocked_by(owner):
     #     return Q(pk__in=[])
-    elif viewer.is_authenticated and viewer.is_following(owner):
+    elif viewer.is_following(owner):
-        return Q(visibility__in=[0, 1])
+        return Q(owner=owner, visibility__in=[0, 1])
     else:
-        return Q(visibility=0)
+        return Q(owner=owner, visibility=0)
 
 
-def max_visiblity_to(viewer, owner):
+def max_visiblity_to_user(viewing_user: User, owner: APIdentity):
+    if (
+        not viewing_user
+        or not viewing_user.is_authenticated
+        or not viewing_user.identity
+    ):
+        return 0
+    viewer = viewing_user.identity
     if viewer == owner:
         return 2
-    # elif viewer.is_blocked_by(owner):
+    elif viewer.is_following(owner):
-    #     return Q(pk__in=[])
-    elif viewer.is_authenticated and viewer.is_following(owner):
         return 1
     else:
         return 0
 
 
-def query_visible(user):
+def q_piece_visible_to_user(user: User):
+    if not user or not user.is_authenticated or not user.identity:
+        return Q(visibility=0)
     return (
-        (
+        Q(visibility=0)
-            Q(visibility=0)
+        | Q(owner_id__in=user.identity.following, visibility=1)
-            | Q(owner_id__in=user.following, visibility=1)
+        | Q(owner_id=user.identity.pk)
-            | Q(owner_id=user.id)
+    ) & ~Q(owner_id__in=user.identity.ignoring)
-        )
+
-        & ~Q(owner_id__in=user.ignoring)
+
-        if user.is_authenticated
+def q_piece_in_home_feed_of_user(user: User):
-        else Q(visibility=0)
+    return Q(owner_id__in=user.identity.following, visibility__lt=2) | Q(
+        owner_id=user.identity.pk
     )
 
 
-def query_following(user):
+def q_item_in_category(item_category: ItemCategory | AvailableItemCategory):
-    return Q(owner_id__in=user.following, visibility__lt=2) | Q(owner_id=user.id)
-
-
-def query_item_category(item_category):
     classes = item_categories()[item_category]
     # q = Q(item__instance_of=classes[0])
     # for cls in classes[1:]:
@@ -92,7 +93,7 @@ def query_item_category(item_category):
 
 
 # class ImportSession(models.Model):
-#     owner = models.ForeignKey(User, on_delete=models.CASCADE)
+#     owner = models.ForeignKey(APIdentity, on_delete=models.CASCADE)
 #     status = models.PositiveSmallIntegerField(default=ImportStatus.QUEUED)
 #     importer = models.CharField(max_length=50)
 #     file = models.CharField()
@@ -115,6 +116,13 @@ def query_item_category(item_category):
 class Piece(PolymorphicModel, UserOwnedObjectMixin):
     url_path = "p"  # subclass must specify this
     uid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True)
+    local = models.BooleanField(default=True)
+    post_id = models.BigIntegerField(null=True, default=None)
+
+    class Meta:
+        indexes = [
+            models.Index(fields=["post_id"]),
+        ]
 
     @property
     def uuid(self):
@@ -132,9 +140,18 @@ class Piece(PolymorphicModel, UserOwnedObjectMixin):
     def api_url(self):
         return f"/api/{self.url}" if self.url_path else None
 
+    @property
+    def shared_link(self):
+        return Takahe.get_post_url(self.post_id) if self.post_id else None
+
     @property
     def like_count(self):
-        return self.likes.all().count()
+        return (
+            Takahe.get_post_stats(self.post_id).get("likes", 0) if self.post_id else 0
+        )
+
+    def is_liked_by(self, user):
+        return self.post_id and Takahe.post_liked_by(self.post_id, user)
 
     @classmethod
     def get_by_url(cls, url_or_b62):
@@ -149,9 +166,17 @@ class Piece(PolymorphicModel, UserOwnedObjectMixin):
             obj = None
         return obj
 
+    @classmethod
+    def update_by_ap_object(cls, owner, item, obj, post_id, visibility):
+        raise NotImplemented
+
+    @property
+    def ap_object(self):
+        raise NotImplemented
+
 
 class Content(Piece):
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
+    owner = models.ForeignKey(APIdentity, on_delete=models.PROTECT)
     visibility = models.PositiveSmallIntegerField(
         default=0
     )  # 0: Public / 1: Follower only / 2: Self only
@@ -161,6 +186,7 @@ class Content(Piece):
     )  # auto_now=True   FIXME revert this after migration
     metadata = models.JSONField(default=dict)
     item = models.ForeignKey(Item, on_delete=models.PROTECT)
+    remote_id = models.CharField(max_length=200, null=True, default=None)
 
     def __str__(self):
         return f"{self.uuid}@{self.item}"

journal/models/itemlist.py

@@ -5,7 +5,7 @@ from django.db import models
 from django.utils import timezone
 
 from catalog.models import Item, ItemCategory
-from users.models import User
+from users.models import APIdentity
 
 from .common import Piece
 
@@ -15,24 +15,21 @@ list_remove = django.dispatch.Signal()
 
 class List(Piece):
     """
-    List (abstract class)
+    List (abstract model)
     """
 
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
+    owner = models.ForeignKey(APIdentity, on_delete=models.PROTECT)
     visibility = models.PositiveSmallIntegerField(
         default=0
     )  # 0: Public / 1: Follower only / 2: Self only
-    created_time = models.DateTimeField(
+    created_time = models.DateTimeField(default=timezone.now)
-        default=timezone.now
+    edited_time = models.DateTimeField(default=timezone.now)
-    )  # auto_now_add=True  FIXME revert this after migration
-    edited_time = models.DateTimeField(
-        default=timezone.now
-    )  # auto_now=True   FIXME revert this after migration
     metadata = models.JSONField(default=dict)
 
     class Meta:
         abstract = True
 
+    MEMBER_CLASS: Piece
     # MEMBER_CLASS = None  # subclass must override this
     # subclass must add this:
     # items = models.ManyToManyField(Item, through='ListMember')
@@ -146,14 +143,12 @@ class ListMember(Piece):
     parent = models.ForeignKey('List', related_name='members', on_delete=models.CASCADE)
     """
 
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
+    owner = models.ForeignKey(APIdentity, on_delete=models.PROTECT)
     visibility = models.PositiveSmallIntegerField(
         default=0
     )  # 0: Public / 1: Follower only / 2: Self only
     created_time = models.DateTimeField(default=timezone.now)
-    edited_time = models.DateTimeField(
+    edited_time = models.DateTimeField(default=timezone.now)
-        default=timezone.now
-    )  # auto_now=True   FIXME revert this after migration
     metadata = models.JSONField(default=dict)
     item = models.ForeignKey(Item, on_delete=models.PROTECT)
     position = models.PositiveIntegerField()

journal/models/like.py

@@ -3,13 +3,13 @@ from django.db import connection, models
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
-from users.models import User
+from users.models import APIdentity
 
 from .common import Piece
 
 
-class Like(Piece):
+class Like(Piece):  # TODO remove
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
+    owner = models.ForeignKey(APIdentity, on_delete=models.PROTECT)
     visibility = models.PositiveSmallIntegerField(
         default=0
     )  # 0: Public / 1: Follower only / 2: Self only
@@ -18,25 +18,27 @@ class Like(Piece):
     target = models.ForeignKey(Piece, on_delete=models.CASCADE, related_name="likes")
 
     @staticmethod
-    def user_liked_piece(user, piece):
+    def user_liked_piece(owner, piece):
-        return Like.objects.filter(owner=user, target=piece).exists()
+        return Like.objects.filter(owner=owner.identity, target=piece).exists()
 
     @staticmethod
-    def user_like_piece(user, piece):
+    def user_like_piece(owner, piece):
         if not piece:
             return
-        like = Like.objects.filter(owner=user, target=piece).first()
+        like = Like.objects.filter(owner=owner.identity, target=piece).first()
         if not like:
-            like = Like.objects.create(owner=user, target=piece)
+            like = Like.objects.create(owner=owner.identity, target=piece)
         return like
 
     @staticmethod
-    def user_unlike_piece(user, piece):
+    def user_unlike_piece(owner, piece):
         if not piece:
             return
-        Like.objects.filter(owner=user, target=piece).delete()
+        Like.objects.filter(owner=owner.identity, target=piece).delete()
 
     @staticmethod
-    def user_likes_by_class(user, cls):
+    def user_likes_by_class(owner, cls):
         ctype_id = ContentType.objects.get_for_model(cls)
-        return Like.objects.filter(owner=user, target__polymorphic_ctype=ctype_id)
+        return Like.objects.filter(
+            owner=owner.identity, target__polymorphic_ctype=ctype_id
+        )

journal/models/mark.py

@@ -12,6 +12,7 @@ from django.db.models import Avg, Count, Q
 from django.utils import timezone
 from django.utils.baseconv import base62
 from django.utils.translation import gettext_lazy as _
+from loguru import logger
 from markdownx.models import MarkdownxField
 from polymorphic.models import PolymorphicModel
 
@@ -20,16 +21,14 @@ from catalog.common import jsondata
 from catalog.common.models import Item, ItemCategory
 from catalog.common.utils import DEFAULT_ITEM_COVER, piece_cover_path
 from catalog.models import *
-from mastodon.api import share_review
+from takahe.utils import Takahe
-from users.models import User
+from users.models import APIdentity
 
 from .comment import Comment
 from .rating import Rating
 from .review import Review
 from .shelf import Shelf, ShelfLogEntry, ShelfManager, ShelfMember, ShelfType
 
-_logger = logging.getLogger(__name__)
-
 
 class Mark:
     """
@@ -38,8 +37,8 @@ class Mark:
     it mimics previous mark behaviour.
     """
 
-    def __init__(self, user, item):
+    def __init__(self, owner: APIdentity, item: Item):
-        self.owner = user
+        self.owner = owner
         self.item = item
 
     @cached_property
@@ -60,7 +59,7 @@ class Mark:
 
     @property
     def action_label(self) -> str:
-        if self.shelfmember:
+        if self.shelfmember and self.shelf_type:
             return ShelfManager.get_action_label(self.shelf_type, self.item.category)
         if self.comment:
             return ShelfManager.get_action_label(
@@ -72,7 +71,7 @@ class Mark:
     def shelf_label(self) -> str | None:
         return (
             ShelfManager.get_label(self.shelf_type, self.item.category)
-            if self.shelfmember
+            if self.shelf_type
             else None
         )
 
@@ -86,19 +85,23 @@ class Mark:
 
     @property
     def visibility(self) -> int:
-        return (
+        if self.shelfmember:
-            self.shelfmember.visibility
+            return self.shelfmember.visibility
-            if self.shelfmember
+        else:
-            else self.owner.preference.default_visibility
+            logger.warning(f"no shelfmember for mark {self.owner}, {self.item}")
-        )
+            return 2
 
     @cached_property
     def tags(self) -> list[str]:
         return self.owner.tag_manager.get_item_tags(self.item)
 
+    @cached_property
+    def rating(self):
+        return Rating.objects.filter(owner=self.owner, item=self.item).first()
+
     @cached_property
     def rating_grade(self) -> int | None:
-        return Rating.get_item_rating_by_user(self.item, self.owner)
+        return Rating.get_item_rating(self.item, self.owner)
 
     @cached_property
     def comment(self) -> Comment | None:
@@ -118,29 +121,24 @@ class Mark:
 
     def update(
         self,
-        shelf_type: ShelfType | None,
+        shelf_type,
-        comment_text: str | None,
+        comment_text,
-        rating_grade: int | None,
+        rating_grade,
-        visibility: int,
+        visibility,
         metadata=None,
         created_time=None,
         share_to_mastodon=False,
-        silence=False,
     ):
-        # silence=False means update is logged.
+        post_to_feed = shelf_type is not None and (
-        share = (
+            shelf_type != self.shelf_type
-            share_to_mastodon
+            or comment_text != self.comment_text
-            and self.owner.mastodon_username
+            or rating_grade != self.rating_grade
-            and shelf_type is not None
-            and (
-                shelf_type != self.shelf_type
-                or comment_text != self.comment_text
-                or rating_grade != self.rating_grade
-            )
         )
+        if shelf_type is None:
+            Takahe.delete_mark(self)
         if created_time and created_time >= timezone.now():
             created_time = None
-        share_as_new_post = shelf_type != self.shelf_type
+        post_as_new = shelf_type != self.shelf_type
         original_visibility = self.visibility
         if shelf_type != self.shelf_type or visibility != original_visibility:
             self.shelfmember = self.owner.shelf_manager.move_item(
@@ -148,9 +146,8 @@ class Mark:
                 shelf_type,
                 visibility=visibility,
                 metadata=metadata,
-                silence=silence,
             )
-        if not silence and self.shelfmember and created_time:
+        if self.shelfmember and created_time:
             # if it's an update(not delete) and created_time is specified,
             # update the timestamp of the shelfmember and log
             log = ShelfLogEntry.objects.filter(
@@ -172,7 +169,7 @@ class Mark:
                     timestamp=created_time,
                 )
         if comment_text != self.comment_text or visibility != original_visibility:
-            self.comment = Comment.comment_item_by_user(
+            self.comment = Comment.comment_item(
                 self.item,
                 self.owner,
                 comment_text,
@@ -180,35 +177,15 @@ class Mark:
                 self.shelfmember.created_time if self.shelfmember else None,
             )
         if rating_grade != self.rating_grade or visibility != original_visibility:
-            Rating.rate_item_by_user(self.item, self.owner, rating_grade, visibility)
+            Rating.update_item_rating(self.item, self.owner, rating_grade, visibility)
             self.rating_grade = rating_grade
-        if share:
-            # this is a bit hacky but let's keep it until move to implement ActivityPub,
-            # by then, we'll just change this to boost
-            from mastodon.api import share_mark
 
-            self.shared_link = (
+        if post_to_feed:
-                self.shelfmember.metadata.get("shared_link")
+            Takahe.post_mark(self, post_as_new)
-                if self.shelfmember.metadata and not share_as_new_post
-                else None
-            )
-            self.save = lambda **args: None
-            result, code = share_mark(self)
-            if not result:
-                if code == 401:
-                    raise PermissionDenied()
-                else:
-                    raise ValueError(code)
-            if self.shelfmember.metadata.get("shared_link") != self.shared_link:
-                self.shelfmember.metadata["shared_link"] = self.shared_link
-                self.shelfmember.save()
-        elif share_as_new_post and self.shelfmember:
-            self.shelfmember.metadata["shared_link"] = None
-            self.shelfmember.save()
 
-    def delete(self, silence=False):
+    def delete(self):
         # self.logs.delete()  # When deleting a mark, all logs of the mark are deleted first.
-        self.update(None, None, None, 0, silence=silence)
+        self.update(None, None, None, 0)
 
     def delete_log(self, log_id):
         ShelfLogEntry.objects.filter(

journal/models/mixins.py

@@ -1,4 +1,6 @@
-from typing import TYPE_CHECKING, Type
+from typing import TYPE_CHECKING
+
+from users.models import APIdentity, User
 
 if TYPE_CHECKING:
     from .common import Piece
@@ -9,18 +11,24 @@ class UserOwnedObjectMixin:
     UserOwnedObjectMixin
 
     Models must add these:
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
+    owner = models.ForeignKey(APIdentity, on_delete=models.PROTECT)
     visibility = models.PositiveSmallIntegerField(default=0)
     """
 
-    def is_visible_to(self: "Piece", viewer):  # type: ignore
+    owner: APIdentity
+    visibility: int
+
+    def is_visible_to(self: "Piece | Self", viewing_user: User) -> bool:  # type: ignore
         owner = self.owner
-        if owner == viewer:
+        if not owner or not owner.is_active:
-            return True
-        if not owner.is_active:
             return False
-        if not viewer.is_authenticated:
+        if owner.user == viewing_user:
+            return True
+        if not viewing_user.is_authenticated:
             return self.visibility == 0
+        viewer = viewing_user.identity  # type: ignore[assignment]
+        if not viewer:
+            return False
         if self.visibility == 2:
             return False
         if viewer.is_blocking(owner) or owner.is_blocking(viewer):
@@ -30,27 +38,9 @@ class UserOwnedObjectMixin:
         else:
             return True
 
-    def is_editable_by(self: "Piece", viewer):  # type: ignore
+    def is_editable_by(self: "Piece", viewing_user: User):  # type: ignore
-        return viewer.is_authenticated and (
+        return viewing_user.is_authenticated and (
-            viewer.is_staff or viewer.is_superuser or viewer == self.owner
+            viewing_user.is_staff
+            or viewing_user.is_superuser
+            or viewing_user == self.owner.user
         )
-
-    @classmethod
-    def get_available(cls: "Type[Piece]", entity, request_user, following_only=False):  # type: ignore
-        # e.g. SongMark.get_available(song, request.user)
-        query_kwargs = {entity.__class__.__name__.lower(): entity}
-        all_entities = cls.objects.filter(**query_kwargs).order_by(
-            "-created_time"
-        )  # get all marks for song
-        visible_entities = list(
-            filter(
-                lambda _entity: _entity.is_visible_to(request_user)
-                and (
-                    _entity.owner.mastodon_acct in request_user.mastodon_following
-                    if following_only
-                    else True
-                ),
-                all_entities,
-            )
-        )
-        return visible_entities

journal/models/rating.py

@@ -1,10 +1,12 @@
+from datetime import datetime
+
 from django.core.validators import MaxValueValidator, MinValueValidator, RegexValidator
 from django.db import connection, models
 from django.db.models import Avg, Count, Q
 from django.utils.translation import gettext_lazy as _
 
 from catalog.models import Item, ItemCategory
-from users.models import User
+from users.models import APIdentity
 
 from .common import Content
 
@@ -20,6 +22,51 @@ class Rating(Content):
         default=0, validators=[MaxValueValidator(10), MinValueValidator(1)], null=True
     )
 
+    @property
+    def ap_object(self):
+        return {
+            "id": self.absolute_url,
+            "type": "Rating",
+            "best": 10,
+            "worst": 1,
+            "value": self.grade,
+            "published": self.created_time.isoformat(),
+            "updated": self.edited_time.isoformat(),
+            "attributedTo": self.owner.actor_uri,
+            "relatedWith": self.item.absolute_url,
+            "url": self.absolute_url,
+        }
+
+    @classmethod
+    def update_by_ap_object(cls, owner, item, obj, post_id, visibility):
+        value = obj.get("value", 0) if obj else 0
+        if not value:
+            cls.objects.filter(owner=owner, item=item).delete()
+            return
+        best = obj.get("best", 5)
+        worst = obj.get("worst", 1)
+        if best <= worst:
+            return
+        if value < worst:
+            value = worst
+        if value > best:
+            value = best
+        if best != 10 or worst != 1:
+            value = round(9 * (value - worst) / (best - worst)) + 1
+        else:
+            value = round(value)
+        d = {
+            "grade": value,
+            "local": False,
+            "remote_id": obj["id"],
+            "post_id": post_id,
+            "visibility": visibility,
+            "created_time": datetime.fromisoformat(obj["published"]),
+            "edited_time": datetime.fromisoformat(obj["updated"]),
+        }
+        p, _ = cls.objects.update_or_create(owner=owner, item=item, defaults=d)
+        return p
+
     @staticmethod
     def get_rating_for_item(item: Item) -> float | None:
         stat = Rating.objects.filter(grade__isnull=False)
@@ -65,19 +112,19 @@ class Rating(Content):
         return r
 
     @staticmethod
-    def rate_item_by_user(
+    def update_item_rating(
-        item: Item, user: User, rating_grade: int | None, visibility: int = 0
+        item: Item, owner: APIdentity, rating_grade: int | None, visibility: int = 0
     ):
         if rating_grade and (rating_grade < 1 or rating_grade > 10):
             raise ValueError(f"Invalid rating grade: {rating_grade}")
-        rating = Rating.objects.filter(owner=user, item=item).first()
+        rating = Rating.objects.filter(owner=owner, item=item).first()
         if not rating_grade:
             if rating:
                 rating.delete()
                 rating = None
         elif rating is None:
             rating = Rating.objects.create(
-                owner=user, item=item, grade=rating_grade, visibility=visibility
+                owner=owner, item=item, grade=rating_grade, visibility=visibility
             )
         elif rating.grade != rating_grade or rating.visibility != visibility:
             rating.visibility = visibility
@@ -86,6 +133,6 @@ class Rating(Content):
         return rating
 
     @staticmethod
-    def get_item_rating_by_user(item: Item, user: User) -> int | None:
+    def get_item_rating(item: Item, owner: APIdentity) -> int | None:
-        rating = Rating.objects.filter(owner=user, item=item).first()
+        rating = Rating.objects.filter(owner=owner, item=item).first()
         return (rating.grade or None) if rating else None

journal/models/renderers.py

@@ -19,7 +19,7 @@ _mistune_plugins = [
 _markdown = mistune.create_markdown(plugins=_mistune_plugins)
 
 
-def convert_leading_space_in_md(body) -> str:
+def convert_leading_space_in_md(body: str) -> str:
     body = re.sub(r"^\s+$", "", body, flags=re.MULTILINE)
     body = re.sub(
         r"^(\u2003*)( +)",
@@ -30,11 +30,11 @@ def convert_leading_space_in_md(body) -> str:
     return body
 
 
-def render_md(s) -> str:
+def render_md(s: str) -> str:
     return cast(str, _markdown(s))
 
 
-def _spolier(s):
+def _spolier(s: str) -> str:
     l = s.split(">!", 1)
     if len(l) == 1:
         return escape(s)
@@ -48,5 +48,5 @@ def _spolier(s):
     )
 
 
-def render_text(s):
+def render_text(s: str) -> str:
     return _spolier(s)

journal/models/review.py

@@ -7,8 +7,7 @@ from django.utils.translation import gettext_lazy as _
 from markdownx.models import MarkdownxField
 
 from catalog.models import Item
-from mastodon.api import share_review
+from users.models import APIdentity
-from users.models import User
 
 from .common import Content
 from .rating import Rating
@@ -44,21 +43,20 @@ class Review(Content):
 
     @cached_property
     def rating_grade(self):
-        return Rating.get_item_rating_by_user(self.item, self.owner)
+        return Rating.get_item_rating(self.item, self.owner)
 
     @classmethod
-    def review_item_by_user(
+    def update_item_review(
         cls,
         item: Item,
-        user: User,
+        owner: APIdentity,
         title: str | None,
         body: str | None,
         visibility=0,
         created_time=None,
-        share_to_mastodon=False,
     ):
         if title is None:
-            review = Review.objects.filter(owner=user, item=item).first()
+            review = Review.objects.filter(owner=owner, item=item).first()
             if review is not None:
                 review.delete()
             return None
@@ -71,9 +69,7 @@ class Review(Content):
             defaults["created_time"] = (
                 created_time if created_time < timezone.now() else timezone.now()
             )
-        review, created = cls.objects.update_or_create(
+        review, _ = cls.objects.update_or_create(
-            item=item, owner=user, defaults=defaults
+            item=item, owner=owner, defaults=defaults
         )
-        if share_to_mastodon and user.mastodon_username:
-            share_review(review)
         return review

journal/models/shelf.py

@@ -1,14 +1,17 @@
+from datetime import datetime
 from functools import cached_property
 from typing import TYPE_CHECKING
 
 from django.db import connection, models
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
+from loguru import logger
 
 from catalog.models import Item, ItemCategory
-from users.models import User
+from takahe.models import Identity
+from users.models import APIdentity
 
-from .common import query_item_category
+from .common import q_item_in_category
 from .itemlist import List, ListMember
 
 if TYPE_CHECKING:
@@ -60,6 +63,43 @@ class ShelfMember(ListMember):
             models.Index(fields=["parent_id", "visibility", "created_time"]),
         ]
 
+    @property
+    def ap_object(self):
+        return {
+            "id": self.absolute_url,
+            "type": "Status",
+            "status": self.parent.shelf_type,
+            "published": self.created_time.isoformat(),
+            "updated": self.edited_time.isoformat(),
+            "attributedTo": self.owner.actor_uri,
+            "relatedWith": self.item.absolute_url,
+            "url": self.absolute_url,
+        }
+
+    @classmethod
+    def update_by_ap_object(
+        cls, owner: APIdentity, item: Identity, obj: dict, post_id: int, visibility: int
+    ):
+        if not obj:
+            cls.objects.filter(owner=owner, item=item).delete()
+            return
+        shelf = owner.shelf_manager.get_shelf(obj["status"])
+        if not shelf:
+            logger.warning(f"unable to locate shelf for {owner}, {obj}")
+            return
+        d = {
+            "parent": shelf,
+            "position": 0,
+            "local": False,
+            # "remote_id": obj["id"],
+            "post_id": post_id,
+            "visibility": visibility,
+            "created_time": datetime.fromisoformat(obj["published"]),
+            "edited_time": datetime.fromisoformat(obj["updated"]),
+        }
+        p, _ = cls.objects.update_or_create(owner=owner, item=item, defaults=d)
+        return p
+
     @cached_property
     def mark(self) -> "Mark":
         from .mark import Mark
@@ -108,7 +148,7 @@ class Shelf(List):
 
 
 class ShelfLogEntry(models.Model):
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
+    owner = models.ForeignKey(APIdentity, on_delete=models.PROTECT)
     shelf_type = models.CharField(choices=ShelfType.choices, max_length=100, null=True)
     item = models.ForeignKey(Item, on_delete=models.PROTECT)
     timestamp = models.DateTimeField()  # this may later be changed by user
@@ -135,8 +175,8 @@ class ShelfManager:
     ShelfLogEntry can later be modified if user wish to change history
     """
 
-    def __init__(self, user):
+    def __init__(self, owner):
-        self.owner = user
+        self.owner = owner
         qs = Shelf.objects.filter(owner=self.owner)
         self.shelf_list = {v.shelf_type: v for v in qs}
         if len(self.shelf_list) == 0:
@@ -146,13 +186,18 @@ class ShelfManager:
         for qt in ShelfType:
             self.shelf_list[qt] = Shelf.objects.create(owner=self.owner, shelf_type=qt)
 
-    def locate_item(self, item) -> ShelfMember | None:
+    def locate_item(self, item: Item) -> ShelfMember | None:
         return ShelfMember.objects.filter(item=item, owner=self.owner).first()
 
-    def move_item(self, item, shelf_type, visibility=0, metadata=None, silence=False):
+    def move_item(
+        self,
+        item: Item,
+        shelf_type: ShelfType,
+        visibility: int = 0,
+        metadata: dict | None = None,
+    ):
         # shelf_type=None means remove from current shelf
         # metadata=None means no change
-        # silence=False means move_item is logged.
         if not item:
             raise ValueError("empty item")
         new_shelfmember = None
@@ -185,7 +230,7 @@ class ShelfManager:
                 elif visibility != last_visibility:  # change visibility
                     last_shelfmember.visibility = visibility
                     last_shelfmember.save()
-        if changed and not silence:
+        if changed:
             if metadata is None:
                 metadata = last_metadata or {}
             log_time = (
@@ -205,18 +250,20 @@ class ShelfManager:
     def get_log(self):
         return ShelfLogEntry.objects.filter(owner=self.owner).order_by("timestamp")
 
-    def get_log_for_item(self, item):
+    def get_log_for_item(self, item: Item):
         return ShelfLogEntry.objects.filter(owner=self.owner, item=item).order_by(
             "timestamp"
         )
 
-    def get_shelf(self, shelf_type):
+    def get_shelf(self, shelf_type: ShelfType):
         return self.shelf_list[shelf_type]
 
-    def get_latest_members(self, shelf_type, item_category=None):
+    def get_latest_members(
+        self, shelf_type: ShelfType, item_category: ItemCategory | None = None
+    ):
         qs = self.shelf_list[shelf_type].members.all().order_by("-created_time")
         if item_category:
-            return qs.filter(query_item_category(item_category))
+            return qs.filter(q_item_in_category(item_category))
         else:
             return qs
 
@@ -229,14 +276,16 @@ class ShelfManager:
     #     return shelf.members.all().order_by
 
     @classmethod
-    def get_action_label(cls, shelf_type, item_category) -> str:
+    def get_action_label(
+        cls, shelf_type: ShelfType, item_category: ItemCategory
+    ) -> str:
         sts = [
             n[2] for n in ShelfTypeNames if n[0] == item_category and n[1] == shelf_type
         ]
         return sts[0] if sts else str(shelf_type)
 
     @classmethod
-    def get_label(cls, shelf_type, item_category):
+    def get_label(cls, shelf_type: ShelfType, item_category: ItemCategory):
         ic = ItemCategory(item_category).label
         st = cls.get_action_label(shelf_type, item_category)
         return (
@@ -246,10 +295,10 @@ class ShelfManager:
         )
 
     @staticmethod
-    def get_manager_for_user(user):
+    def get_manager_for_user(owner: APIdentity):
-        return ShelfManager(user)
+        return ShelfManager(owner)
 
-    def get_calendar_data(self, max_visiblity):
+    def get_calendar_data(self, max_visiblity: int):
         shelf_id = self.get_shelf(ShelfType.COMPLETE).pk
         timezone_offset = timezone.localtime(timezone.now()).strftime("%z")
         timezone_offset = timezone_offset[: len(timezone_offset) - 2]

journal/models/tag.py

@@ -8,7 +8,7 @@ from django.utils.translation import gettext_lazy as _
 
 from catalog.collection.models import Collection as CatalogCollection
 from catalog.models import Item
-from users.models import User
+from users.models import APIdentity
 
 from .itemlist import List, ListMember
 
@@ -66,9 +66,9 @@ class TagManager:
         return tag_titles
 
     @staticmethod
-    def all_tags_for_user(user, public_only=False):
+    def all_tags_by_owner(owner, public_only=False):
         tags = (
-            user.tag_set.all()
+            owner.tag_set.all()
             .values("title")
             .annotate(frequency=Count("members__id"))
             .order_by("-frequency")
@@ -78,46 +78,44 @@ class TagManager:
         return list(map(lambda t: t["title"], tags))
 
     @staticmethod
-    def tag_item_by_user(item, user, tag_titles, default_visibility=0):
+    def tag_item(
+        item: Item,
+        owner: APIdentity,
+        tag_titles: list[str],
+        default_visibility: int = 0,
+    ):
         titles = set([Tag.cleanup_title(tag_title) for tag_title in tag_titles])
         current_titles = set(
-            [m.parent.title for m in TagMember.objects.filter(owner=user, item=item)]
+            [m.parent.title for m in TagMember.objects.filter(owner=owner, item=item)]
         )
         for title in titles - current_titles:
-            tag = Tag.objects.filter(owner=user, title=title).first()
+            tag = Tag.objects.filter(owner=owner, title=title).first()
             if not tag:
                 tag = Tag.objects.create(
-                    owner=user, title=title, visibility=default_visibility
+                    owner=owner, title=title, visibility=default_visibility
                 )
             tag.append_item(item, visibility=default_visibility)
         for title in current_titles - titles:
-            tag = Tag.objects.filter(owner=user, title=title).first()
+            tag = Tag.objects.filter(owner=owner, title=title).first()
             if tag:
                 tag.remove_item(item)
 
     @staticmethod
-    def get_item_tags_by_user(item, user):
+    def get_manager_for_user(owner):
-        current_titles = [
+        return TagManager(owner)
-            m.parent.title for m in TagMember.objects.filter(owner=user, item=item)
-        ]
-        return current_titles
 
-    @staticmethod
+    def __init__(self, owner):
-    def get_manager_for_user(user):
+        self.owner = owner
-        return TagManager(user)
-
-    def __init__(self, user):
-        self.owner = user
 
     @property
     def all_tags(self):
-        return TagManager.all_tags_for_user(self.owner)
+        return TagManager.all_tags_by_owner(self.owner)
 
     @property
     def public_tags(self):
-        return TagManager.all_tags_for_user(self.owner, public_only=True)
+        return TagManager.all_tags_by_owner(self.owner, public_only=True)
 
-    def get_item_tags(self, item):
+    def get_item_tags(self, item: Item):
         return sorted(
             [
                 m["parent__title"]

journal/models/utils.py

@@ -2,7 +2,7 @@ from django.utils.translation import gettext_lazy as _
 from loguru import logger
 
 from catalog.models import Item
-from users.models import User
+from users.models import APIdentity
 
 from .collection import Collection, CollectionMember, FeaturedCollection
 from .comment import Comment
@@ -10,27 +10,28 @@ from .common import Content
 from .itemlist import ListMember
 from .rating import Rating
 from .review import Review
-from .shelf import Shelf, ShelfLogEntry, ShelfManager, ShelfMember
+from .shelf import ShelfLogEntry, ShelfMember
-from .tag import Tag, TagManager, TagMember
+from .tag import Tag, TagMember
 
 
-def reset_journal_visibility_for_user(user: User, visibility: int):
+def reset_journal_visibility_for_user(owner: APIdentity, visibility: int):
-    ShelfMember.objects.filter(owner=user).update(visibility=visibility)
+    ShelfMember.objects.filter(owner=owner).update(visibility=visibility)
-    Comment.objects.filter(owner=user).update(visibility=visibility)
+    Comment.objects.filter(owner=owner).update(visibility=visibility)
-    Rating.objects.filter(owner=user).update(visibility=visibility)
+    Rating.objects.filter(owner=owner).update(visibility=visibility)
-    Review.objects.filter(owner=user).update(visibility=visibility)
+    Review.objects.filter(owner=owner).update(visibility=visibility)
 
 
-def remove_data_by_user(user: User):
+def remove_data_by_user(owner: APIdentity):
-    ShelfMember.objects.filter(owner=user).delete()
+    ShelfMember.objects.filter(owner=owner).delete()
-    Comment.objects.filter(owner=user).delete()
+    ShelfLogEntry.objects.filter(owner=owner).delete()
-    Rating.objects.filter(owner=user).delete()
+    Comment.objects.filter(owner=owner).delete()
-    Review.objects.filter(owner=user).delete()
+    Rating.objects.filter(owner=owner).delete()
-    TagMember.objects.filter(owner=user).delete()
+    Review.objects.filter(owner=owner).delete()
-    Tag.objects.filter(owner=user).delete()
+    TagMember.objects.filter(owner=owner).delete()
-    CollectionMember.objects.filter(owner=user).delete()
+    Tag.objects.filter(owner=owner).delete()
-    Collection.objects.filter(owner=user).delete()
+    CollectionMember.objects.filter(owner=owner).delete()
-    FeaturedCollection.objects.filter(owner=user).delete()
+    Collection.objects.filter(owner=owner).delete()
+    FeaturedCollection.objects.filter(owner=owner).delete()
 
 
 def update_journal_for_merged_item(

journal/templates/_list_item.html

@@ -55,7 +55,7 @@
               <span>
                 <a target="_blank"
                    rel="noopener"
-                   {% if mark.metadata.shared_link %} href="{{ mark.metadata.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if mark.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
+                   {% if mark.shared_link %} href="{{ mark.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if mark.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
               </span>
               <span class="timestamp">{{ mark.created_time|date }}</span>
             </div>
@@ -88,7 +88,7 @@
               <span>
                 <a target="_blank"
                    rel="noopener"
-                   {% if mark.review.metadata.shared_link %} href="{{ mark.review.metadata.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if mark.review.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
+                   {% if mark.review.shared_link %} href="{{ mark.review.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if mark.review.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
               </span>
               <span class="timestamp">{{ mark.review.created_time|date }}</span>
             </span>

journal/templates/profile.html

@@ -15,14 +15,14 @@
     {% else %}
       <title>{{ site_name }} - {{ user.display_name }}</title>
     {% endif %}
-    <meta property="og:title" content="{{ site_name }}用户 - @{{ user.handler }}">
+    <meta property="og:title" content="{{ site_name }}用户 - {{ user.handler }}">
     <meta property="og:url" content="{{ request.build_absolute_uri }}">
     <meta property="og:image" content="{{ user.avatar }}">
     <meta property="og:site_name" content="{{ site_name }}">
     {% if user.preference.no_anonymous_view %}<meta name="robots" content="noindex">{% endif %}
     <link rel="alternate"
           type="application/rss+xml"
-          title="{{ site_name }} - @{{ user.handler }}的评论"
+          title="{{ site_name }} - {{ user.handler }}的评论"
           href="{{ request.build_absolute_uri }}feed/reviews/">
     {% include "common_libs.html" with jquery=0 v2=1 %}
     <script src="{% static 'js/calendar_yearview_blocks.js' %}" defer></script>

journal/templates/review.html

@@ -41,7 +41,7 @@
                   <span>
                     <a target="_blank"
                        rel="noopener"
-                       {% if review.metadata.shared_link %} href="{{ review.metadata.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if review.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
+                       {% if review.shared_link %} href="{{ review.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if review.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
                   </span>
                   {% if request.user == review.owner %}{% endif %}
                 </div>

journal/templates/user_collection_list.html

@@ -37,7 +37,7 @@
               <span>
                 <a target="_blank"
                    rel="noopener"
-                   {% if collection.metadata.shared_link %} href="{{ collection.metadata.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if collection.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
+                   {% if collection.shared_link %} href="{{ collection.shared_link }}" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if collection.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %}"></i></a>
               </span>
               <span class="timestamp">{{ collection.created_time|date }}</span>
             </div>

journal/templatetags/collection.py

@@ -1,32 +1,34 @@
 from django import template
 from django.template.defaultfilters import stringfilter
 
-from journal.models import Collection, Like
+from journal.models import Collection
+from journal.models.mixins import UserOwnedObjectMixin
+from users.models.user import User
 
 register = template.Library()
 
 
 @register.simple_tag(takes_context=True)
-def user_visibility_of(context, piece):
+def user_visibility_of(context, piece: UserOwnedObjectMixin):
     user = context["request"].user
     return piece.is_visible_to(user)
 
 
 @register.simple_tag()
-def user_progress_of(collection, user):
+def user_progress_of(collection: Collection, user: User):
     return (
-        collection.get_progress_for_user(user) if user and user.is_authenticated else 0
+        collection.get_progress(user.identity) if user and user.is_authenticated else 0
     )
 
 
 @register.simple_tag()
-def user_stats_of(collection, user):
+def user_stats_of(collection: Collection, user: User):
-    return collection.get_stats_for_user(user) if user and user.is_authenticated else {}
+    return collection.get_stats(user.identity) if user and user.is_authenticated else {}
 
 
 @register.filter(is_safe=True)
 @stringfilter
-def prural_items(category):
+def prural_items(category: str):
     # TODO support i18n here
     # return _(f"items of {category}")
     if category == "book":

journal/templatetags/user_actions.py

@@ -2,6 +2,7 @@ from django import template
 from django.urls import reverse
 
 from journal.models import Collection, Like
+from takahe.utils import Takahe
 
 register = template.Library()
 
@@ -22,10 +23,9 @@ def wish_item_action(context, item):
 def like_piece_action(context, piece):
     user = context["request"].user
     action = {}
-    if user and user.is_authenticated:
+    if user and user.is_authenticated and piece and piece.post_id:
         action = {
-            "taken": piece.owner == user
+            "taken": Takahe.post_liked_by(piece.post_id, user),
-            or Like.objects.filter(target=piece, owner=user).first() is not None,
             "url": reverse("journal:like", args=[piece.uuid]),
         }
     return action
@@ -34,4 +34,9 @@ def like_piece_action(context, piece):
 @register.simple_tag(takes_context=True)
 def liked_piece(context, piece):
     user = context["request"].user
-    return user and user.is_authenticated and Like.user_liked_piece(user, piece)
+    return (
+        user
+        and user.is_authenticated
+        and piece.post_id
+        and Takahe.get_user_interaction(piece.post_id, user, "like")
+    )

journal/tests.py

@@ -9,15 +9,16 @@ from .models import *
 
 
 class CollectionTest(TestCase):
+    databases = "__all__"
+
     def setUp(self):
         self.book1 = Edition.objects.create(title="Hyperion")
         self.book2 = Edition.objects.create(title="Andymion")
-        self.user = User.register(email="a@b.com")
+        self.user = User.register(email="a@b.com", username="user")
-        pass
 
     def test_collection(self):
-        collection = Collection.objects.create(title="test", owner=self.user)
+        Collection.objects.create(title="test", owner=self.user.identity)
-        collection = Collection.objects.filter(title="test", owner=self.user).first()
+        collection = Collection.objects.get(title="test", owner=self.user.identity)
         self.assertEqual(collection.catalog_item.title, "test")
         member1 = collection.append_item(self.book1)
         member1.note = "my notes"
@@ -38,13 +39,15 @@ class CollectionTest(TestCase):
 
 
 class ShelfTest(TestCase):
+    databases = "__all__"
+
     def setUp(self):
         pass
 
     def test_shelf(self):
-        user = User.register(mastodon_site="site", mastodon_username="name")
+        user = User.register(email="a@b.com", username="user")
-        shelf_manager = ShelfManager(user=user)
+        shelf_manager = user.identity.shelf_manager
-        self.assertEqual(user.shelf_set.all().count(), 3)
+        self.assertEqual(len(shelf_manager.shelf_list.items()), 3)
         book1 = Edition.objects.create(title="Hyperion")
         book2 = Edition.objects.create(title="Andymion")
         q1 = shelf_manager.get_shelf(ShelfType.WISHLIST)
@@ -64,90 +67,86 @@ class ShelfTest(TestCase):
         self.assertEqual(q2.members.all().count(), 1)
         log = shelf_manager.get_log_for_item(book1)
         self.assertEqual(log.count(), 2)
-        self.assertEqual(log.last().metadata, {})
+        last_log = log.last()
+        self.assertEqual(last_log.metadata if last_log else 42, {})
         shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={"progress": 1})
         time.sleep(0.001)
         self.assertEqual(q1.members.all().count(), 1)
         self.assertEqual(q2.members.all().count(), 1)
         log = shelf_manager.get_log_for_item(book1)
         self.assertEqual(log.count(), 3)
-        self.assertEqual(log.last().metadata, {"progress": 1})
+        last_log = log.last()
+        self.assertEqual(last_log.metadata if last_log else 42, {"progress": 1})
         shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={"progress": 1})
         time.sleep(0.001)
         log = shelf_manager.get_log_for_item(book1)
         self.assertEqual(log.count(), 3)
-        self.assertEqual(log.last().metadata, {"progress": 1})
+        last_log = log.last()
+        self.assertEqual(last_log.metadata if last_log else 42, {"progress": 1})
         shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={"progress": 10})
         time.sleep(0.001)
         log = shelf_manager.get_log_for_item(book1)
         self.assertEqual(log.count(), 4)
-        self.assertEqual(log.last().metadata, {"progress": 10})
+
+        last_log = log.last()
+        self.assertEqual(last_log.metadata if last_log else 42, {"progress": 10})
         shelf_manager.move_item(book1, ShelfType.PROGRESS)
         time.sleep(0.001)
         log = shelf_manager.get_log_for_item(book1)
         self.assertEqual(log.count(), 4)
-        self.assertEqual(log.last().metadata, {"progress": 10})
+        last_log = log.last()
+        self.assertEqual(last_log.metadata if last_log else 42, {"progress": 10})
         shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={"progress": 90})
         time.sleep(0.001)
         log = shelf_manager.get_log_for_item(book1)
         self.assertEqual(log.count(), 5)
-        self.assertEqual(Mark(user, book1).visibility, 0)
+        self.assertEqual(Mark(user.identity, book1).visibility, 0)
         shelf_manager.move_item(
             book1, ShelfType.PROGRESS, metadata={"progress": 90}, visibility=1
         )
         time.sleep(0.001)
-        self.assertEqual(Mark(user, book1).visibility, 1)
+        self.assertEqual(Mark(user.identity, book1).visibility, 1)
         self.assertEqual(shelf_manager.get_log_for_item(book1).count(), 5)
 
-        # test silence mark mode -> no log
+        # test delete mark ->  one more log
-        shelf_manager.move_item(book1, ShelfType.WISHLIST, silence=True)
+        Mark(user.identity, book1).delete()
-        self.assertEqual(log.count(), 5)
+        self.assertEqual(log.count(), 6)
-        shelf_manager.move_item(book1, ShelfType.PROGRESS, silence=True)
-        self.assertEqual(log.count(), 5)
-        # test delete one log
-        first_log = log.first()
-        Mark(user, book1).delete_log(first_log.id)
-        self.assertEqual(log.count(), 4)
-        # # test delete mark -> leave one log: 移除标记
-        # Mark(user, book1).delete()
-        # self.assertEqual(log.count(), 1)
-        # # test delete all logs
-        # shelf_manager.move_item(book1, ShelfType.PROGRESS)
-        # self.assertEqual(log.count(), 2)
-        # Mark(user, book1).delete(silence=True)
-        # self.assertEqual(log.count(), 0)
 
 
 class TagTest(TestCase):
+    databases = "__all__"
+
     def setUp(self):
         self.book1 = Edition.objects.create(title="Hyperion")
         self.book2 = Edition.objects.create(title="Andymion")
-        self.movie1 = Edition.objects.create(title="Hyperion, The Movie")
+        self.movie1 = Edition.objects.create(title="Fight Club")
-        self.user1 = User.register(mastodon_site="site", mastodon_username="name")
+        self.user1 = User.register(email="a@b.com", username="user")
-        self.user2 = User.register(mastodon_site="site2", mastodon_username="name2")
+        self.user2 = User.register(email="x@b.com", username="user2")
-        self.user3 = User.register(mastodon_site="site2", mastodon_username="name3")
+        self.user3 = User.register(email="y@b.com", username="user3")
         pass
 
     def test_user_tag(self):
         t1 = "tag 1"
         t2 = "tag 2"
         t3 = "tag 3"
-        TagManager.tag_item_by_user(self.book1, self.user2, [t1, t3])
+        TagManager.tag_item(self.book1, self.user2.identity, [t1, t3])
         self.assertEqual(self.book1.tags, [t1, t3])
-        TagManager.tag_item_by_user(self.book1, self.user2, [t2, t3])
+        TagManager.tag_item(self.book1, self.user2.identity, [t2, t3])
         self.assertEqual(self.book1.tags, [t2, t3])
 
 
 class MarkTest(TestCase):
+    databases = "__all__"
+
     def setUp(self):
         self.book1 = Edition.objects.create(title="Hyperion")
-        self.user1 = User.register(mastodon_site="site", mastodon_username="name")
+        self.user1 = User.register(email="a@b.com", username="user")
         pref = self.user1.preference
         pref.default_visibility = 2
         pref.save()
 
     def test_mark(self):
-        mark = Mark(self.user1, self.book1)
+        mark = Mark(self.user1.identity, self.book1)
         self.assertEqual(mark.shelf_type, None)
         self.assertEqual(mark.shelf_label, None)
         self.assertEqual(mark.comment_text, None)
@@ -157,7 +156,7 @@ class MarkTest(TestCase):
         self.assertEqual(mark.tags, [])
         mark.update(ShelfType.WISHLIST, "a gentle comment", 9, 1)
 
-        mark = Mark(self.user1, self.book1)
+        mark = Mark(self.user1.identity, self.book1)
         self.assertEqual(mark.shelf_type, ShelfType.WISHLIST)
         self.assertEqual(mark.shelf_label, "想读的书")
         self.assertEqual(mark.comment_text, "a gentle comment")
@@ -166,10 +165,17 @@ class MarkTest(TestCase):
         self.assertEqual(mark.review, None)
         self.assertEqual(mark.tags, [])
 
-        review = Review.review_item_by_user(self.book1, self.user1, "Critic", "Review")
+    def test_review(self):
-        mark = Mark(self.user1, self.book1)
+        review = Review.update_item_review(
+            self.book1, self.user1.identity, "Critic", "Review"
+        )
+        mark = Mark(self.user1.identity, self.book1)
         self.assertEqual(mark.review, review)
+        Review.update_item_review(self.book1, self.user1.identity, None, None)
+        mark = Mark(self.user1.identity, self.book1)
+        self.assertIsNone(mark.review)
 
-        TagManager.tag_item_by_user(self.book1, self.user1, [" Sci-Fi ", " fic "])
+    def test_tag(self):
-        mark = Mark(self.user1, self.book1)
+        TagManager.tag_item(self.book1, self.user1.identity, [" Sci-Fi ", " fic "])
+        mark = Mark(self.user1.identity, self.book1)
         self.assertEqual(mark.tags, ["Sci-Fi", "fic"])

journal/views/collection.py

@@ -1,28 +1,28 @@
 from django.contrib.auth.decorators import login_required
 from django.core.exceptions import BadRequest, ObjectDoesNotExist, PermissionDenied
-from django.http import Http404, HttpResponse, HttpResponseRedirect
+from django.http import Http404, HttpRequest, HttpResponse, HttpResponseRedirect
 from django.shortcuts import get_object_or_404, redirect, render
 from django.urls import reverse
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
-from catalog.models import *
+from catalog.models import Item
-from common.utils import PageLinksGenerator, get_uuid_or_404
+from common.utils import AuthedHttpRequest, get_uuid_or_404
-from journal.models.renderers import convert_leading_space_in_md
 from mastodon.api import share_collection
 from users.models import User
+from users.models.apidentity import APIdentity
 from users.views import render_user_blocked, render_user_not_found
 
 from ..forms import *
 from ..models import *
-from .common import render_relogin
+from .common import render_relogin, target_identity_required
 
 
 @login_required
-def add_to_collection(request, item_uuid):
+def add_to_collection(request: AuthedHttpRequest, item_uuid):
     item = get_object_or_404(Item, uid=get_uuid_or_404(item_uuid))
     if request.method == "GET":
-        collections = Collection.objects.filter(owner=request.user)
+        collections = Collection.objects.filter(owner=request.user.identity)
         return render(
             request,
             "add_to_collection.html",
@@ -35,14 +35,14 @@ def add_to_collection(request, item_uuid):
         cid = int(request.POST.get("collection_id", default=0))
         if not cid:
             cid = Collection.objects.create(
-                owner=request.user, title=f"{request.user.display_name}的收藏单"
+                owner=request.user.identity, title=f"{request.user.display_name}的收藏单"
             ).id
-        collection = Collection.objects.get(owner=request.user, id=cid)
+        collection = Collection.objects.get(owner=request.user.identity, id=cid)
         collection.append_item(item, note=request.POST.get("note"))
-        return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
+        return HttpResponseRedirect(request.META.get("HTTP_REFERER", "/"))
 
 
-def collection_retrieve(request, collection_uuid):
+def collection_retrieve(request: AuthedHttpRequest, collection_uuid):
     collection = get_object_or_404(Collection, uid=get_uuid_or_404(collection_uuid))
     if not collection.is_visible_to(request.user):
         raise PermissionDenied()
@@ -53,19 +53,19 @@ def collection_retrieve(request, collection_uuid):
         else False
     )
     featured_since = (
-        collection.featured_by_user_since(request.user)
+        collection.featured_since(request.user.identity)
         if request.user.is_authenticated
         else None
     )
     available_as_featured = (
         request.user.is_authenticated
-        and (following or request.user == collection.owner)
+        and (following or request.user.identity == collection.owner)
         and not featured_since
         and collection.members.all().exists()
     )
     stats = {}
     if featured_since:
-        stats = collection.get_stats_for_user(request.user)
+        stats = collection.get_stats(request.user.identity)
         stats["wishlist_deg"] = (
             round(stats["wishlist"] / stats["total"] * 360) if stats["total"] else 0
         )
@@ -90,33 +90,35 @@ def collection_retrieve(request, collection_uuid):
 
 
 @login_required
-def collection_add_featured(request, collection_uuid):
+def collection_add_featured(request: AuthedHttpRequest, collection_uuid):
     if request.method != "POST":
         raise BadRequest()
     collection = get_object_or_404(Collection, uid=get_uuid_or_404(collection_uuid))
     if not collection.is_visible_to(request.user):
         raise PermissionDenied()
-    FeaturedCollection.objects.update_or_create(owner=request.user, target=collection)
+    FeaturedCollection.objects.update_or_create(
-    return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
+        owner=request.user.identity, target=collection
+    )
+    return HttpResponseRedirect(request.META.get("HTTP_REFERER", "/"))
 
 
 @login_required
-def collection_remove_featured(request, collection_uuid):
+def collection_remove_featured(request: AuthedHttpRequest, collection_uuid):
     if request.method != "POST":
         raise BadRequest()
     collection = get_object_or_404(Collection, uid=get_uuid_or_404(collection_uuid))
     if not collection.is_visible_to(request.user):
         raise PermissionDenied()
     fc = FeaturedCollection.objects.filter(
-        owner=request.user, target=collection
+        owner=request.user.identity, target=collection
     ).first()
     if fc:
         fc.delete()
-    return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
+    return HttpResponseRedirect(request.META.get("HTTP_REFERER", "/"))
 
 
 @login_required
-def collection_share(request, collection_uuid):
+def collection_share(request: AuthedHttpRequest, collection_uuid):
     collection = (
         get_object_or_404(Collection, uid=get_uuid_or_404(collection_uuid))
         if collection_uuid
@@ -130,14 +132,16 @@ def collection_share(request, collection_uuid):
         visibility = int(request.POST.get("visibility", default=0))
         comment = request.POST.get("comment")
         if share_collection(collection, comment, request.user, visibility):
-            return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
+            return HttpResponseRedirect(request.META.get("HTTP_REFERER", "/"))
         else:
             return render_relogin(request)
     else:
         raise BadRequest()
 
 
-def collection_retrieve_items(request, collection_uuid, edit=False, msg=None):
+def collection_retrieve_items(
+    request: AuthedHttpRequest, collection_uuid, edit=False, msg=None
+):
     collection = get_object_or_404(Collection, uid=get_uuid_or_404(collection_uuid))
     if not collection.is_visible_to(request.user):
         raise PermissionDenied()
@@ -155,7 +159,7 @@ def collection_retrieve_items(request, collection_uuid, edit=False, msg=None):
 
 
 @login_required
-def collection_append_item(request, collection_uuid):
+def collection_append_item(request: AuthedHttpRequest, collection_uuid):
     if request.method != "POST":
         raise BadRequest()
     collection = get_object_or_404(Collection, uid=get_uuid_or_404(collection_uuid))
@@ -175,7 +179,7 @@ def collection_append_item(request, collection_uuid):
 
 
 @login_required
-def collection_remove_item(request, collection_uuid, item_uuid):
+def collection_remove_item(request: AuthedHttpRequest, collection_uuid, item_uuid):
     if request.method != "POST":
         raise BadRequest()
     collection = get_object_or_404(Collection, uid=get_uuid_or_404(collection_uuid))
@@ -187,7 +191,9 @@ def collection_remove_item(request, collection_uuid, item_uuid):
 
 
 @login_required
-def collection_move_item(request, direction, collection_uuid, item_uuid):
+def collection_move_item(
+    request: AuthedHttpRequest, direction, collection_uuid, item_uuid
+):
     if request.method != "POST":
         raise BadRequest()
     collection = get_object_or_404(Collection, uid=get_uuid_or_404(collection_uuid))
@@ -202,7 +208,7 @@ def collection_move_item(request, direction, collection_uuid, item_uuid):
 
 
 @login_required
-def collection_update_member_order(request, collection_uuid):
+def collection_update_member_order(request: AuthedHttpRequest, collection_uuid):
     if request.method != "POST":
         raise BadRequest()
     collection = get_object_or_404(Collection, uid=get_uuid_or_404(collection_uuid))
@@ -217,7 +223,7 @@ def collection_update_member_order(request, collection_uuid):
 
 
 @login_required
-def collection_update_item_note(request, collection_uuid, item_uuid):
+def collection_update_item_note(request: AuthedHttpRequest, collection_uuid, item_uuid):
     collection = get_object_or_404(Collection, uid=get_uuid_or_404(collection_uuid))
     if not collection.is_editable_by(request.user):
         raise PermissionDenied()
@@ -241,7 +247,7 @@ def collection_update_item_note(request, collection_uuid, item_uuid):
 
 
 @login_required
-def collection_edit(request, collection_uuid=None):
+def collection_edit(request: AuthedHttpRequest, collection_uuid=None):
     collection = (
         get_object_or_404(Collection, uid=get_uuid_or_404(collection_uuid))
         if collection_uuid
@@ -259,7 +265,7 @@ def collection_edit(request, collection_uuid=None):
             {
                 "form": form,
                 "collection": collection,
-                "user": collection.owner if collection else request.user,
+                "user": collection.owner.user if collection else request.user,
             },
         )
     elif request.method == "POST":
@@ -270,7 +276,7 @@ def collection_edit(request, collection_uuid=None):
         )
         if form.is_valid():
             if not collection:
-                form.instance.owner = request.user
+                form.instance.owner = request.user.identity
             form.instance.edited_time = timezone.now()
             form.save()
             return redirect(
@@ -283,47 +289,34 @@ def collection_edit(request, collection_uuid=None):
 
 
 @login_required
-def user_collection_list(request, user_name):
+@target_identity_required
-    user = User.get(user_name)
+def user_collection_list(request: AuthedHttpRequest, user_name):
-    if user is None:
+    target = request.target_identity
-        return render_user_not_found(request)
+    collections = Collection.objects.filter(owner=target).filter(
-    if user != request.user and (
+        q_owned_piece_visible_to_user(request.user, target)
-        request.user.is_blocked_by(user) or request.user.is_blocking(user)
+    )
-    ):
-        return render_user_blocked(request)
-    collections = Collection.objects.filter(owner=user)
-    if user != request.user:
-        if request.user.is_following(user):
-            collections = collections.filter(visibility__in=[0, 1])
-        else:
-            collections = collections.filter(visibility=0)
     return render(
         request,
         "user_collection_list.html",
         {
-            "user": user,
+            "user": target.user,
             "collections": collections,
         },
     )
 
 
 @login_required
-def user_liked_collection_list(request, user_name):
+@target_identity_required
-    user = User.get(user_name)
+def user_liked_collection_list(request: AuthedHttpRequest, user_name):
-    if user is None:
+    target = request.target_identity
-        return render_user_not_found(request)
+    collections = Collection.objects.filter(likes__owner=target)
-    if user != request.user and (
+    if target.user != request.user:
-        request.user.is_blocked_by(user) or request.user.is_blocking(user)
+        collections = collections.filter(q_piece_visible_to_user(request.user))
-    ):
-        return render_user_blocked(request)
-    collections = Collection.objects.filter(likes__owner=user)
-    if user != request.user:
-        collections = collections.filter(query_visible(request.user))
     return render(
         request,
         "user_collection_list.html",
         {
-            "user": user,
+            "user": target.user,
             "collections": collections,
             "liked": True,
         },

journal/views/common.py

@@ -1,3 +1,5 @@
+import functools
+
 from django.contrib.auth.decorators import login_required
 from django.core.exceptions import BadRequest, ObjectDoesNotExist, PermissionDenied
 from django.core.paginator import Paginator
@@ -6,8 +8,8 @@ from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 
 from catalog.models import *
-from common.utils import PageLinksGenerator, get_uuid_or_404
+from common.utils import AuthedHttpRequest, PageLinksGenerator, get_uuid_or_404
-from users.models import User
+from users.models import APIdentity
 from users.views import render_user_blocked, render_user_not_found
 
 from ..forms import *
@@ -16,6 +18,25 @@ from ..models import *
 PAGE_SIZE = 10
 
 
+def target_identity_required(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        request = kwargs["request"]
+        handler = kwargs["user_name"]
+        try:
+            target = APIdentity.get_by_handler(handler)
+        except:
+            return render_user_not_found(request)
+        if not target.is_visible_to_user(request.user):
+            return render_user_blocked(request)
+        request.target_identity = target
+        # request.identity = (
+        #     request.user.identity if request.user.is_authenticated else None
+        # )
+
+    return wrapper
+
+
 def render_relogin(request):
     return render(
         request,
@@ -41,42 +62,45 @@ def render_list_not_found(request):
     )
 
 
+@login_required
+@target_identity_required
 def render_list(
-    request, user_name, type, shelf_type=None, item_category=None, tag_title=None
+    request: AuthedHttpRequest,
+    user_name,
+    type,
+    shelf_type=None,
+    item_category=None,
+    tag_title=None,
 ):
-    user = User.get(user_name)
+    target = request.target_identity
-    if user is None:
+    viewer = request.user.identity
-        return render_user_not_found(request)
-    if user != request.user and (
-        request.user.is_blocked_by(user) or request.user.is_blocking(user)
-    ):
-        return render_user_blocked(request)
     tag = None
     if type == "mark":
-        queryset = user.shelf_manager.get_latest_members(shelf_type, item_category)
+        queryset = target.user.shelf_manager.get_latest_members(
+            shelf_type, item_category
+        )
     elif type == "tagmember":
-        tag = Tag.objects.filter(owner=user, title=tag_title).first()
+        tag = Tag.objects.filter(owner=target, title=tag_title).first()
         if not tag:
             return render_list_not_found(request)
-        if tag.visibility != 0 and user != request.user:
+        if tag.visibility != 0 and target != viewer:
             return render_list_not_found(request)
         queryset = TagMember.objects.filter(parent=tag)
-    elif type == "review":
+    elif type == "review" and item_category:
-        queryset = Review.objects.filter(owner=user)
+        queryset = Review.objects.filter(q_item_in_category(item_category))
-        queryset = queryset.filter(query_item_category(item_category))
     else:
         raise BadRequest()
-    queryset = queryset.filter(q_visible_to(request.user, user)).order_by(
+    queryset = queryset.filter(
-        "-created_time"
+        q_owned_piece_visible_to_user(request.user, target)
-    )
+    ).order_by("-created_time")
     paginator = Paginator(queryset, PAGE_SIZE)
-    page_number = request.GET.get("page", default=1)
+    page_number = int(request.GET.get("page", default=1))
     members = paginator.get_page(page_number)
     pagination = PageLinksGenerator(PAGE_SIZE, page_number, paginator.num_pages)
     return render(
         request,
         f"user_{type}_list.html",
-        {"user": user, "members": members, "tag": tag, "pagination": pagination},
+        {"user": target.user, "members": members, "tag": tag, "pagination": pagination},
     )
 
 

journal/views/mark.py

@@ -12,17 +12,18 @@ from django.utils.dateparse import parse_datetime
 from django.utils.translation import gettext_lazy as _
 
 from catalog.models import *
-from common.utils import PageLinksGenerator, get_uuid_or_404
+from common.utils import AuthedHttpRequest, PageLinksGenerator, get_uuid_or_404
 from mastodon.api import (
     get_spoiler_text,
     get_status_id_by_url,
     get_visibility,
     post_toot,
 )
+from takahe.utils import Takahe
 
 from ..forms import *
 from ..models import *
-from .common import render_list, render_relogin
+from .common import render_list, render_relogin, target_identity_required
 
 _logger = logging.getLogger(__name__)
 PAGE_SIZE = 10
@@ -31,28 +32,29 @@ _checkmark = "✔️".encode("utf-8")
 
 
 @login_required
-def wish(request, item_uuid):
+def wish(request: AuthedHttpRequest, item_uuid):
     if request.method != "POST":
         raise BadRequest()
     item = get_object_or_404(Item, uid=get_uuid_or_404(item_uuid))
     if not item:
         raise Http404()
-    request.user.shelf_manager.move_item(item, ShelfType.WISHLIST)
+    request.user.identity.shelf_manager.move_item(item, ShelfType.WISHLIST)
     if request.GET.get("back"):
-        return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
+        return HttpResponseRedirect(request.META.get("HTTP_REFERER", "/"))
     return HttpResponse(_checkmark)
 
 
 @login_required
-def like(request, piece_uuid):
+def like(request: AuthedHttpRequest, piece_uuid):
     if request.method != "POST":
         raise BadRequest()
     piece = get_object_or_404(Piece, uid=get_uuid_or_404(piece_uuid))
     if not piece:
         raise Http404()
-    Like.user_like_piece(request.user, piece)
+    if piece.post_id:
+        Takahe.like_post(piece.post_id, request.user.identity.pk)
     if request.GET.get("back"):
-        return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
+        return HttpResponseRedirect(request.META.get("HTTP_REFERER", "/"))
     elif request.GET.get("stats"):
         return render(
             request,
@@ -68,15 +70,16 @@ def like(request, piece_uuid):
 
 
 @login_required
-def unlike(request, piece_uuid):
+def unlike(request: AuthedHttpRequest, piece_uuid):
     if request.method != "POST":
         raise BadRequest()
     piece = get_object_or_404(Piece, uid=get_uuid_or_404(piece_uuid))
     if not piece:
         raise Http404()
-    Like.user_unlike_piece(request.user, piece)
+    if piece.post_id:
+        Takahe.unlike_post(piece.post_id, request.user.identity.pk)
     if request.GET.get("back"):
-        return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
+        return HttpResponseRedirect(request.META.get("HTTP_REFERER", "/"))
     elif request.GET.get("stats"):
         return render(
             request,
@@ -92,11 +95,11 @@ def unlike(request, piece_uuid):
 
 
 @login_required
-def mark(request, item_uuid):
+def mark(request: AuthedHttpRequest, item_uuid):
     item = get_object_or_404(Item, uid=get_uuid_or_404(item_uuid))
-    mark = Mark(request.user, item)
+    mark = Mark(request.user.identity, item)
     if request.method == "GET":
-        tags = TagManager.get_item_tags_by_user(item, request.user)
+        tags = request.user.identity.tag_manager.get_item_tags(item)
         shelf_types = [
             (n[1], n[2]) for n in iter(ShelfTypeNames) if n[0] == item.category
         ]
@@ -115,15 +118,8 @@ def mark(request, item_uuid):
         )
     elif request.method == "POST":
         if request.POST.get("delete", default=False):
-            silence = request.POST.get("silence", False)
+            mark.delete()
-            mark.delete(silence=silence)
+            return HttpResponseRedirect(request.META.get("HTTP_REFERER", "/"))
-            if (
-                silence
-            ):  # this means the mark is deleted from mark_history, thus redirect to item page
-                return redirect(
-                    reverse("catalog:retrieve", args=[item.url_path, item.uuid])
-                )
-            return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
         else:
             visibility = int(request.POST.get("visibility", default=0))
             rating_grade = request.POST.get("rating_grade", default=0)
@@ -143,7 +139,7 @@ def mark(request, item_uuid):
                 )
                 if mark_date and mark_date >= timezone.now():
                     mark_date = None
-            TagManager.tag_item_by_user(item, request.user, tags, visibility)
+            TagManager.tag_item(item, request.user.identity, tags, visibility)
             try:
                 mark.update(
                     status,
@@ -167,7 +163,7 @@ def mark(request, item_uuid):
                         "secondary_msg": err,
                     },
                 )
-            return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
+            return HttpResponseRedirect(request.META.get("HTTP_REFERER", "/"))
     raise BadRequest()
 
 
@@ -202,12 +198,12 @@ def share_comment(user, item, text, visibility, shared_link=None, position=None)
 
 
 @login_required
-def mark_log(request, item_uuid, log_id):
+def mark_log(request: AuthedHttpRequest, item_uuid, log_id):
     """
     Delete log of one item by log id.
     """
     item = get_object_or_404(Item, uid=get_uuid_or_404(item_uuid))
-    mark = Mark(request.user, item)
+    mark = Mark(request.user.identity, item)
     if request.method == "POST":
         if request.GET.get("delete", default=False):
             if log_id:
@@ -219,7 +215,7 @@ def mark_log(request, item_uuid, log_id):
 
 
 @login_required
-def comment(request, item_uuid):
+def comment(request: AuthedHttpRequest, item_uuid):
     item = get_object_or_404(Item, uid=get_uuid_or_404(item_uuid))
     if not item.class_name in ["podcastepisode", "tvepisode"]:
         raise BadRequest("不支持评论此类型的条目")
@@ -246,7 +242,7 @@ def comment(request, item_uuid):
             if not comment:
                 raise Http404()
             comment.delete()
-            return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
+            return HttpResponseRedirect(request.META.get("HTTP_REFERER", "/"))
         visibility = int(request.POST.get("visibility", default=0))
         text = request.POST.get("text")
         position = None
@@ -302,12 +298,11 @@ def comment(request, item_uuid):
         #     )
         if post_error:
             return render_relogin(request)
-        return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
+        return HttpResponseRedirect(request.META.get("HTTP_REFERER", "/"))
     raise BadRequest()
 
 
-@login_required
+def user_mark_list(request: AuthedHttpRequest, user_name, shelf_type, item_category):
-def user_mark_list(request, user_name, shelf_type, item_category):
     return render_list(
         request, user_name, "mark", shelf_type=shelf_type, item_category=item_category
     )

journal/views/profile.py

@@ -6,30 +6,32 @@ from django.utils.translation import gettext_lazy as _
 from user_messages import api as msg
 
 from catalog.models import *
-from users.models import User
+from common.utils import AuthedHttpRequest
+from users.models import APIdentity, User
 from users.views import render_user_blocked, render_user_not_found
 
 from ..forms import *
 from ..models import *
-from .common import render_list
+from .common import render_list, target_identity_required
 
 
-def profile(request, user_name):
+@target_identity_required
+def profile(request: AuthedHttpRequest, user_name):
     if request.method != "GET":
         raise BadRequest()
-    user = User.get(user_name, case_sensitive=True)
+    target = request.target_identity
-    if user is None or not user.is_active:
+    # if user.mastodon_acct != user_name and user.username != user_name:
-        return render_user_not_found(request)
+    #     return redirect(user.url)
-    if user.mastodon_acct != user_name and user.username != user_name:
+    if not request.user.is_authenticated and target.preference.no_anonymous_view:
-        return redirect(user.url)
+        return render(request, "users/home_anonymous.html", {"user": target.user})
-    if not request.user.is_authenticated and user.preference.no_anonymous_view:
+    me = target.user == request.user
-        return render(request, "users/home_anonymous.html", {"user": user})
+    if not me and (
-    if user != request.user and (
+        target.is_blocked_by(request.user.identity)
-        user.is_blocked_by(request.user) or user.is_blocking(request.user)
+        or target.is_blocking(request.user.identity)
     ):
         return render_user_blocked(request)
 
-    qv = q_visible_to(request.user, user)
+    qv = q_owned_piece_visible_to_user(request.user, target)
     shelf_list = {}
     visbile_categories = [
         ItemCategory.Book,
@@ -43,9 +45,9 @@ def profile(request, user_name):
     for category in visbile_categories:
         shelf_list[category] = {}
         for shelf_type in ShelfType:
-            label = user.shelf_manager.get_label(shelf_type, category)
+            label = target.shelf_manager.get_label(shelf_type, category)
             if label is not None:
-                members = user.shelf_manager.get_latest_members(
+                members = target.shelf_manager.get_latest_members(
                     shelf_type, category
                 ).filter(qv)
                 shelf_list[category][shelf_type] = {
@@ -53,35 +55,32 @@ def profile(request, user_name):
                     "count": members.count(),
                     "members": members[:10].prefetch_related("item"),
                 }
-        reviews = (
+        reviews = Review.objects.filter(q_item_in_category(category)).order_by(
-            Review.objects.filter(owner=user)
+            "-created_time"
-            .filter(qv)
-            .filter(query_item_category(category))
-            .order_by("-created_time")
         )
         shelf_list[category]["reviewed"] = {
             "title": "评论过的" + category.label,
             "count": reviews.count(),
             "members": reviews[:10].prefetch_related("item"),
         }
-    collections = (
+    collections = Collection.objects.filter(qv).order_by("-created_time")
-        Collection.objects.filter(owner=user).filter(qv).order_by("-created_time")
-    )
     liked_collections = (
-        Like.user_likes_by_class(user, Collection)
+        Like.user_likes_by_class(target, Collection)
         .order_by("-edited_time")
         .values_list("target_id", flat=True)
     )
-    if user != request.user:
+    if not me:
-        liked_collections = liked_collections.filter(query_visible(request.user))
+        liked_collections = liked_collections.filter(
-        top_tags = user.tag_manager.public_tags[:10]
+            q_piece_visible_to_user(request.user)
+        )
+        top_tags = target.tag_manager.public_tags[:10]
     else:
-        top_tags = user.tag_manager.all_tags[:10]
+        top_tags = target.tag_manager.all_tags[:10]
     return render(
         request,
         "profile.html",
         {
-            "user": user,
+            "user": target.user,
             "top_tags": top_tags,
             "shelf_list": shelf_list,
             "collections": collections[:10],
@@ -91,7 +90,7 @@ def profile(request, user_name):
                 for i in liked_collections.order_by("-edited_time")[:10]
             ],
             "liked_collections_count": liked_collections.count(),
-            "layout": user.preference.profile_layout,
+            "layout": target.preference.profile_layout,
         },
     )
 
@@ -102,7 +101,7 @@ def user_calendar_data(request, user_name):
     user = User.get(user_name)
     if user is None or not request.user.is_authenticated:
         return HttpResponse("")
-    max_visiblity = max_visiblity_to(request.user, user)
+    max_visiblity = max_visiblity_to_user(request.user, user.identity)
     calendar_data = user.shelf_manager.get_calendar_data(max_visiblity)
     return render(
         request,

journal/views/review.py

@@ -12,9 +12,11 @@ from django.utils.dateparse import parse_datetime
 from django.utils.translation import gettext_lazy as _
 
 from catalog.models import *
-from common.utils import PageLinksGenerator, get_uuid_or_404
+from common.utils import AuthedHttpRequest, PageLinksGenerator, get_uuid_or_404
 from journal.models.renderers import convert_leading_space_in_md, render_md
+from mastodon.api import share_review
 from users.models import User
+from users.models.apidentity import APIdentity
 
 from ..forms import *
 from ..models import *
@@ -32,7 +34,7 @@ def review_retrieve(request, review_uuid):
 
 
 @login_required
-def review_edit(request, item_uuid, review_uuid=None):
+def review_edit(request: AuthedHttpRequest, item_uuid, review_uuid=None):
     item = get_object_or_404(Item, uid=get_uuid_or_404(item_uuid))
     review = (
         get_object_or_404(Review, uid=get_uuid_or_404(review_uuid))
@@ -65,24 +67,28 @@ def review_edit(request, item_uuid, review_uuid=None):
         if form.is_valid():
             mark_date = None
             if request.POST.get("mark_anotherday"):
-                dt = parse_datetime(request.POST.get("mark_date") + " 20:00:00")
+                dt = parse_datetime(request.POST.get("mark_date", "") + " 20:00:00")
                 mark_date = (
                     dt.replace(tzinfo=timezone.get_current_timezone()) if dt else None
                 )
             body = form.instance.body
             if request.POST.get("leading_space"):
                 body = convert_leading_space_in_md(body)
-            review = Review.review_item_by_user(
+            review = Review.update_item_review(
                 item,
-                request.user,
+                request.user.identity,
                 form.cleaned_data["title"],
                 body,
                 form.cleaned_data["visibility"],
                 mark_date,
-                form.cleaned_data["share_to_mastodon"],
             )
             if not review:
                 raise BadRequest()
+            if (
+                form.cleaned_data["share_to_mastodon"]
+                and request.user.mastodon_username
+            ):
+                share_review(review)
             return redirect(reverse("journal:review_retrieve", args=[review.uuid]))
         else:
             raise BadRequest()
@@ -90,7 +96,6 @@ def review_edit(request, item_uuid, review_uuid=None):
         raise BadRequest()
 
 
-@login_required
 def user_review_list(request, user_name, item_category):
     return render_list(request, user_name, "review", item_category=item_category)
 
@@ -100,16 +105,16 @@ MAX_ITEM_PER_TYPE = 10
 
 class ReviewFeed(Feed):
     def get_object(self, request, id):
-        return User.get(id)
+        return APIdentity.get_by_handler(id)
 
-    def title(self, user):
+    def title(self, owner):
-        return "%s的评论" % user.display_name if user else "无效链接"
+        return "%s的评论" % owner.display_name if owner else "无效链接"
 
-    def link(self, user):
+    def link(self, owner):
-        return user.url if user else settings.SITE_INFO["site_url"]
+        return owner.url if owner else settings.SITE_INFO["site_url"]
 
-    def description(self, user):
+    def description(self, owner):
-        return "%s的评论合集 - NeoDB" % user.display_name if user else "无效链接"
+        return "%s的评论合集 - NeoDB" % owner.display_name if owner else "无效链接"
 
     def items(self, user):
         if user is None or user.preference.no_anonymous_view:

journal/views/tag.py

@@ -13,29 +13,24 @@ from users.views import render_user_blocked, render_user_not_found
 
 from ..forms import *
 from ..models import *
-from .common import render_list
+from .common import render_list, target_identity_required
 
 PAGE_SIZE = 10
 
 
 @login_required
+@target_identity_required
 def user_tag_list(request, user_name):
-    user = User.get(user_name)
+    target = request.target
-    if user is None:
+    tags = Tag.objects.filter(owner=target)
-        return render_user_not_found(request)
+    if target.user != request.user:
-    if user != request.user and (
-        request.user.is_blocked_by(user) or request.user.is_blocking(user)
-    ):
-        return render_user_blocked(request)
-    tags = Tag.objects.filter(owner=user)
-    if user != request.user:
         tags = tags.filter(visibility=0)
     tags = tags.values("title").annotate(total=Count("members")).order_by("-total")
     return render(
         request,
         "user_tag_list.html",
         {
-            "user": user,
+            "user": target.user,
             "tags": tags,
         },
     )
@@ -47,7 +42,7 @@ def user_tag_edit(request):
         tag_title = Tag.cleanup_title(request.GET.get("tag", ""), replace=False)
         if not tag_title:
             raise Http404()
-        tag = Tag.objects.filter(owner=request.user, title=tag_title).first()
+        tag = Tag.objects.filter(owner=request.user.identity, title=tag_title).first()
         if not tag:
             raise Http404()
         return render(request, "tag_edit.html", {"tag": tag})
@@ -55,7 +50,7 @@ def user_tag_edit(request):
         tag_title = Tag.cleanup_title(request.POST.get("title", ""), replace=False)
         tag_id = request.POST.get("id")
         tag = (
-            Tag.objects.filter(owner=request.user, id=tag_id).first()
+            Tag.objects.filter(owner=request.user.identity, id=tag_id).first()
             if tag_id
             else None
         )
@@ -70,7 +65,9 @@ def user_tag_edit(request):
             )
         elif (
             tag_title != tag.title
-            and Tag.objects.filter(owner=request.user, title=tag_title).exists()
+            and Tag.objects.filter(
+                owner=request.user.identity, title=tag_title
+            ).exists()
         ):
             msg.error(request.user, _("标签已存在"))
             return HttpResponseRedirect(request.META.get("HTTP_REFERER"))
@@ -88,6 +85,5 @@ def user_tag_edit(request):
     raise BadRequest()
 
 
-@login_required
 def user_tag_member_list(request, user_name, tag_title):
     return render_list(request, user_name, "tagmember", tag_title=tag_title)

mastodon/api.py

@@ -1,5 +1,5 @@
 import functools
-import logging
+import html
 import random
 import re
 import string
@@ -193,7 +193,7 @@ def detect_server_info(login_domain):
     try:
         response = get(url, headers={"User-Agent": USER_AGENT})
     except Exception as e:
-        logger.error(f"Error connecting {login_domain} {e}")
+        logger.error(f"Error connecting {login_domain}: {e}")
         raise Exception(f"无法连接 {login_domain}")
     if response.status_code != 200:
         logger.error(f"Error connecting {login_domain}: {response.status_code}")
@@ -363,7 +363,7 @@ def get_visibility(visibility, user):
 def share_mark(mark):
     from catalog.common import ItemCategory
 
-    user = mark.owner
+    user = mark.owner.user
     if mark.visibility == 2:
         visibility = TootVisibilityEnum.DIRECT
     elif mark.visibility == 1:
@@ -466,10 +466,10 @@ def share_collection(collection, comment, user, visibility_no):
     )
     user_str = (
         "我"
-        if user == collection.owner
+        if user == collection.owner.user
         else (
-            " @" + collection.owner.mastodon_acct + " "
+            " @" + collection.owner.user.mastodon_acct + " "
-            if collection.owner.mastodon_acct
+            if collection.owner.user.mastodon_acct
             else " " + collection.owner.username + " "
         )
     )

pyproject.toml

@@ -1,5 +1,5 @@
 [tool.pyright]
-exclude = [ "media", ".venv", ".git", "playground", "**/tests.py", "neodb", "**/migrations", "**/commands", "**/sites/douban_*" ]
+exclude = [ "media", ".venv", ".git", "playground", "catalog/*/tests.py", "neodb", "**/migrations",  "**/sites/douban_*" ]
 
 [tool.djlint]
 ignore="T002,T003,H006,H019,H020,H021,H023,H030,H031"

requirements-dev.txt

@@ -4,5 +4,6 @@ django-debug-toolbar
 django-stubs
 djlint~=1.32.1
 isort~=5.12.0
+lxml-stubs
 pre-commit
 pyright==1.1.322

requirements.txt

@@ -1,8 +1,8 @@
+cachetools
 dateparser
 discord.py
 django~=4.2.4
 django-anymail
-django-auditlog
 django-auditlog @ git+https://github.com/jazzband/django-auditlog.git@45591463e8192b4ac0095e259cc4dcea0ac2fd6c
 django-bleach
 django-compressor
@@ -25,6 +25,7 @@ easy-thumbnails
 filetype
 fontawesomefree
 gunicorn
+httpx
 igdb-api-v4
 libsass
 listparser
@@ -41,3 +42,4 @@ rq>=1.12.0
 setproctitle
 tqdm
 typesense
+urlman

social/migrations/0007_alter_localactivity_owner.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.4 on 2023-08-09 13:26
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("users", "0012_apidentity"),
+        ("social", "0006_alter_localactivity_template"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="localactivity",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE, to="users.apidentity"
+            ),
+        ),
+    ]

social/models.py

@@ -27,7 +27,7 @@ from journal.models import (
     ShelfMember,
     UserOwnedObjectMixin,
 )
-from users.models import User
+from users.models import APIdentity
 
 _logger = logging.getLogger(__name__)
 
@@ -42,10 +42,8 @@ class ActivityTemplate(models.TextChoices):
 
 
 class LocalActivity(models.Model, UserOwnedObjectMixin):
-    owner = models.ForeignKey(User, on_delete=models.CASCADE)
+    owner = models.ForeignKey(APIdentity, on_delete=models.CASCADE)  # type: ignore
-    visibility = models.PositiveSmallIntegerField(
+    visibility = models.PositiveSmallIntegerField(default=0)  # type: ignore
-        default=0
-    )  # 0: Public / 1: Follower only / 2: Self only
     template = models.CharField(
         blank=False, choices=ActivityTemplate.choices, max_length=50
     )
@@ -62,11 +60,11 @@ class LocalActivity(models.Model, UserOwnedObjectMixin):
 
 
 class ActivityManager:
-    def __init__(self, user):
+    def __init__(self, owner: APIdentity):
-        self.owner = user
+        self.owner = owner
 
     def get_timeline(self, before_time=None):
-        following = [x for x in self.owner.following if x not in self.owner.ignoring]
+        following = [x for x in self.owner.following if x not in self.owner.muting]
         q = Q(owner_id__in=following, visibility__lt=2) | Q(owner=self.owner)
         if before_time:
             q = q & Q(created_time__lt=before_time)
@@ -205,5 +203,5 @@ class CommentChildItemProcessor(DefaultActivityProcessor):
             super().updated()
 
 
-def reset_social_visibility_for_user(user: User, visibility: int):
+def reset_social_visibility_for_user(owner: APIdentity, visibility: int):
-    LocalActivity.objects.filter(owner=user).update(visibility=visibility)
+    LocalActivity.objects.filter(owner=owner).update(visibility=visibility)

social/templates/activity/comment_child_item.html

@@ -53,7 +53,7 @@
     {% endif %}
   </span>
   <span>
-    <a {% if activity.action_object.metadata.shared_link %} href="{{ activity.action_object.metadata.shared_link }}" target="_blank" rel="noopener" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if activity.action_object.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %} "></i></a>
+    <a {% if activity.action_object.shared_link %} href="{{ activity.action_object.shared_link }}" target="_blank" rel="noopener" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if activity.action_object.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %} "></i></a>
   </span>
 </span>
 <div class="spacing">

social/templates/activity/mark_item.html

@@ -40,7 +40,7 @@
     {% endif %}
   </span>
   <span>
-    <a {% if activity.action_object.metadata.shared_link %} href="{{ activity.action_object.metadata.shared_link }}" target="_blank" rel="noopener" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if activity.action_object.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %} "></i></a>
+    <a {% if activity.action_object.shared_link %} href="{{ activity.action_object.shared_link }}" target="_blank" rel="noopener" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if activity.action_object.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %} "></i></a>
   </span>
 </span>
 <div class="spacing">

social/templates/activity/review_item.html

@@ -33,7 +33,7 @@
     {% endif %}
   </span>
   <span>
-    <a {% if activity.action_object.metadata.shared_link %} href="{{ activity.action_object.metadata.shared_link }}" target="_blank" rel="noopener" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if activity.action_object.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %} "></i></a>
+    <a {% if activity.action_object.shared_link %} href="{{ activity.action_object.shared_link }}" target="_blank" rel="noopener" title="打开联邦宇宙分享链接" {% else %} class="disabled" {% endif %}><i class="fa-solid {% if activity.action_object.visibility > 0 %} fa-lock {% else %} fa-globe {% endif %} "></i></a>
   </span>
 </span>
 <div class="spacing">

social/tests.py

@@ -2,65 +2,86 @@ from django.test import TestCase
 
 from catalog.models import *
 from journal.models import *
+from takahe.utils import Takahe
 from users.models import User
 
 from .models import *
 
 
 class SocialTest(TestCase):
+    databases = "__all__"
+
     def setUp(self):
         self.book1 = Edition.objects.create(title="Hyperion")
         self.book2 = Edition.objects.create(title="Andymion")
         self.movie = Edition.objects.create(title="Fight Club")
-        self.alice = User.register(mastodon_site="MySpace", mastodon_username="Alice")
+        self.alice = User.register(
-        self.bob = User.register(mastodon_site="KKCity", mastodon_username="Bob")
+            username="Alice", mastodon_site="MySpace", mastodon_username="Alice"
+        )
+        self.bob = User.register(
+            username="Bob", mastodon_site="KKCity", mastodon_username="Bob"
+        )
 
     def test_timeline(self):
+        alice_feed = self.alice.identity.activity_manager
+        bob_feed = self.bob.identity.activity_manager
+
         # alice see 0 activity in timeline in the beginning
-        timeline = self.alice.activity_manager.get_timeline()
+        self.assertEqual(len(alice_feed.get_timeline()), 0)
-        self.assertEqual(len(timeline), 0)
 
         # 1 activity after adding first book to shelf
-        self.alice.shelf_manager.move_item(self.book1, ShelfType.WISHLIST, visibility=1)
+        self.alice.identity.shelf_manager.move_item(
-        timeline = self.alice.activity_manager.get_timeline()
+            self.book1, ShelfType.WISHLIST, visibility=1
-        self.assertEqual(len(timeline), 1)
+        )
+        self.assertEqual(len(alice_feed.get_timeline()), 1)
 
         # 2 activities after adding second book to shelf
-        self.alice.shelf_manager.move_item(self.book2, ShelfType.WISHLIST)
+        self.alice.identity.shelf_manager.move_item(self.book2, ShelfType.WISHLIST)
-        timeline = self.alice.activity_manager.get_timeline()
+        self.assertEqual(len(alice_feed.get_timeline()), 2)
-        self.assertEqual(len(timeline), 2)
 
         # 2 activities after change first mark
-        self.alice.shelf_manager.move_item(self.book1, ShelfType.PROGRESS)
+        self.alice.identity.shelf_manager.move_item(self.book1, ShelfType.PROGRESS)
-        timeline = self.alice.activity_manager.get_timeline()
+        self.assertEqual(len(alice_feed.get_timeline()), 2)
-        self.assertEqual(len(timeline), 2)
 
         # bob see 0 activity in timeline in the beginning
-        timeline2 = self.bob.activity_manager.get_timeline()
+        self.assertEqual(len(bob_feed.get_timeline()), 0)
-        self.assertEqual(len(timeline2), 0)
 
         # bob follows alice, see 2 activities
-        self.bob.mastodon_following = ["Alice@MySpace"]
+        self.bob.identity.follow(self.alice.identity)
-        self.alice.mastodon_follower = ["Bob@KKCity"]
+        Takahe._force_state_cycle()
-        self.bob.merge_relationships()
+        self.assertEqual(len(bob_feed.get_timeline()), 2)
-        timeline2 = self.bob.activity_manager.get_timeline()
+
-        self.assertEqual(len(timeline2), 2)
+        # bob mute, then unmute alice
+        self.bob.identity.mute(self.alice.identity)
+        Takahe._force_state_cycle()
+        self.assertEqual(len(bob_feed.get_timeline()), 0)
+        self.bob.identity.unmute(self.alice.identity)
+        Takahe._force_state_cycle()
+        self.assertEqual(len(bob_feed.get_timeline()), 2)
 
         # alice:3 bob:2 after alice adding second book to shelf as private
-        self.alice.shelf_manager.move_item(self.movie, ShelfType.WISHLIST, visibility=2)
+        self.alice.identity.shelf_manager.move_item(
-        timeline = self.alice.activity_manager.get_timeline()
+            self.movie, ShelfType.WISHLIST, visibility=2
-        self.assertEqual(len(timeline), 3)
+        )
-        timeline2 = self.bob.activity_manager.get_timeline()
+        self.assertEqual(len(alice_feed.get_timeline()), 3)
-        self.assertEqual(len(timeline2), 2)
+        self.assertEqual(len(bob_feed.get_timeline()), 2)
 
-        # remote unfollow
+        # alice mute bob
-        self.bob.mastodon_following = []
+        self.alice.identity.mute(self.bob.identity)
-        self.alice.mastodon_follower = []
+        Takahe._force_state_cycle()
-        self.bob.merge_relationships()
+        self.assertEqual(len(bob_feed.get_timeline()), 2)
-        timeline = self.bob.activity_manager.get_timeline()
-        self.assertEqual(len(timeline), 0)
 
-        # local follow
+        # bob unfollow alice
-        self.bob.follow(self.alice)
+        self.bob.identity.unfollow(self.alice.identity)
-        timeline = self.bob.activity_manager.get_timeline()
+        Takahe._force_state_cycle()
-        self.assertEqual(len(timeline), 2)
+        self.assertEqual(len(bob_feed.get_timeline()), 0)
+
+        # bob follow alice
+        self.bob.identity.follow(self.alice.identity)
+        Takahe._force_state_cycle()
+        self.assertEqual(len(bob_feed.get_timeline()), 2)
+
+        # alice block bob
+        self.alice.identity.block(self.bob.identity)
+        Takahe._force_state_cycle()
+        self.assertEqual(len(bob_feed.get_timeline()), 0)

social/views.py

@@ -1,7 +1,6 @@
 import logging
 
-from django.conf import settings
+from django.contrib.auth.decorators import login_required
-from django.contrib.auth.decorators import login_required, permission_required
 from django.core.exceptions import BadRequest
 from django.shortcuts import render
 from django.utils.translation import gettext_lazy as _
@@ -65,7 +64,7 @@ def data(request):
         request,
         "feed_data.html",
         {
-            "activities": ActivityManager(request.user).get_timeline(
+            "activities": ActivityManager(request.user.identity).get_timeline(
                 before_time=request.GET.get("last")
             )[:PAGE_SIZE],
         },

takahe/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

takahe/ap_handlers.py

@@ -0,0 +1,123 @@
+from datetime import datetime
+
+from loguru import logger
+
+from catalog.common import *
+from journal.models import Comment, Piece, Rating, Review, ShelfMember
+from users.models import User as NeoUser
+
+from .models import Follow, Identity, Post
+from .utils import Takahe
+
+_supported_ap_catalog_item_types = [
+    "Edition",
+    "Movie",
+    "TVShow",
+    "TVSeason",
+    "TVEpisode",
+    "Album",
+    "Game",
+    "Podcast",
+    "Performance",
+    "PerformanceProduction",
+]
+
+_supported_ap_journal_types = {
+    "Status": ShelfMember,
+    "Rating": Rating,
+    "Comment": Comment,
+    "Review": Review,
+}
+
+
+def _parse_links(objects):
+    logger.debug(f"Parsing links from {objects}")
+    items = []
+    pieces = []
+    for obj in objects:
+        if obj["type"] in _supported_ap_catalog_item_types:
+            items.append(obj["url"])
+        elif obj["type"] in _supported_ap_journal_types.keys():
+            pieces.append(obj)
+        else:
+            logger.warning(f'Unknown link type {obj["type"]}')
+    return items, pieces
+
+
+def _get_or_create_item_by_ap_url(url):
+    logger.debug(f"Fetching item by ap from {url}")
+    site = SiteManager.get_site_by_url(url)
+    if not site:
+        return None
+    site.get_resource_ready()
+    item = site.get_item()
+    return item
+
+
+def _get_visibility(post_visibility):
+    match post_visibility:
+        case 2:
+            return 1
+        case 3:
+            return 2
+        case _:
+            return 0
+
+
+def _update_or_create_post(pk, obj):
+    post = Post.objects.get(pk=pk)
+    owner = Takahe.get_or_create_apidentity(post.author)
+    if not post.type_data:
+        logger.warning(f"Post {post} has no type_data")
+        return
+    items, pieces = _parse_links(post.type_data["object"]["relatedWith"])
+    logger.info(f"Post {post} has items {items} and pieces {pieces}")
+    if len(items) == 0:
+        logger.warning(f"Post {post} has no remote items")
+        return
+    elif len(items) > 1:
+        logger.warning(f"Post {post} has more than one remote item")
+        return
+    remote_url = items[0]
+    item = _get_or_create_item_by_ap_url(remote_url)
+    if not item:
+        logger.warning(f"Post {post} has no local item")
+        return
+    for p in pieces:
+        cls = _supported_ap_journal_types[p["type"]]
+        cls.update_by_ap_object(owner, item, p, pk, _get_visibility(post.visibility))
+
+
+def post_created(pk, obj):
+    _update_or_create_post(pk, obj)
+
+
+def post_updated(pk, obj):
+    _update_or_create_post(pk, obj)
+
+
+def post_deleted(pk, obj):
+    Piece.objects.filter(post_id=pk, local=False).delete()
+
+
+def user_follow_updated(source_identity_pk, target_identity_pk):
+    u = Takahe.get_local_user_by_identity(source_identity_pk)
+    # Takahe.update_user_following(u)
+    logger.info(f"User {u} following updated")
+
+
+def user_mute_updated(source_identity_pk, target_identity_pk):
+    u = Takahe.get_local_user_by_identity(source_identity_pk)
+    # Takahe.update_user_muting(u)
+    logger.info(f"User {u} muting updated")
+
+
+def user_block_updated(source_identity_pk, target_identity_pk):
+    u = Takahe.get_local_user_by_identity(source_identity_pk)
+    if u:
+        # Takahe.update_user_rejecting(u)
+        logger.info(f"User {u} rejecting updated")
+    u = Takahe.get_local_user_by_identity(target_identity_pk)
+    if u:
+        # Takahe.update_user_rejecting(u)
+        logger.info(f"User {u} rejecting updated")

takahe/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class TakaheConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "takahe"

takahe/db_routes.py

@@ -0,0 +1,27 @@
+from django.conf import settings
+
+_is_testing = "testserver" in settings.ALLOWED_HOSTS
+
+
+class TakaheRouter:
+    def db_for_read(self, model, **hints):
+        if model._meta.app_label == "takahe":
+            return "takahe"
+        return None
+
+    def db_for_write(self, model, **hints):
+        if model._meta.app_label == "takahe":
+            return "takahe"
+        return None
+
+    def allow_relation(self, obj1, obj2, **hints):
+        # skip this check but please make sure
+        # not create relations between takahe models and other apps
+        if obj1._meta.app_label == "takahe" or obj2._meta.app_label == "takahe":
+            return obj1._meta.app_label == obj2._meta.app_label
+        return None
+
+    def allow_migrate(self, db, app_label, model_name=None, **hints):
+        if app_label == "takahe" or db == "takahe":
+            return _is_testing and app_label == db
+        return None

takahe/html.py

@@ -0,0 +1,379 @@
+import html
+import re
+from html.parser import HTMLParser
+
+from django.utils.safestring import mark_safe
+
+
+class FediverseHtmlParser(HTMLParser):
+    """
+    A custom HTML parser that only allows a certain tag subset and behaviour:
+    - br, p tags are passed through
+    - a tags are passed through if they're not hashtags or mentions
+    - Another set of tags are converted to p
+
+    It also linkifies URLs, mentions, hashtags, and imagifies emoji.
+    """
+
+    REWRITE_TO_P = [
+        "p",
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        "blockquote",
+        "pre",
+        "ul",
+        "ol",
+    ]
+
+    REWRITE_TO_BR = [
+        "br",
+        "li",
+    ]
+
+    MENTION_REGEX = re.compile(
+        r"(^|[^\w\d\-_/])@([\w\d\-_]+(?:@[\w\d\-_\.]+[\w\d\-_]+)?)"
+    )
+
+    HASHTAG_REGEX = re.compile(r"\B#([a-zA-Z0-9(_)]+\b)(?!;)")
+
+    EMOJI_REGEX = re.compile(r"\B:([a-zA-Z0-9(_)-]+):\B")
+
+    URL_REGEX = re.compile(
+        r"""(\(*  # Match any opening parentheses.
+        \b(?<![@.])(?:https?://(?:(?:\w+:)?\w+@)?)  # http://
+        (?:[\w-]+\.)+(?:[\w-]+)(?:\:[0-9]+)?(?!\.\w)\b   # xx.yy.tld(:##)?
+        (?:[/?][^\s\{{\}}\|\\\^\[\]`<>"]*)?)
+        # /path/zz (excluding "unsafe" chars from RFC 1738,
+        # except for # and ~, which happen in practice)
+        """,
+        re.IGNORECASE | re.VERBOSE | re.UNICODE,
+    )
+
+    def __init__(
+        self,
+        html: str,
+        uri_domain: str | None = None,
+        mentions: list | None = None,
+        find_mentions: bool = False,
+        find_hashtags: bool = False,
+        find_emojis: bool = False,
+        emoji_domain=None,
+    ):
+        super().__init__()
+        self.uri_domain = uri_domain
+        self.emoji_domain = emoji_domain
+        self.find_mentions = find_mentions
+        self.find_hashtags = find_hashtags
+        self.find_emojis = find_emojis
+        self.calculate_mentions(mentions)
+        self._data_buffer = ""
+        self.html_output = ""
+        self.text_output = ""
+        self.emojis: set[str] = set()
+        self.mentions: set[str] = set()
+        self.hashtags: set[str] = set()
+        self._pending_a: dict | None = None
+        self._fresh_p = False
+        self.feed(html.replace("\n", ""))
+        self.flush_data()
+
+    def calculate_mentions(self, mentions: list | None):
+        """
+        Prepares a set of content that we expect to see mentions look like
+        (this imp)
+        """
+        self.mention_matches: dict[str, str] = {}
+        self.mention_aliases: dict[str, str] = {}
+        for mention in mentions or []:
+            if self.uri_domain:
+                url = mention.absolute_profile_uri()
+            else:
+                url = str(mention.urls.view)
+            if mention.username:
+                username = mention.username.lower()
+                domain = mention.domain_id.lower()
+                self.mention_matches[f"{username}"] = url
+                self.mention_matches[f"{username}@{domain}"] = url
+                self.mention_matches[mention.absolute_profile_uri()] = url
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag in self.REWRITE_TO_P:
+            self.flush_data()
+            self.html_output += "<p>"
+        elif tag in self.REWRITE_TO_BR:
+            self.flush_data()
+            if not self._fresh_p:
+                self.html_output += "<br>"
+                self.text_output += "\n"
+        elif tag == "a":
+            self.flush_data()
+            self._pending_a = {"attrs": dict(attrs), "content": ""}
+        self._fresh_p = tag in self.REWRITE_TO_P
+
+    def handle_endtag(self, tag: str) -> None:
+        self._fresh_p = False
+        if tag in self.REWRITE_TO_P:
+            self.flush_data()
+            self.html_output += "</p>"
+            self.text_output += "\n\n"
+        elif tag == "a":
+            if self._pending_a:
+                href = self._pending_a["attrs"].get("href")
+                content = self._pending_a["content"].strip()
+                has_ellipsis = "ellipsis" in self._pending_a["attrs"].get("class", "")
+                # Is it a mention?
+                if content.lower().lstrip("@") in self.mention_matches:
+                    self.html_output += self.create_mention(content, href)
+                    self.text_output += content
+                # Is it a hashtag?
+                elif self.HASHTAG_REGEX.match(content):
+                    self.html_output += self.create_hashtag(content)
+                    self.text_output += content
+                elif content:
+                    # Shorten the link if we need to
+                    self.html_output += self.create_link(
+                        href,
+                        content,
+                        has_ellipsis=has_ellipsis,
+                    )
+                    self.text_output += href
+                self._pending_a = None
+
+    def handle_data(self, data: str) -> None:
+        self._fresh_p = False
+        if self._pending_a:
+            self._pending_a["content"] += data
+        else:
+            self._data_buffer += data
+
+    def flush_data(self) -> None:
+        """
+        We collect data segments until we encounter a tag we care about,
+        so we can treat <span>#</span>hashtag as #hashtag
+        """
+        self.text_output += self._data_buffer
+        self.html_output += self.linkify(self._data_buffer)
+        self._data_buffer = ""
+
+    def create_link(self, href, content, has_ellipsis=False):
+        """
+        Generates a link, doing optional shortening.
+
+        All return values from this function should be HTML-safe.
+        """
+        looks_like_link = bool(self.URL_REGEX.match(content))
+        if looks_like_link:
+            protocol, content = content.split("://", 1)
+        else:
+            protocol = ""
+        if (looks_like_link and len(content) > 30) or has_ellipsis:
+            return f'<a href="{html.escape(href)}" rel="nofollow" class="ellipsis" title="{html.escape(content)}"><span class="invisible">{html.escape(protocol)}://</span><span class="ellipsis">{html.escape(content[:30])}</span><span class="invisible">{html.escape(content[30:])}</span></a>'
+        elif looks_like_link:
+            return f'<a href="{html.escape(href)}" rel="nofollow"><span class="invisible">{html.escape(protocol)}://</span>{html.escape(content)}</a>'
+        else:
+            return f'<a href="{html.escape(href)}" rel="nofollow">{html.escape(content)}</a>'
+
+    def create_mention(self, handle, href: str | None = None) -> str:
+        """
+        Generates a mention link. Handle should have a leading @.
+
+        All return values from this function should be HTML-safe
+        """
+        handle = handle.lstrip("@")
+        if "@" in handle:
+            short_handle = handle.split("@", 1)[0]
+        else:
+            short_handle = handle
+        handle_hash = handle.lower()
+        short_hash = short_handle.lower()
+        self.mentions.add(handle_hash)
+        url = self.mention_matches.get(handle_hash)
+        # If we have a captured link out, use that as the actual resolver
+        if href and href in self.mention_matches:
+            url = self.mention_matches[href]
+        if url:
+            if short_hash not in self.mention_aliases:
+                self.mention_aliases[short_hash] = handle_hash
+            elif self.mention_aliases.get(short_hash) != handle_hash:
+                short_handle = handle
+            return f'<span class="h-card"><a href="{html.escape(url)}" class="u-url mention" rel="nofollow noopener noreferrer" target="_blank">@<span>{html.escape(short_handle)}</span></a></span>'
+        else:
+            return "@" + html.escape(handle)
+
+    def create_hashtag(self, hashtag) -> str:
+        """
+        Generates a hashtag link. Hashtag does not need to start with #
+
+        All return values from this function should be HTML-safe
+        """
+        hashtag = hashtag.lstrip("#")
+        self.hashtags.add(hashtag.lower())
+        if self.uri_domain:
+            return f'<a href="https://{self.uri_domain}/tags/{hashtag.lower()}/" class="mention hashtag" rel="tag">#{hashtag}</a>'
+        else:
+            return f'<a href="/tags/{hashtag.lower()}/" rel="tag">#{hashtag}</a>'
+
+    def create_emoji(self, shortcode) -> str:
+        """
+        Generates an emoji <img> tag
+
+        All return values from this function should be HTML-safe
+        """
+        from .models import Emoji
+
+        emoji = Emoji.get_by_domain(shortcode, self.emoji_domain)
+        if emoji and emoji.is_usable:
+            self.emojis.add(shortcode)
+            return emoji.as_html()
+        return f":{shortcode}:"
+
+    def linkify(self, data):
+        """
+        Linkifies some content that is plaintext.
+
+        Handles URLs first, then mentions. Note that this takes great care to
+        keep track of what is HTML and what needs to be escaped.
+        """
+        # Split the string by the URL regex so we know what to escape and what
+        # not to escape.
+        bits = self.URL_REGEX.split(data)
+        result = ""
+        # Even indices are data we should pass though, odd indices are links
+        for i, bit in enumerate(bits):
+            # A link!
+            if i % 2 == 1:
+                result += self.create_link(bit, bit)
+            # Not a link
+            elif self.mention_matches or self.find_mentions:
+                result += self.linkify_mentions(bit)
+            elif self.find_hashtags:
+                result += self.linkify_hashtags(bit)
+            elif self.find_emojis:
+                result += self.linkify_emoji(bit)
+            else:
+                result += html.escape(bit)
+        return result
+
+    def linkify_mentions(self, data):
+        """
+        Linkifies mentions
+        """
+        bits = self.MENTION_REGEX.split(data)
+        result = ""
+        for i, bit in enumerate(bits):
+            # Mention content
+            if i % 3 == 2:
+                result += self.create_mention(bit)
+            # Not part of a mention (0) or mention preamble (1)
+            elif self.find_hashtags:
+                result += self.linkify_hashtags(bit)
+            elif self.find_emojis:
+                result += self.linkify_emoji(bit)
+            else:
+                result += html.escape(bit)
+        return result
+
+    def linkify_hashtags(self, data):
+        """
+        Linkifies hashtags
+        """
+        bits = self.HASHTAG_REGEX.split(data)
+        result = ""
+        for i, bit in enumerate(bits):
+            # Not part of a hashtag
+            if i % 2 == 0:
+                if self.find_emojis:
+                    result += self.linkify_emoji(bit)
+                else:
+                    result += html.escape(bit)
+            # Hashtag content
+            else:
+                result += self.create_hashtag(bit)
+        return result
+
+    def linkify_emoji(self, data):
+        """
+        Linkifies emoji
+        """
+        bits = self.EMOJI_REGEX.split(data)
+        result = ""
+        for i, bit in enumerate(bits):
+            # Not part of an emoji
+            if i % 2 == 0:
+                result += html.escape(bit)
+            # Emoji content
+            else:
+                result += self.create_emoji(bit)
+        return result
+
+    @property
+    def html(self):
+        return self.html_output.strip()
+
+    @property
+    def plain_text(self):
+        return self.text_output.strip()
+
+
+class ContentRenderer:
+    """
+    Renders HTML for posts, identity fields, and more.
+
+    The `local` parameter affects whether links are absolute (False) or relative (True)
+    """
+
+    def __init__(self, local: bool):
+        self.local = local
+
+    def render_post(self, html: str, post) -> str:
+        """
+        Given post HTML, normalises it and renders it for presentation.
+        """
+        if not html:
+            return ""
+        parser = FediverseHtmlParser(
+            html,
+            mentions=post.mentions.all(),
+            uri_domain=(None if self.local else post.author.domain.uri_domain),
+            find_hashtags=True,
+            find_emojis=self.local,
+            emoji_domain=post.author.domain,
+        )
+        return mark_safe(parser.html)
+
+    def render_identity_summary(self, html: str, identity) -> str:
+        """
+        Given identity summary HTML, normalises it and renders it for presentation.
+        """
+        if not html:
+            return ""
+        parser = FediverseHtmlParser(
+            html,
+            uri_domain=(None if self.local else identity.domain.uri_domain),
+            find_hashtags=True,
+            find_emojis=self.local,
+            emoji_domain=identity.domain,
+        )
+        return mark_safe(parser.html)
+
+    def render_identity_data(self, html: str, identity, strip: bool = False) -> str:
+        """
+        Given name/basic value HTML, normalises it and renders it for presentation.
+        """
+        if not html:
+            return ""
+        parser = FediverseHtmlParser(
+            html,
+            uri_domain=(None if self.local else identity.domain.uri_domain),
+            find_hashtags=False,
+            find_emojis=self.local,
+            emoji_domain=identity.domain,
+        )
+        if strip:
+            return mark_safe(parser.html)
+        else:
+            return mark_safe(parser.html)

takahe/management/commands/takahe.py

@@ -0,0 +1,42 @@
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.db.models import Count, F
+from loguru import logger
+from tqdm import tqdm
+
+from catalog.common import *
+from catalog.common.models import *
+from catalog.models import *
+from journal.models import Tag, update_journal_for_merged_item
+from takahe.utils import *
+from users.models import User as NeoUser
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--verbose",
+            action="store_true",
+        )
+        parser.add_argument(
+            "--sync",
+            action="store_true",
+        )
+
+    def sync(self):
+        logger.info(f"Syncing domain...")
+        Takahe.get_domain()
+        logger.info(f"Syncing users...")
+        for u in tqdm(NeoUser.objects.filter(is_active=True, username__isnull=False)):
+            Takahe.init_identity_for_local_user(u)
+            # Takahe.update_user_following(u)
+            # Takahe.update_user_muting(u)
+            # Takahe.update_user_rejecting(u)
+
+    def handle(self, *args, **options):
+        self.verbose = options["verbose"]
+
+        if options["sync"]:
+            self.sync()
+
+        self.stdout.write(self.style.SUCCESS(f"Done."))

takahe/migrations/0001_initial.py

@@ -0,0 +1,489 @@
+# Generated by Django 4.2.4 on 2023-08-12 16:48
+
+import django.db.models.deletion
+import django.utils.timezone
+from django.db import migrations, models
+
+import takahe.models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="Domain",
+            fields=[
+                (
+                    "domain",
+                    models.CharField(max_length=250, primary_key=True, serialize=False),
+                ),
+                (
+                    "service_domain",
+                    models.CharField(
+                        blank=True,
+                        db_index=True,
+                        max_length=250,
+                        null=True,
+                        unique=True,
+                    ),
+                ),
+                ("state", models.CharField(default="outdated", max_length=100)),
+                ("state_changed", models.DateTimeField(auto_now_add=True)),
+                ("nodeinfo", models.JSONField(blank=True, null=True)),
+                ("local", models.BooleanField()),
+                ("blocked", models.BooleanField(default=False)),
+                ("public", models.BooleanField(default=False)),
+                ("default", models.BooleanField(default=False)),
+                ("notes", models.TextField(blank=True, null=True)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+            ],
+            options={
+                "db_table": "users_domain",
+            },
+        ),
+        migrations.CreateModel(
+            name="Emoji",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("shortcode", models.SlugField(max_length=100)),
+                ("local", models.BooleanField(default=True)),
+                ("public", models.BooleanField(null=True)),
+                (
+                    "object_uri",
+                    models.CharField(
+                        blank=True, max_length=500, null=True, unique=True
+                    ),
+                ),
+                ("mimetype", models.CharField(max_length=200)),
+                ("file", models.ImageField(blank=True, null=True, upload_to="")),
+                ("remote_url", models.CharField(blank=True, max_length=500, null=True)),
+                ("category", models.CharField(blank=True, max_length=100, null=True)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+                (
+                    "domain",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="takahe.domain",
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "activities_emoji",
+            },
+        ),
+        migrations.CreateModel(
+            name="Hashtag",
+            fields=[
+                (
+                    "hashtag",
+                    models.SlugField(max_length=100, primary_key=True, serialize=False),
+                ),
+                (
+                    "name_override",
+                    models.CharField(blank=True, max_length=100, null=True),
+                ),
+                ("public", models.BooleanField(null=True)),
+                ("state", models.CharField(default="outdated", max_length=100)),
+                ("state_changed", models.DateTimeField(auto_now_add=True)),
+                ("stats", models.JSONField(blank=True, null=True)),
+                ("stats_updated", models.DateTimeField(blank=True, null=True)),
+                ("aliases", models.JSONField(blank=True, null=True)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+            ],
+            options={
+                "db_table": "activities_hashtag",
+            },
+        ),
+        migrations.CreateModel(
+            name="Identity",
+            fields=[
+                (
+                    "id",
+                    models.BigIntegerField(
+                        default=takahe.models.Snowflake.generate_identity,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                ("actor_uri", models.CharField(max_length=500, unique=True)),
+                ("state", models.CharField(default="outdated", max_length=100)),
+                ("state_changed", models.DateTimeField(auto_now_add=True)),
+                ("local", models.BooleanField(db_index=True)),
+                ("username", models.CharField(blank=True, max_length=500, null=True)),
+                ("name", models.CharField(blank=True, max_length=500, null=True)),
+                ("summary", models.TextField(blank=True, null=True)),
+                (
+                    "manually_approves_followers",
+                    models.BooleanField(blank=True, null=True),
+                ),
+                ("discoverable", models.BooleanField(default=True)),
+                (
+                    "profile_uri",
+                    models.CharField(blank=True, max_length=500, null=True),
+                ),
+                ("inbox_uri", models.CharField(blank=True, max_length=500, null=True)),
+                (
+                    "shared_inbox_uri",
+                    models.CharField(blank=True, max_length=500, null=True),
+                ),
+                ("outbox_uri", models.CharField(blank=True, max_length=500, null=True)),
+                ("icon_uri", models.CharField(blank=True, max_length=500, null=True)),
+                ("image_uri", models.CharField(blank=True, max_length=500, null=True)),
+                (
+                    "followers_uri",
+                    models.CharField(blank=True, max_length=500, null=True),
+                ),
+                (
+                    "following_uri",
+                    models.CharField(blank=True, max_length=500, null=True),
+                ),
+                (
+                    "featured_collection_uri",
+                    models.CharField(blank=True, max_length=500, null=True),
+                ),
+                ("actor_type", models.CharField(default="person", max_length=100)),
+                ("metadata", models.JSONField(blank=True, null=True)),
+                ("pinned", models.JSONField(blank=True, null=True)),
+                ("sensitive", models.BooleanField(default=False)),
+                (
+                    "restriction",
+                    models.IntegerField(
+                        choices=[(0, "None"), (1, "Limited"), (2, "Blocked")],
+                        db_index=True,
+                        default=0,
+                    ),
+                ),
+                ("admin_notes", models.TextField(blank=True, null=True)),
+                ("private_key", models.TextField(blank=True, null=True)),
+                ("public_key", models.TextField(blank=True, null=True)),
+                ("public_key_id", models.TextField(blank=True, null=True)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+                ("fetched", models.DateTimeField(blank=True, null=True)),
+                ("deleted", models.DateTimeField(blank=True, null=True)),
+                (
+                    "domain",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="identities",
+                        to="takahe.domain",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name_plural": "identities",
+                "db_table": "users_identity",
+            },
+        ),
+        migrations.CreateModel(
+            name="Post",
+            fields=[
+                (
+                    "id",
+                    models.BigIntegerField(
+                        default=takahe.models.Snowflake.generate_post,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                ("state", models.CharField(default="new", max_length=100)),
+                ("state_changed", models.DateTimeField(auto_now_add=True)),
+                ("local", models.BooleanField()),
+                (
+                    "object_uri",
+                    models.CharField(
+                        blank=True, max_length=2048, null=True, unique=True
+                    ),
+                ),
+                (
+                    "visibility",
+                    models.IntegerField(
+                        choices=[
+                            (0, "Public"),
+                            (4, "Local Only"),
+                            (1, "Unlisted"),
+                            (2, "Followers"),
+                            (3, "Mentioned"),
+                        ],
+                        default=0,
+                    ),
+                ),
+                ("content", models.TextField()),
+                (
+                    "type",
+                    models.CharField(
+                        choices=[
+                            ("Article", "Article"),
+                            ("Audio", "Audio"),
+                            ("Event", "Event"),
+                            ("Image", "Image"),
+                            ("Note", "Note"),
+                            ("Page", "Page"),
+                            ("Question", "Question"),
+                            ("Video", "Video"),
+                        ],
+                        default="Note",
+                        max_length=20,
+                    ),
+                ),
+                ("type_data", models.JSONField(blank=True, null=True)),
+                ("sensitive", models.BooleanField(default=False)),
+                ("summary", models.TextField(blank=True, null=True)),
+                ("url", models.CharField(blank=True, max_length=2048, null=True)),
+                (
+                    "in_reply_to",
+                    models.CharField(
+                        blank=True, db_index=True, max_length=500, null=True
+                    ),
+                ),
+                ("hashtags", models.JSONField(blank=True, null=True)),
+                ("stats", models.JSONField(blank=True, null=True)),
+                ("published", models.DateTimeField(default=django.utils.timezone.now)),
+                ("edited", models.DateTimeField(blank=True, null=True)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+                (
+                    "author",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="posts",
+                        to="takahe.identity",
+                    ),
+                ),
+                (
+                    "emojis",
+                    models.ManyToManyField(
+                        blank=True, related_name="posts_using_emoji", to="takahe.emoji"
+                    ),
+                ),
+                (
+                    "mentions",
+                    models.ManyToManyField(
+                        blank=True,
+                        related_name="posts_mentioning",
+                        to="takahe.identity",
+                    ),
+                ),
+                (
+                    "to",
+                    models.ManyToManyField(
+                        blank=True, related_name="posts_to", to="takahe.identity"
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "activities_post",
+            },
+        ),
+        migrations.CreateModel(
+            name="User",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("password", models.CharField(max_length=128, verbose_name="password")),
+                (
+                    "last_login",
+                    models.DateTimeField(
+                        blank=True, null=True, verbose_name="last login"
+                    ),
+                ),
+                ("email", models.EmailField(max_length=254, unique=True)),
+                ("admin", models.BooleanField(default=False)),
+                ("moderator", models.BooleanField(default=False)),
+                ("banned", models.BooleanField(default=False)),
+                ("deleted", models.BooleanField(default=False)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+                ("last_seen", models.DateTimeField(auto_now_add=True)),
+            ],
+            options={
+                "db_table": "users_user",
+            },
+        ),
+        migrations.CreateModel(
+            name="PostInteraction",
+            fields=[
+                (
+                    "id",
+                    models.BigIntegerField(
+                        default=takahe.models.Snowflake.generate_post_interaction,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                ("state", models.CharField(default="new", max_length=100)),
+                ("state_changed", models.DateTimeField(auto_now_add=True)),
+                (
+                    "object_uri",
+                    models.CharField(
+                        blank=True, max_length=500, null=True, unique=True
+                    ),
+                ),
+                (
+                    "type",
+                    models.CharField(
+                        choices=[
+                            ("like", "Like"),
+                            ("boost", "Boost"),
+                            ("vote", "Vote"),
+                            ("pin", "Pin"),
+                        ],
+                        max_length=100,
+                    ),
+                ),
+                ("value", models.CharField(blank=True, max_length=50, null=True)),
+                ("published", models.DateTimeField(default=django.utils.timezone.now)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+                (
+                    "identity",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="interactions",
+                        to="takahe.identity",
+                    ),
+                ),
+                (
+                    "post",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="interactions",
+                        to="takahe.post",
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "activities_postinteraction",
+            },
+        ),
+        migrations.AddField(
+            model_name="identity",
+            name="users",
+            field=models.ManyToManyField(
+                blank=True, related_name="identities", to="takahe.user"
+            ),
+        ),
+        migrations.AddField(
+            model_name="domain",
+            name="users",
+            field=models.ManyToManyField(
+                blank=True, related_name="domains", to="takahe.user"
+            ),
+        ),
+        migrations.CreateModel(
+            name="Block",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("state", models.CharField(default="new", max_length=100)),
+                ("state_changed", models.DateTimeField(auto_now_add=True)),
+                ("uri", models.CharField(blank=True, max_length=500, null=True)),
+                ("mute", models.BooleanField()),
+                ("include_notifications", models.BooleanField(default=False)),
+                ("expires", models.DateTimeField(blank=True, null=True)),
+                ("note", models.TextField(blank=True, null=True)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+                (
+                    "source",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="outbound_blocks",
+                        to="takahe.identity",
+                    ),
+                ),
+                (
+                    "target",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="inbound_blocks",
+                        to="takahe.identity",
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "users_block",
+            },
+        ),
+        migrations.AlterUniqueTogether(
+            name="identity",
+            unique_together={("username", "domain")},
+        ),
+        migrations.CreateModel(
+            name="Follow",
+            fields=[
+                (
+                    "id",
+                    models.BigIntegerField(
+                        default=takahe.models.Snowflake.generate_follow,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                (
+                    "boosts",
+                    models.BooleanField(
+                        default=True, help_text="Also follow boosts from this user"
+                    ),
+                ),
+                ("uri", models.CharField(blank=True, max_length=500, null=True)),
+                ("note", models.TextField(blank=True, null=True)),
+                ("state", models.CharField(default="unrequested", max_length=100)),
+                ("state_changed", models.DateTimeField(auto_now_add=True)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+                (
+                    "source",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="outbound_follows",
+                        to="takahe.identity",
+                    ),
+                ),
+                (
+                    "target",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="inbound_follows",
+                        to="takahe.identity",
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "users_follow",
+                "unique_together": {("source", "target")},
+            },
+        ),
+    ]

takahe/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

takahe/uris.py

@@ -0,0 +1,89 @@
+import hashlib
+import sys
+from urllib.parse import urljoin
+
+from django.conf import settings
+from django.contrib.staticfiles.storage import staticfiles_storage
+
+
+class RelativeAbsoluteUrl:
+    """
+    Represents a URL that can have both "relative" and "absolute" forms
+    for various use either locally or remotely.
+    """
+
+    absolute: str
+    relative: str
+
+    def __init__(self, absolute: str, relative: str | None = None):
+        if "://" not in absolute:
+            raise ValueError(f"Absolute URL {absolute!r} is not absolute!")
+        self.absolute = absolute
+        self.relative = relative or absolute
+
+
+class AutoAbsoluteUrl(RelativeAbsoluteUrl):
+    """
+    Automatically makes the absolute variant by using either settings.MAIN_DOMAIN
+    or a passed identity's URI domain.
+    """
+
+    def __init__(
+        self,
+        relative: str,
+        identity=None,
+    ):
+        self.relative = relative
+        if identity:
+            absolute_prefix = f"https://{identity.domain.uri_domain}/"
+        else:
+            absolute_prefix = f"https://{settings.MAIN_DOMAIN}/"
+        self.absolute = urljoin(absolute_prefix, self.relative)
+
+
+class ProxyAbsoluteUrl(AutoAbsoluteUrl):
+    """
+    AutoAbsoluteUrl variant for proxy paths, that also attaches a remote URI hash
+    plus extension to the end if it can.
+    """
+
+    def __init__(
+        self,
+        relative: str,
+        identity=None,
+        remote_url: str | None = None,
+    ):
+        if remote_url:
+            # See if there is a file extension we can grab
+            extension = "bin"
+            remote_filename = remote_url.split("/")[-1]
+            if "." in remote_filename:
+                extension = remote_filename.split(".")[-1]
+            # When provided, attach a hash of the remote URL
+            # SHA1 chosen as it generally has the best performance in modern python, and security is not a concern
+            # Hash truncation is generally fine, as in the typical use case the hash is scoped to the identity PK.
+            relative += f"{hashlib.sha1(remote_url.encode('ascii')).hexdigest()[:10]}.{extension}"
+        super().__init__(relative, identity)
+
+
+class StaticAbsoluteUrl(RelativeAbsoluteUrl):
+    """
+    Creates static URLs given only the static-relative path
+    """
+
+    def __init__(self, path: str):
+        try:
+            static_url = staticfiles_storage.url(path)
+        except ValueError:
+            # Suppress static issues during the first collectstatic
+            # Yes, I know it's a big hack! Pull requests welcome :)
+            if "collectstatic" in sys.argv:
+                super().__init__("https://example.com/")
+                return
+            raise
+        if "://" in static_url:
+            super().__init__(static_url)
+        else:
+            super().__init__(
+                urljoin(f"https://{settings.MAIN_DOMAIN}/", static_url), static_url
+            )

takahe/utils.py

@@ -0,0 +1,486 @@
+from typing import TYPE_CHECKING
+
+from django.conf import settings
+
+from .models import *
+
+if TYPE_CHECKING:
+    from users.models import APIdentity
+    from users.models import User as NeoUser
+
+
+def _int(s: str):
+    try:
+        return int(s)
+    except:
+        return -1
+
+
+def _rating_to_emoji(score: int, star_mode=0):
+    """convert score(0~10) to mastodon star emoji code"""
+    if score is None or score == "" or score == 0:
+        return ""
+    solid_stars = score // 2
+    half_star = int(bool(score % 2))
+    empty_stars = 5 - solid_stars if not half_star else 5 - solid_stars - 1
+    if star_mode == 1:
+        emoji_code = "🌕" * solid_stars + "🌗" * half_star + "🌑" * empty_stars
+    else:
+        emoji_code = (
+            settings.STAR_SOLID * solid_stars
+            + settings.STAR_HALF * half_star
+            + settings.STAR_EMPTY * empty_stars
+        )
+    emoji_code = emoji_code.replace("::", ": :")
+    emoji_code = " " + emoji_code + " "
+    return emoji_code
+
+
+class Takahe:
+    Visibilities = Post.Visibilities
+
+    @staticmethod
+    def get_domain():
+        domain = settings.SITE_INFO["site_domain"]
+        d = Domain.objects.filter(domain=domain).first()
+        if not d:
+            logger.info(f"Creating takahe domain {domain}")
+            d = Domain.objects.create(
+                domain=domain,
+                local=True,
+                service_domain=None,
+                notes="NeoDB",
+                nodeinfo=None,
+            )
+        return d
+
+    @staticmethod
+    def get_node_name_for_domain(d: str):
+        domain = Domain.objects.filter(domain=d).first()
+        if domain and domain.nodeinfo:
+            return domain.nodeinfo.get("metadata", {}).get("nodeName")
+
+    @staticmethod
+    def init_identity_for_local_user(u: "NeoUser"):
+        """
+        When a new local NeoDB user is created,
+        create a takahe user with the NeoDB user pk,
+        create a takahe identity,
+        then create a NeoDB APIdentity with the takahe identity pk.
+        """
+        from users.models import APIdentity
+
+        if not u.username:
+            logger.warning(f"User {u} has no username")
+            return None
+        user = User.objects.filter(pk=u.pk).first()
+        handler = "@" + u.username
+        if not user:
+            logger.info(f"Creating takahe user {u}")
+            user = User.objects.create(pk=u.pk, email=handler)
+        else:
+            if user.email != handler:
+                logger.warning(f"Updating takahe user {u} email to {handler}")
+                user.email = handler
+                user.save()
+        domain = Domain.objects.get(domain=settings.SITE_INFO["site_domain"])
+        identity = Identity.objects.filter(username=u.username, local=True).first()
+        if not identity:
+            logger.info(f"Creating takahe identity {u}@{domain}")
+            identity = Identity.objects.create(
+                actor_uri=f"https://{domain.uri_domain}/@{u.username}@{domain.domain}/",
+                username=u.username,
+                domain=domain,
+                name=u.username,
+                local=True,
+                discoverable=not u.preference.no_anonymous_view,
+            )
+            identity.generate_keypair()
+        if not user.identities.filter(pk=identity.pk).exists():
+            user.identities.add(identity)
+        apidentity = APIdentity.objects.filter(pk=identity.pk).first()
+        if not apidentity:
+            logger.info(f"Creating APIdentity for {identity}")
+            apidentity = APIdentity.objects.create(
+                user=u,
+                id=identity.pk,
+                local=True,
+                username=u.username,
+                domain_name=domain.domain,
+                deleted=identity.deleted,
+            )
+        elif apidentity.username != identity.username:
+            logger.warning(
+                f"Updating APIdentity {apidentity} username to {identity.username}"
+            )
+            apidentity.username = identity.username
+            apidentity.save()
+        if u.identity != apidentity:
+            logger.warning(f"Linking user {u} identity to {apidentity}")
+            u.identity = apidentity
+            u.save(update_fields=["identity"])
+        return apidentity
+
+    @staticmethod
+    def get_identity(pk: int):
+        return Identity.objects.get(pk=pk)
+
+    @staticmethod
+    def get_identity_by_local_user(u: "NeoUser"):
+        return (
+            Identity.objects.filter(pk=u.identity.pk, local=True).first()
+            if u and u.is_authenticated and u.identity
+            else None
+        )
+
+    @staticmethod
+    def get_or_create_apidentity(identity: Identity):
+        from users.models import APIdentity
+
+        apid = APIdentity.objects.filter(pk=identity.pk).first()
+        if not apid:
+            if identity.local:
+                raise ValueError(f"local takahe identity {identity} missing APIdentity")
+            if not identity.domain:
+                raise ValueError(f"remote takahe identity {identity} missing domain")
+            apid = APIdentity.objects.create(
+                id=identity.pk,
+                local=False,
+                username=identity.username,
+                domain_name=identity.domain.domain,
+                deleted=identity.deleted,
+            )
+        return apid
+
+    @staticmethod
+    def get_local_user_by_identity(identity: Identity):
+        from users.models import User as NeoUser
+
+        return NeoUser.objects.get(identity_id=identity.pk) if identity.local else None
+
+    @staticmethod
+    def get_following_ids(identity_pk: int):
+        targets = Follow.objects.filter(
+            source_id=identity_pk, state="accepted"
+        ).values_list("target", flat=True)
+        return list(targets)
+
+    @staticmethod
+    def get_follower_ids(identity_pk: int):
+        targets = Follow.objects.filter(
+            target_id=identity_pk, state="accepted"
+        ).values_list("target", flat=True)
+        return list(targets)
+
+    @staticmethod
+    def get_following_request_ids(identity_pk: int):
+        targets = Follow.objects.filter(
+            source_id=identity_pk, state="pending_approval"
+        ).values_list("target", flat=True)
+        return list(targets)
+
+    @staticmethod
+    def get_requested_follower_ids(identity_pk: int):
+        targets = Follow.objects.filter(
+            target_id=identity_pk, state="pending_approval"
+        ).values_list("target", flat=True)
+        return list(targets)
+
+    @staticmethod
+    def update_follow_state(
+        source_pk: int, target_pk: int, from_states: list[str], to_state: str
+    ):
+        follow = Follow.objects.filter(source_id=source_pk, target_id=target_pk).first()
+        if (
+            follow
+            and (not from_states or follow.state in from_states)
+            and follow.state != to_state
+        ):
+            follow.state = to_state
+            follow.save()
+        return follow
+
+    @staticmethod
+    def follow(source_pk: int, target_pk: int):
+        try:
+            follow = Follow.objects.get(source_id=source_pk, target_id=target_pk)
+            if follow.state != "accepted":
+                follow.state = "unrequested"
+                follow.save()
+        except Follow.DoesNotExist:
+            source = Identity.objects.get(pk=source_pk)
+            follow = Follow.objects.create(
+                source_id=source_pk,
+                target_id=target_pk,
+                boosts=True,
+                uri="",
+                state="unrequested",
+            )
+            follow.uri = source.actor_uri + f"follow/{follow.pk}/"
+            follow.save()
+
+    @staticmethod
+    def unfollow(source_pk: int, target_pk: int):
+        Takahe.update_follow_state(source_pk, target_pk, [], "undone")
+        # InboxMessage.create_internal(
+        #     {
+        #         "type": "ClearTimeline",
+        #         "object": target_identity.pk,
+        #         "actor": self.identity.pk,
+        #     }
+        # )
+
+    @staticmethod
+    def accept_follow_request(source_pk: int, target_pk: int):
+        Takahe.update_follow_state(source_pk, target_pk, [], "accepting")
+
+    @staticmethod
+    def reject_follow_request(source_pk: int, target_pk: int):
+        Takahe.update_follow_state(source_pk, target_pk, [], "rejecting")
+
+    @staticmethod
+    def get_muting_ids(identity_pk: int) -> list[int]:
+        targets = Block.objects.filter(
+            source_id=identity_pk,
+            mute=True,
+            state__in=["new", "sent", "awaiting_expiry"],
+        ).values_list("target", flat=True)
+        return list(targets)
+
+    @staticmethod
+    def get_blocking_ids(identity_pk: int) -> list[int]:
+        targets = Block.objects.filter(
+            source_id=identity_pk,
+            mute=False,
+            state__in=["new", "sent", "awaiting_expiry"],
+        ).values_list("target", flat=True)
+        return list(targets)
+
+    @staticmethod
+    def get_rejecting_ids(identity_pk: int) -> list[int]:
+        pks1 = Block.objects.filter(
+            source_id=identity_pk,
+            mute=False,
+            state__in=["new", "sent", "awaiting_expiry"],
+        ).values_list("target", flat=True)
+        pks2 = Block.objects.filter(
+            target_id=identity_pk,
+            mute=False,
+            state__in=["new", "sent", "awaiting_expiry"],
+        ).values_list("source", flat=True)
+        return list(set(list(pks1) + list(pks2)))
+
+    @staticmethod
+    def block_or_mute(source_pk: int, target_pk: int, is_mute: bool):
+        source = Identity.objects.get(pk=source_pk)
+        if not source.local:
+            raise ValueError(f"Cannot block/mute from remote identity {source}")
+        with transaction.atomic():
+            block, _ = Block.objects.update_or_create(
+                defaults={"state": "new"},
+                source_id=source_pk,
+                target_id=target_pk,
+                mute=is_mute,
+            )
+            if block.state != "new" or not block.uri:
+                block.state = "new"
+                block.uri = source.actor_uri + f"block/{block.pk}/"
+                block.save()
+            if not is_mute:
+                Takahe.unfollow(source_pk, target_pk)
+                Takahe.reject_follow_request(target_pk, source_pk)
+            return block
+
+    @staticmethod
+    def undo_block_or_mute(source_pk: int, target_pk: int, is_mute: bool):
+        Block.objects.filter(
+            source_id=source_pk, target_id=target_pk, mute=is_mute
+        ).update(state="undone")
+
+    @staticmethod
+    def block(source_pk: int, target_pk: int):
+        return Takahe.block_or_mute(source_pk, target_pk, False)
+
+    @staticmethod
+    def unblock(source_pk: int, target_pk: int):
+        return Takahe.undo_block_or_mute(source_pk, target_pk, False)
+
+    @staticmethod
+    def mute(source_pk: int, target_pk: int):
+        return Takahe.block_or_mute(source_pk, target_pk, True)
+
+    @staticmethod
+    def unmute(source_pk: int, target_pk: int):
+        return Takahe.undo_block_or_mute(source_pk, target_pk, True)
+
+    @staticmethod
+    def _force_state_cycle():  # for unit testing only
+        Follow.objects.filter(
+            state__in=["rejecting", "undone", "pending_removal"]
+        ).delete()
+        Follow.objects.all().update(state="accepted")
+        Block.objects.filter(state="new").update(state="sent")
+        Block.objects.exclude(state="sent").delete()
+
+    @staticmethod
+    def post(
+        author_pk: int,
+        pre_conetent: str,
+        content: str,
+        visibility: Visibilities,
+        data: dict | None = None,
+        post_pk: int | None = None,
+        post_time: datetime.datetime | None = None,
+    ) -> int | None:
+        identity = Identity.objects.get(pk=author_pk)
+        post = (
+            Post.objects.filter(author=identity, pk=post_pk).first()
+            if post_pk
+            else None
+        )
+        if post:
+            post.edit_local(
+                pre_conetent, content, visibility=visibility, type_data=data
+            )
+        else:
+            post = Post.create_local(
+                identity,
+                pre_conetent,
+                content,
+                visibility=visibility,
+                type_data=data,
+                published=post_time,
+            )
+        return post.pk if post else None
+
+    @staticmethod
+    def get_post_url(post_pk: int) -> str | None:
+        post = Post.objects.filter(pk=post_pk).first() if post_pk else None
+        return post.object_uri if post else None
+
+    @staticmethod
+    def delete_mark(mark):
+        if mark.shelfmember and mark.shelfmember.post_id:
+            Post.objects.filter(pk=mark.shelfmember.post_id).update(state="deleted")
+
+    @staticmethod
+    def post_mark(mark, share_as_new_post: bool):
+        from catalog.common import ItemCategory
+        from takahe.utils import Takahe
+
+        user = mark.owner.user
+        tags = (
+            "\n"
+            + user.preference.mastodon_append_tag.replace(
+                "[category]", str(ItemCategory(mark.item.category).label)
+            )
+            if user.preference.mastodon_append_tag
+            else ""
+        )
+        stars = _rating_to_emoji(mark.rating_grade, 0)
+        item_link = f"{settings.SITE_INFO['site_url']}/~neodb~{mark.item.url}"
+
+        pre_conetent = (
+            f'{mark.action_label}<a href="{item_link}">《{mark.item.display_title}》</a>'
+        )
+        content = f"{stars}\n{mark.comment_text or ''}{tags}"
+        data = {
+            "object": {
+                "relatedWith": [mark.item.ap_object_ref, mark.shelfmember.ap_object]
+            }
+        }
+        if mark.comment:
+            data["object"]["relatedWith"].append(mark.comment.ap_object)
+        if mark.rating:
+            data["object"]["relatedWith"].append(mark.rating.ap_object)
+        if mark.visibility == 1:
+            v = Takahe.Visibilities.followers
+        elif mark.visibility == 2:
+            v = Takahe.Visibilities.mentioned
+        elif user.preference.mastodon_publish_public:
+            v = Takahe.Visibilities.public
+        else:
+            v = Takahe.Visibilities.unlisted
+        post_pk = Takahe.post(
+            mark.owner.pk,
+            pre_conetent,
+            content,
+            v,
+            data,
+            None if share_as_new_post else mark.shelfmember.post_id,
+            mark.shelfmember.created_time,
+        )
+        if post_pk != mark.shelfmember.post_id:
+            mark.shelfmember.post_id = post_pk
+            mark.shelfmember.save(update_fields=["post_id"])
+        if mark.comment and post_pk != mark.comment.post_id:
+            mark.comment.post_id = post_pk
+            mark.comment.save(update_fields=["post_id"])
+        if mark.rating and post_pk != mark.rating.post_id:
+            mark.rating.post_id = post_pk
+            mark.rating.save(update_fields=["post_id"])
+
+    @staticmethod
+    def interact_post(post_pk: int, identity_pk: int, type: str):
+        post = Post.objects.filter(pk=post_pk).first()
+        if not post:
+            logger.warning(f"Cannot find post {post_pk}")
+            return
+        interaction = PostInteraction.objects.get_or_create(
+            type=type,
+            identity_id=identity_pk,
+            post=post,
+        )[0]
+        if interaction.state not in ["new", "fanned_out"]:
+            interaction.state = "new"
+            interaction.save()
+        post.calculate_stats()
+        return interaction
+
+    @staticmethod
+    def uninteract_post(post_pk: int, identity_pk: int, type: str):
+        post = Post.objects.filter(pk=post_pk).first()
+        if not post:
+            logger.warning(f"Cannot find post {post_pk}")
+            return
+        for interaction in PostInteraction.objects.filter(
+            type=type,
+            identity_id=identity_pk,
+            post=post,
+        ):
+            interaction.state = "undone"
+            interaction.save()
+        post.calculate_stats()
+
+    @staticmethod
+    def like_post(post_pk: int, identity_pk: int):
+        return Takahe.interact_post(post_pk, identity_pk, "like")
+
+    @staticmethod
+    def unlike_post(post_pk: int, identity_pk: int):
+        return Takahe.uninteract_post(post_pk, identity_pk, "like")
+
+    @staticmethod
+    def post_liked_by(post_pk: int, identity_pk: int) -> bool:
+        interaction = Takahe.get_user_interaction(post_pk, identity_pk, "like")
+        return interaction is not None and interaction.state in ["new", "fanned_out"]
+
+    @staticmethod
+    def get_user_interaction(post_pk: int, identity_pk: int, type: str):
+        post = Post.objects.filter(pk=post_pk).first()
+        if not post:
+            logger.warning(f"Cannot find post {post_pk}")
+            return None
+        return PostInteraction.objects.filter(
+            type=type,
+            identity_id=identity_pk,
+            post=post,
+        ).first()
+
+    @staticmethod
+    def get_post_stats(post_pk: int) -> dict:
+        post = Post.objects.filter(pk=post_pk).first()
+        if not post:
+            logger.warning(f"Cannot find post {post_pk}")
+            return {}
+        return post.stats or {}

takahe/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

users/account.py

@@ -396,6 +396,7 @@ def register(request):
             )
             messages.add_message(request, messages.INFO, _("已发送验证邮件，请查收。"))
         if username_changed:
+            request.user.initiatialize()
             messages.add_message(request, messages.INFO, _("用户名已设置。"))
         if email_cleared:
             messages.add_message(request, messages.INFO, _("电子邮件地址已取消关联。"))
@@ -480,9 +481,9 @@ def auth_logout(request):
 def clear_data_task(user_id):
     user = User.objects.get(pk=user_id)
     user_str = str(user)
-    remove_data_by_user(user)
+    if user.identity:
+        remove_data_by_user(user.identity)
     user.clear()
-    user.save()
     logger.warning(f"User {user_str} data cleared.")
 
 

users/management/commands/refresh_following.py

@@ -1,21 +0,0 @@
-from datetime import timedelta
-
-from django.core.management.base import BaseCommand
-from django.utils import timezone
-from tqdm import tqdm
-
-from users.models import User
-
-
-class Command(BaseCommand):
-    help = "Refresh following data for all users"
-
-    def handle(self, *args, **options):
-        count = 0
-        for user in tqdm(User.objects.all()):
-            user.following = user.merged_following_ids()
-            if user.following:
-                count += 1
-                user.save(update_fields=["following"])
-
-        print(f"{count} users updated")

users/migrations/0012_apidentity.py

@@ -0,0 +1,63 @@
+# Generated by Django 4.2.4 on 2023-08-09 13:37
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    # replaces = [
+    #     ("users", "0012_user_local"),
+    #     ("users", "0013_user_identity"),
+    #     ("users", "0014_remove_user_identity_apidentity_user"),
+    #     ("users", "0015_alter_apidentity_user"),
+    # ]
+
+    dependencies = [
+        ("users", "0011_preference_hidden_categories"),
+        ("takahe", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="APIdentity",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("local", models.BooleanField()),
+                ("username", models.CharField(blank=True, max_length=500, null=True)),
+                (
+                    "domain_name",
+                    models.CharField(blank=True, max_length=500, null=True),
+                ),
+                ("deleted", models.DateTimeField(blank=True, null=True)),
+                (
+                    "user",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="identity",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "indexes": [
+                    models.Index(
+                        fields=["local", "username"],
+                        name="users_apide_local_2d8170_idx",
+                    ),
+                    models.Index(
+                        fields=["domain_name", "username"],
+                        name="users_apide_domain__53ffa5_idx",
+                    ),
+                ],
+            },
+        ),
+    ]

users/migrations/0013_init_identity.py

@@ -0,0 +1,77 @@
+# Generated by Django 4.2.4 on 2023-08-09 16:54
+
+from django.conf import settings
+from django.db import migrations, models, transaction
+from loguru import logger
+from tqdm import tqdm
+
+from takahe.models import Domain as TakaheDomain
+from takahe.models import Identity as TakaheIdentity
+from takahe.models import User as TakaheUser
+
+domain = settings.SITE_INFO["site_domain"]
+service_domain = settings.SITE_INFO.get("site_service_domain")
+
+
+def init_domain(apps, schema_editor):
+    d = TakaheDomain.objects.filter(domain=domain).first()
+    if not d:
+        logger.info(f"Creating takahe domain {domain}")
+        TakaheDomain.objects.create(
+            domain=domain,
+            local=True,
+            service_domain=service_domain,
+            notes="NeoDB",
+            nodeinfo={},
+        )
+    else:
+        logger.info(f"Takahe domain {domain} already exists")
+
+
+def init_identity(apps, schema_editor):
+    User = apps.get_model("users", "User")
+    APIdentity = apps.get_model("users", "APIdentity")
+    tdomain = TakaheDomain.objects.filter(domain=domain).first()
+    if User.objects.filter(username__isnull=True).exists():
+        raise ValueError("null username detected, aborting migration")
+    if TakaheUser.objects.exists():
+        raise ValueError("existing Takahe users detected, aborting migration")
+    if TakaheIdentity.objects.exists():
+        raise ValueError("existing Takahe identities detected, aborting migration")
+    if APIdentity.objects.exists():
+        raise ValueError("existing APIdentity data detected, aborting migration")
+    logger.info(f"Creating takahe users/identities")
+    for user in tqdm(User.objects.all()):
+        username = user.username
+        handler = "@" + username
+        identity = APIdentity.objects.create(
+            pk=user.pk,
+            user=user,
+            local=True,
+            username=username,
+            domain_name=domain,
+            deleted=None if user.is_active else user.updated,
+        )
+        takahe_user = TakaheUser.objects.create(pk=user.pk, email=handler)
+        takahe_identity = TakaheIdentity.objects.create(
+            pk=user.pk,
+            actor_uri=f"https://{service_domain or domain}/@{username}@{domain}/",
+            username=username,
+            domain=tdomain,
+            name=username,
+            local=True,
+            discoverable=not user.preference.no_anonymous_view,
+        )
+        takahe_user.identities.add(takahe_identity)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("users", "0012_apidentity"),
+    ]
+
+    operations = [
+        migrations.RunPython(init_domain),
+        migrations.RunPython(init_identity),
+    ]