migrate and sync user relationship in Mastodon

common/management/commands/cron.py

@@ -3,6 +3,7 @@ from loguru import logger
 
 from catalog.jobs import *  # noqa
 from common.models import JobManager
+from users.jobs import *  # noqa
 
 
 class Command(BaseCommand):

neodb-takahe

@@ -1 +1 @@
-Subproject commit 25ead63fb1cacb34cf3f8a2e0706843636f78034
+Subproject commit 401be8d99e793f05761802ea7ebaae8449a44974

users/data.py

@@ -13,7 +13,6 @@ from journal.importers.douban import DoubanImporter
 from journal.importers.goodreads import GoodreadsImporter
 from journal.importers.opml import OPMLImporter
 from journal.models import reset_journal_visibility_for_user
-from mastodon import mastodon_request_included
 from mastodon.api import *
 from social.models import reset_social_visibility_for_user
 
@@ -21,7 +20,6 @@ from .account import *
 from .tasks import *
 
 
-@mastodon_request_included
 @login_required
 def preferences(request):
     preference = request.user.preference
@@ -54,7 +52,6 @@ def preferences(request):
     return render(request, "users/preferences.html")
 
 
-@mastodon_request_included
 @login_required
 def data(request):
     return render(
@@ -79,7 +76,6 @@ def data_import_status(request):
     )
 
 
-@mastodon_request_included
 @login_required
 def export_reviews(request):
     if request.method != "POST":
@@ -87,7 +83,6 @@ def export_reviews(request):
     return render(request, "users/data.html")
 
 
-@mastodon_request_included
 @login_required
 def export_marks(request):
     if request.method == "POST":
@@ -120,6 +115,20 @@ def sync_mastodon(request):
     return redirect(reverse("users:info"))
 
 
+@login_required
+def sync_mastodon_preference(request):
+    if request.method == "POST":
+        request.user.preference.mastodon_skip_userinfo = (
+            request.POST.get("mastodon_sync_userinfo", "") == ""
+        )
+        request.user.preference.mastodon_skip_relationship = (
+            request.POST.get("mastodon_sync_relationship", "") == ""
+        )
+        request.user.preference.save()
+        messages.add_message(request, messages.INFO, _("同步设置已保存。"))
+    return redirect(reverse("users:info"))
+
+
 @login_required
 def reset_visibility(request):
     if request.method == "POST":

users/jobs/__init__.py

@@ -0,0 +1 @@
+from .sync import MastodonUserSync

users/jobs/sync.py

@@ -0,0 +1,35 @@
+import pprint
+from datetime import timedelta
+from time import sleep
+
+from django.utils import timezone
+from loguru import logger
+
+from common.models import BaseJob, JobManager
+from users.models import Preference, User
+
+
+@JobManager.register
+class MastodonUserSync(BaseJob):
+    interval = timedelta(hours=2)
+
+    def run(self):
+        logger.info("Mastodon User Sync start.")
+        count = 0
+        ttl_hours = 12
+        qs = (
+            User.objects.exclude(
+                preference__mastodon_skip_userinfo=True, mastodon_skip_relationship=True
+            )
+            .filter(
+                mastodon_last_refresh__lt=timezone.now() - timedelta(hours=ttl_hours)
+            )
+            .filter(
+                is_active=True,
+            )
+            .exclude(mastodon_token__isnull=True)
+            .exclude(mastodon_token="")
+        )
+        for user in qs:
+            user.refresh_mastodon_data()
+        logger.info(f"Mastodon User Sync finished.")

users/management/commands/refresh_mastodon.py

@@ -1,10 +0,0 @@
-from django.core.management.base import BaseCommand
-
-from users.tasks import refresh_all_mastodon_data_task
-
-
-class Command(BaseCommand):
-    help = "Refresh Mastodon data for all users if not updated in last 24h"
-
-    def handle(self, *args, **options):
-        refresh_all_mastodon_data_task(24)

users/migrations/0014_preference_mastodon_skip_relationship_and_more.py

@@ -0,0 +1,44 @@
+# Generated by Django 4.2.7 on 2023-11-06 01:46
+
+from django.db import migrations, models
+from loguru import logger
+from tqdm import tqdm
+
+
+def migrate_relationships(apps, schema_editor):
+    User = apps.get_model("users", "User")
+    APIdentity = apps.get_model("users", "APIdentity")
+    logger.info(f"Migrate user relationship")
+    for user in tqdm(User.objects.all()):
+        for target in user.local_following:
+            user.identity.follow(User.objects.get(pk=target).identity)
+        for target in user.local_blocking:
+            user.identity.block(User.objects.get(pk=target).identity)
+        for target in user.local_muting:
+            user.identity.block(User.objects.get(pk=target).identity)
+        user.sync_relationship()
+    for user in tqdm(User.objects.all()):
+        for req in user.identity.following_request:
+            target_identity = APIdentity.objects.get(pk=req)
+            target_identity.accept_follow_request(user.identity)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("users", "0013_init_identity"),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_relationships),
+        migrations.AddField(
+            model_name="preference",
+            name="mastodon_skip_relationship",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="preference",
+            name="mastodon_skip_userinfo",
+            field=models.BooleanField(default=False),
+        ),
+    ]

users/models/apidentity.py

@@ -82,7 +82,7 @@ class APIdentity(models.Model):
             return (
                 self.takahe_identity.icon.url
                 if self.takahe_identity.icon
-                else settings.SITE_INFO["user_icon"]
+                else self.takahe_identity.icon_uri or settings.SITE_INFO["user_icon"]
             )
         else:
             return f"/proxy/identity_icon/{self.pk}/"

users/models/preference.py

@@ -49,6 +49,8 @@ class Preference(models.Model):
     show_last_edit = models.PositiveSmallIntegerField(default=0)
     no_anonymous_view = models.PositiveSmallIntegerField(default=0)
     hidden_categories = models.JSONField(default=list)
+    mastodon_skip_userinfo = models.BooleanField(null=False, default=False)
+    mastodon_skip_relationship = models.BooleanField(null=False, default=False)
 
     def __str__(self):
         return str(self.user)

users/models/user.py

@@ -231,12 +231,44 @@ class User(AbstractUser):
         self.identity.deleted = timezone.now()
         self.identity.save()
 
-    def sync_relationships(self):
-        # FIXME
-        pass
+    def sync_relationship(self):
+        for target in self.mastodon_followers:
+            t = target.split("@")
+            target_user = User.objects.filter(
+                mastodon_username=t[0], mastodon_site=t[1]
+            ).first()
+            if target_user and not self.identity.is_following(target_user.identity):
+                self.identity.follow(target_user.identity)
+        for target in self.mastodon_blocks:
+            t = target.split("@")
+            target_user = User.objects.filter(
+                mastodon_username=t[0], mastodon_site=t[1]
+            ).first()
+            if target_user and not self.identity.is_blocking(target_user.identity):
+                self.identity.block(target_user.identity)
+        for target in self.mastodon_mutes:
+            t = target.split("@")
+            target_user = User.objects.filter(
+                mastodon_username=t[0], mastodon_site=t[1]
+            ).first()
+            if target_user and not self.identity.is_muting(target_user.identity):
+                self.identity.mute(target_user.identity)
+
+    def sync_identity(self):
+        identity = self.identity.takahe_identity
+        identity.name = (
+            self.mastodon_account.get("display_name")
+            or identity.name
+            or identity.username
+        )
+        identity.summary = self.mastodon_account.get("note") or identity.summary
+        identity.manually_approves_followers = self.mastodon_locked
+        identity.icon_uri = self.mastodon_account.get("avatar")
+        identity.save()
 
     def refresh_mastodon_data(self):
         """Try refresh account data from mastodon server, return true if refreshed successfully, note it will not save to db"""
+        logger.debug(f"Refreshing Mastodon data for {self}")
         self.mastodon_last_refresh = timezone.now()
         code, mastodon_account = verify_account(self.mastodon_site, self.mastodon_token)
         if code == 401 and self.mastodon_refresh_token:
@@ -247,7 +279,6 @@ class User(AbstractUser):
                 code, mastodon_account = verify_account(
                     self.mastodon_site, self.mastodon_token
                 )
-        updated = False
         if mastodon_account:
             self.mastodon_account = mastodon_account
             self.mastodon_locked = mastodon_account["locked"]
@@ -277,12 +308,17 @@ class User(AbstractUser):
             self.mastodon_domain_blocks = get_related_acct_list(
                 self.mastodon_site, self.mastodon_token, "/api/v1/domain_blocks"
             )
-            self.sync_relationships()
-            updated = True
+            self.save()
+            if not self.preference.mastodon_skip_userinfo:
+                self.sync_identity()
+            if not self.preference.mastodon_skip_relationship:
+                self.sync_relationship()
+            return True
         elif code == 401:
             logger.error(f"Refresh mastodon data error 401 for {self}")
             self.mastodon_token = ""
-        return updated
+            self.save(update_fields=["mastodon_token"])
+        return False
 
     @property
     def unread_announcements(self):

users/tasks.py

@@ -16,30 +16,6 @@ def refresh_mastodon_data_task(user_id, token=None):
     if token:
         user.mastodon_token = token
     if user.refresh_mastodon_data():
-        user.save()
         logger.info(f"{user} mastodon data refreshed")
     else:
         logger.warning(f"{user} mastodon data refresh failed")
-
-
-def refresh_all_mastodon_data_task(ttl_hours):
-    logger.info(f"Mastodon data refresh start")
-    count = 0
-    for user in tqdm(
-        User.objects.filter(
-            mastodon_last_refresh__lt=timezone.now() - timedelta(hours=ttl_hours),
-            is_active=True,
-        )
-    ):
-        if user.mastodon_token or user.mastodon_refresh_token:
-            logger.info(f"Refreshing {user}")
-            if user.refresh_mastodon_data():
-                logger.info(f"Refreshed {user}")
-                count += 1
-            else:
-                logger.warning(f"Refresh failed for {user}")
-            user.save()
-        else:
-            logger.warning(f"Missing token for {user}")
-    logger.info(f"{count} users updated")
-    logger.info(f"Mastodon data refresh done")

users/templates/users/account.html

@@ -139,20 +139,49 @@
         </article>
         <article>
           <details>
-            <summary>{% trans '同步联邦宇宙身份和社交关系数据' %}</summary>
+            <summary>{% trans '同步联邦宇宙信息和社交数据' %}</summary>
+            <form action="{% url 'users:sync_mastodon_preference' %}"
+                  method="post"
+                  enctype="multipart/form-data">
+              {% csrf_token %}
+              <fieldset>
+                <label>
+                  <input type="checkbox"
+                         name="mastodon_sync_userinfo"
+                         {% if not request.user.preference.mastodon_skip_userinfo %}checked{% endif %}>
+                  {% trans '自动同步用户昵称等基本信息' %}
+                </label>
+              </fieldset>
+              <fieldset>
+                <label>
+                  <input type="checkbox"
+                         name="mastodon_sync_relationship"
+                         {% if not request.user.preference.mastodon_skip_relationship %}checked{% endif %}>
+                  {% trans '自动导入新增的关注、屏蔽和隐藏列表' %}
+                </label>
+              </fieldset>
+              <input type="submit"
+                     value="{% trans '保存同步设置' %}"
+                     {% if not request.user.mastodon_username %}disabled{% endif %} />
+              <small>
+                {{ site_name }}会按照以上设置每天自动导入你在联邦宇宙实例中新增的关注、屏蔽和隐藏列表；
+                <br>
+                如果你在联邦宇宙实例中关注的用户加入了NeoDB，你会自动关注她；
+                <br>
+                如果你在联邦宇宙实例中取消了关注、屏蔽或隐藏，{{ site_name }}不会自动取消，但你可以手动移除。
+              </small>
+            </form>
             <form action="{% url 'users:sync_mastodon' %}"
                   method="post"
                   enctype="multipart/form-data">
               {% csrf_token %}
+              <small>如果希望立即开始同步，可以点击下方按钮。</small>
               <input type="submit"
-                     value="{% trans '同步' %}"
+                     value="{% trans '立即同步' %}"
                      {% if not request.user.mastodon_username %}disabled{% endif %} />
               <small>
                 {% if request.user.mastodon_last_refresh %}上次更新时间 {{ request.user.mastodon_last_refresh }}{% endif %}
               </small>
-              <div>
-                为了正确高效的展示短评和评论，{{ site_name }}会缓存你在联邦宇宙的关注、屏蔽和隐藏列表。如果你刚刚更新过帐户的上锁状态、增减过关注、隐藏或屏蔽，希望立即生效，可以点击这里立刻更新；这类信息也会每天自动同步。
-              </div>
             </form>
           </details>
         </article>

users/urls.py

@@ -23,6 +23,11 @@ urlpatterns = [
     path("data/export/reviews", export_reviews, name="export_reviews"),
     path("data/export/marks", export_marks, name="export_marks"),
     path("data/sync_mastodon", sync_mastodon, name="sync_mastodon"),
+    path(
+        "data/sync_mastodon_preference",
+        sync_mastodon_preference,
+        name="sync_mastodon_preference",
+    ),
     path("data/reset_visibility", reset_visibility, name="reset_visibility"),
     path("data/clear_data", clear_data, name="clear_data"),
     path("preferences", preferences, name="preferences"),