improve identity deletion

journal/models/__init__.py

@@ -24,7 +24,7 @@ from .shelf import Shelf, ShelfLogEntry, ShelfManager, ShelfMember, ShelfType
 from .tag import Tag, TagManager, TagMember
 from .utils import (
     journal_exists_for_item,
-    remove_data_by_user,
+    remove_data_by_identity,
     reset_journal_visibility_for_user,
     update_journal_for_merged_item,
     update_journal_for_merged_item_task,
@@ -63,7 +63,7 @@ __all__ = [
     "TagManager",
     "TagMember",
     "journal_exists_for_item",
-    "remove_data_by_user",
+    "remove_data_by_identity",
     "reset_journal_visibility_for_user",
     "update_journal_for_merged_item",
     "update_journal_for_merged_item_task",

journal/models/utils.py

@@ -4,6 +4,7 @@ from django.db.utils import IntegrityError
 from loguru import logger
 
 from catalog.models import Item
+from journal.models.index import JournalIndex
 from users.models import APIdentity, User
 
 from .collection import Collection, CollectionMember, FeaturedCollection
@@ -24,7 +25,7 @@ def reset_journal_visibility_for_user(owner: APIdentity, visibility: int):
     Review.objects.filter(owner=owner).update(visibility=visibility)
 
 
-def remove_data_by_user(owner: APIdentity):
+def remove_data_by_identity(owner: APIdentity):
     ShelfMember.objects.filter(owner=owner).delete()
     ShelfLogEntry.objects.filter(owner=owner).delete()
     Comment.objects.filter(owner=owner).delete()
@@ -36,6 +37,9 @@ def remove_data_by_user(owner: APIdentity):
     CollectionMember.objects.filter(owner=owner).delete()
     Collection.objects.filter(owner=owner).delete()
     FeaturedCollection.objects.filter(owner=owner).delete()
+    index = JournalIndex.instance()
+    index.delete_by_owner(owner.pk)
+    logger.info(f"removed journal data by {owner}")
 
 
 def update_journal_for_merged_item_task(editing_user_id: int, legacy_item_uuid: str):

neodb-takahe

@@ -1 +1 @@
-Subproject commit 549f58eb472504f8281d20a7000ab8cc6834f611
+Subproject commit 40dc947009e60e429bfe78a92917209f92106112

takahe/ap_handlers.py

@@ -222,6 +222,18 @@ def post_uninteracted(interaction_pk, interaction, post_pk, identity_pk):
     ).delete()
 
 
+def identity_deleted(pk):
+    apid = APIdentity.objects.filter(pk=pk).first()
+    if not apid:
+        logger.warning(f"APIdentity {apid} not found")
+        return
+
+    logger.warning(f"handle deleting identity {apid}")
+    if apid.user and apid.user.is_active:
+        apid.user.clear()  # for local identity, clear their user as well
+    apid.clear()
+
+
 def identity_fetched(pk):
     try:
         identity = Identity.objects.get(pk=pk)

takahe/utils.py

@@ -125,16 +125,20 @@ class Takahe:
         ).first()
 
     @staticmethod
-    def delete_identity(identity_pk: int):
+    def request_delete_identity(identity_pk: int):
-        identity = Identity.objects.filter(pk=identity_pk).first()
+        from journal.models import remove_data_by_identity
-        if not identity:
+        from users.models import APIdentity
-            logger.warning(f"Cannot find identity {identity_pk}")
+
-            return
+        i = Identity.objects.filter(pk=identity_pk).first()
-        logger.warning(f"Deleting identity {identity}")
+        if i:
-        identity.state = "deleted"
+            InboxMessage.create_internal(
-        identity.deleted = timezone.now()
+                {"type": "DeleteIdentity", "actor": i.actor_uri}
-        identity.state_next_attempt = timezone.now()
+            )
-        identity.save()
+            logger.warning(f"Requested identity {i} deletion")
+        else:
+            logger.error(f"Identity {i} not found, force delete APIdentity")
+            apid = APIdentity.objects.get(pk=identity_pk)
+            remove_data_by_identity(apid)
 
     @staticmethod
     def create_internal_message(message: dict):

users/management/commands/user.py

@@ -1,3 +1,5 @@
+from time import sleep
+
 import httpx
 from django.core.management.base import BaseCommand
 from tqdm import tqdm
@@ -5,6 +7,7 @@ from tqdm import tqdm
 from takahe.models import Domain, Identity
 from takahe.utils import Takahe
 from users.models import Preference, User
+from users.models.apidentity import APIdentity
 
 
 class Command(BaseCommand):
@@ -48,6 +51,9 @@ class Command(BaseCommand):
             self.staff(options["staff"])
         if options["active"]:
             self.set_active(options["active"])
+        if options["delete"]:
+            if input("Are you sure to delete? [Y/N] ").startswith("Y"):
+                self.delete(options["delete"])
 
     def list(self, users):
         for user in users:
@@ -164,3 +170,34 @@ class Command(BaseCommand):
             u.is_active = not u.is_active
             u.save()
             self.stdout.write(f"update {u} is_active: {u.is_active}")
+
+    def delete(self, v):
+        for n in v:
+            try:
+                apid = APIdentity.get_by_handle(n)
+                if apid.deleted:
+                    self.stdout.write(f"{apid} already deleted, try anyway")
+                apid.clear()
+                if apid.user:
+                    apid.user.clear()
+                Takahe.request_delete_identity(apid.pk)
+                count_down = 10
+                while count_down > 0:
+                    i = Identity.objects.filter(pk=apid.pk).first()
+                    if i and i.state != "deleted_fanned_out":
+                        self.stdout.write(f"waiting for takahe-stator...{count_down}")
+                        sleep(1)
+                    else:
+                        break
+                    count_down -= 1
+                if count_down == 0:
+                    self.stdout.write(
+                        self.style.WARNING(
+                            f"Identity {apid} was deleted, but some data in takahe has not been fully processed yet, make sure takahe-stator is running and wait a bit."
+                        )
+                    )
+                else:
+                    self.stdout.write(f"Deleted identity {apid}")
+            except APIdentity.DoesNotExist:
+                self.stdout.write(f"identity {n} not found")
+                continue

users/models/apidentity.py

@@ -2,6 +2,8 @@ from functools import cached_property
 
 from django.conf import settings
 from django.db import models
+from django.utils import timezone
+from loguru import logger
 
 from mastodon.models.mastodon import MastodonAccount
 from takahe.utils import Takahe
@@ -288,3 +290,12 @@ class APIdentity(models.Model):
         from journal.models import TagManager
 
         return TagManager(self)
+
+    def clear(self):
+        """delete data for this identity"""
+        from journal.models import remove_data_by_identity
+
+        remove_data_by_identity(self)
+        self.deleted = timezone.now()
+        self.save()
+        logger.warning(f"Identity {self} cleared.")

users/models/user.py

@@ -12,7 +12,7 @@ from django.core.files.base import ContentFile
 from django.db import models, transaction
 from django.db.models.functions import Lower
 from django.urls import reverse
-from django.utils import timezone, translation
+from django.utils import translation
 from django.utils.deconstruct import deconstructible
 from django.utils.translation import gettext_lazy as _
 from loguru import logger
@@ -199,19 +199,15 @@ class User(AbstractUser):
         return p.edited_time if p else None
 
     def clear(self):
-        if not self.is_active:
-            return
         with transaction.atomic():
             accts = [str(a) for a in self.social_accounts.all()]
-            self.first_name = (";").join(accts)
+            if accts:
+                self.first_name = (";").join(accts)
             self.last_name = self.username
             self.is_active = False
-            # self.username = "~removed~" + str(self.pk)
-            # to get ready for federation, username has to be reserved
             self.save()
-            self.identity.deleted = timezone.now()
-            self.identity.save()
             self.social_accounts.all().delete()
+            logger.warning(f"User {self} cleared.")
 
     def sync_identity(self):
         """sync display name, bio, and avatar from available sources"""

users/views/account.py

@@ -1,6 +1,5 @@
 from urllib.parse import quote
 
-import django_rq
 from django import forms
 from django.conf import settings
 from django.contrib import auth, messages
@@ -11,11 +10,8 @@ from django.shortcuts import redirect, render
 from django.urls import reverse
 from django.utils.translation import gettext as _
 from django.views.decorators.http import require_http_methods
-from loguru import logger
 
 from common.utils import AuthedHttpRequest
-from journal.models import remove_data_by_user
-from journal.models.index import JournalIndex
 from mastodon.models import Email, Mastodon
 from mastodon.models.common import Platform, SocialAccount
 from mastodon.models.email import EmailAccount
@@ -222,33 +218,28 @@ def auth_logout(request):
     return logout_takahe(redirect("/"))
 
 
-def clear_data_task(user_id):
+@require_http_methods(["POST"])
-    user = User.objects.get(pk=user_id)
-    user_str = str(user)
-    if user.identity:
-        remove_data_by_user(user.identity)
-    Takahe.delete_identity(user.identity.pk)
-    user.clear()
-    index = JournalIndex.instance()
-    index.delete_by_owner(user.identity.pk)
-    logger.warning(f"User {user_str} data cleared.")
-
-
 @login_required
 def clear_data(request):
+    # for deletion initiated by local identity in neodb:
+    # 1. clear user data
+    # 2. neodb send DeleteIdentity to Takahe
+    # 3. takahe delete identity and send identity_deleted to neodb
+    # 4. identity_deleted clear user (if not yet) and identity data
+    # 5. log web user out
+    # for deletion initiated by remote/local identity in takahe:
+    # just 3 & 4
     if request.META.get("HTTP_AUTHORIZATION"):
         raise BadRequest("Only for web login")
-    if request.method == "POST":
+    v = request.POST.get("verification", "").strip()
-        v = request.POST.get("verification", "").strip()
+    if v:
-        if v:
+        for acct in request.user.social_accounts.all():
-            for acct in request.user.social_accounts.all():
+            if acct.handle == v:
-                if acct.handle == v:
+                request.user.clear()
-                    django_rq.get_queue("mastodon").enqueue(
+                Takahe.request_delete_identity(request.user.identity.pk)
-                        clear_data_task, request.user.id
+                messages.add_message(
-                    )
+                    request, messages.INFO, _("Account is being deleted.")
-                    messages.add_message(
+                )
-                        request, messages.INFO, _("Account is being deleted.")
+                return auth_logout(request)
-                    )
+    messages.add_message(request, messages.ERROR, _("Account mismatch."))
-                    return auth_logout(request)
-        messages.add_message(request, messages.ERROR, _("Account mismatch."))
     return redirect(reverse("users:data"))