diff --git a/doc/social.md b/doc/social.md index f7cfbd0d..1b31d102 100644 --- a/doc/social.md +++ b/doc/social.md @@ -42,7 +42,7 @@ These are list of activities should be either shown in the site or delivered as - `Add` / `Remove` an *Item* to / from a *List*: + add / remove *Item* to / from a user *Collection* - + mark *Item* as wished / started / done, which are essentially add to / remove from user's predefined *Collection* + + mark *Item* as wishlist / progress / complete, which are essentially add to / remove from user's predefined *Collection* - `Create` / `Update` / `Delete` a user *Collection* - `Create` / `Update` / `Delete` a *Content* with an `Object Link` to *Item* + `Create` / `Update` / `Delete` a *Comment* or *Review* diff --git a/journal/models.py b/journal/models.py index 04501d94..267a5010 100644 --- a/journal/models.py +++ b/journal/models.py @@ -16,9 +16,11 @@ from functools import cached_property from django.db.models import Count, Avg import django.dispatch import math +import uuid class Piece(PolymorphicModel, UserOwnedObjectMixin): + uid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True) owner = models.ForeignKey(User, on_delete=models.PROTECT) visibility = models.PositiveSmallIntegerField(default=0) # 0: Public / 1: Follower only / 2: Self only created_time = models.DateTimeField(auto_now_add=True) @@ -47,13 +49,13 @@ class Comment(Content): @staticmethod def comment_item_by_user(item, user, text, visibility=0): comment = Comment.objects.filter(owner=user, item=item).first() - if text is None: + if not text: if comment is not None: comment.delete() comment = None elif comment is None: comment = Comment.objects.create(owner=user, item=item, text=text, visibility=visibility) - else: + elif comment.text != text or comment.visibility != visibility: comment.text = text comment.visibility = visibility comment.save() @@ -99,16 +101,21 @@ class Rating(Content): return stat['count'] @staticmethod - def set_item_rating_by_user(item, rating_grade, user, visibility=0): - if rating_grade is not None and (rating_grade < 1 or rating_grade > 10): + def rate_item_by_user(item, user, rating_grade, visibility=0): + if not rating_grade and (rating_grade < 1 or rating_grade > 10): raise ValueError(f'Invalid rating grade: {rating_grade}') rating = Rating.objects.filter(owner=user, item=item).first() - if not rating: + if not rating_grade: + if rating: + rating.delete() + rating = None + elif rating is None: rating = Rating.objects.create(owner=user, item=item, grade=rating_grade, visibility=visibility) - else: + elif rating.grade != rating_grade or rating.visibility != visibility: rating.visibility = visibility rating.grade = rating_grade rating.save() + return rating @staticmethod def get_item_rating_by_user(item, user): @@ -173,8 +180,8 @@ class List(Piece): def remove_item(self, item): member = self.members.all().filter(item=item).first() - list_remove.send(sender=self.__class__, instance=self, item=item, member=member) if member: + list_remove.send(sender=self.__class__, instance=self, item=item, member=member) member.delete() def move_up_item(self, item): @@ -228,29 +235,29 @@ Shelf class ShelfType(models.TextChoices): - WISHED = ('wished', '未开始') - STARTED = ('started', '进行中') - DONE = ('done', '完成') + WISHLIST = ('wishlist', '未开始') + PROGRESS = ('progress', '进行中') + COMPLETE = ('complete', '完成') # DISCARDED = ('discarded', '放弃') ShelfTypeNames = [ - [ItemCategory.Book, ShelfType.WISHED, _('想读')], - [ItemCategory.Book, ShelfType.STARTED, _('在读')], - [ItemCategory.Book, ShelfType.DONE, _('读过')], - [ItemCategory.Movie, ShelfType.WISHED, _('想看')], - [ItemCategory.Movie, ShelfType.STARTED, _('在看')], - [ItemCategory.Movie, ShelfType.DONE, _('看过')], - [ItemCategory.TV, ShelfType.WISHED, _('想看')], - [ItemCategory.TV, ShelfType.STARTED, _('在看')], - [ItemCategory.TV, ShelfType.DONE, _('看过')], - [ItemCategory.Music, ShelfType.WISHED, _('想听')], - [ItemCategory.Music, ShelfType.STARTED, _('在听')], - [ItemCategory.Music, ShelfType.DONE, _('听过')], - [ItemCategory.Game, ShelfType.WISHED, _('想玩')], - [ItemCategory.Game, ShelfType.STARTED, _('在玩')], - [ItemCategory.Game, ShelfType.DONE, _('玩过')], - [ItemCategory.Collection, ShelfType.WISHED, _('关注')], + [ItemCategory.Book, ShelfType.WISHLIST, _('想读')], + [ItemCategory.Book, ShelfType.PROGRESS, _('在读')], + [ItemCategory.Book, ShelfType.COMPLETE, _('读过')], + [ItemCategory.Movie, ShelfType.WISHLIST, _('想看')], + [ItemCategory.Movie, ShelfType.PROGRESS, _('在看')], + [ItemCategory.Movie, ShelfType.COMPLETE, _('看过')], + [ItemCategory.TV, ShelfType.WISHLIST, _('想看')], + [ItemCategory.TV, ShelfType.PROGRESS, _('在看')], + [ItemCategory.TV, ShelfType.COMPLETE, _('看过')], + [ItemCategory.Music, ShelfType.WISHLIST, _('想听')], + [ItemCategory.Music, ShelfType.PROGRESS, _('在听')], + [ItemCategory.Music, ShelfType.COMPLETE, _('听过')], + [ItemCategory.Game, ShelfType.WISHLIST, _('想玩')], + [ItemCategory.Game, ShelfType.PROGRESS, _('在玩')], + [ItemCategory.Game, ShelfType.COMPLETE, _('玩过')], + [ItemCategory.Collection, ShelfType.WISHLIST, _('关注')], # TODO add more combinations ] @@ -327,6 +334,7 @@ class ShelfManager: # metadata=None means no change if not item: raise ValueError('empty item') + new_shelfmember = None last_shelfmember = self._shelf_member_for_item(item) last_shelf = last_shelfmember._shelf if last_shelfmember else None last_metadata = last_shelfmember.metadata if last_shelfmember else None @@ -338,10 +346,11 @@ class ShelfManager: if last_shelf: last_shelf.remove_item(item) if shelf: - shelf.append_item(item, visibility=visibility, metadata=metadata or {}) + new_shelfmember = shelf.append_item(item, visibility=visibility, metadata=metadata or {}) elif last_shelf is None: raise ValueError('empty shelf') else: + new_shelfmember = last_shelfmember if metadata is not None and metadata != last_metadata: # change metadata changed = True last_shelfmember.metadata = metadata @@ -354,6 +363,7 @@ class ShelfManager: if metadata is None: metadata = last_metadata or {} ShelfLogEntry.objects.create(owner=self.owner, shelf=shelf, item=item, metadata=metadata) + return new_shelfmember def get_log(self): return ShelfLogEntry.objects.filter(owner=self.owner).order_by('timestamp') @@ -443,6 +453,19 @@ class TagManager: tags = user.tag_set.all().values('title').annotate(frequency=Count('members')).order_by('-frequency') return list(map(lambda t: t['title'], tags)) + @staticmethod + def tag_item_by_user(item, user, tag_titles, default_visibility=0): + titles = set([Tag.cleanup_title(tag_title) for tag_title in tag_titles]) + current_titles = set([m._tag.title for m in TagMember.objects.filter(owner=user, item=item)]) + for title in titles - current_titles: + tag = Tag.objects.filter(owner=user, title=title).first() + if not tag: + tag = Tag.objects.create(owner=user, title=title, visibility=default_visibility) + tag.append_item(item) + for title in current_titles - titles: + tag = Tag.objects.filter(owner=user, title=title).first() + tag.remove_item(item) + @staticmethod def add_tag_by_user(item, tag_title, user, default_visibility=0): title = Tag.cleanup_title(tag_title) @@ -526,12 +549,23 @@ class Mark: def review(self): return Review.objects.filter(owner=self.owner, item=self.item).first() - def update(self, shelf_type, comment_text, rating_grade, visibility): + def update(self, shelf_type, comment_text, rating_grade, visibility, metadata=None, created_time=None): if shelf_type != self.shelf_type or visibility != self.visibility: - self.owner.shelf_manager.move_item(self.item, shelf_type, visibility=visibility) - del self.shelfmember + self.shelfmember = self.owner.shelf_manager.move_item(self.item, shelf_type, visibility=visibility) + if self.shelfmember and (created_time or metadata is not None): + if created_time: + self.shelfmember.created_time = created_time + if metadata is not None: + self.shelfmember.metadata = metadata + self.shelfmember.save() if comment_text != self.text or visibility != self.visibility: self.comment = Comment.comment_item_by_user(self.item, self.owner, comment_text, visibility) + if self.comment and created_time: + self.comment.created_time = created_time + self.comment.save(update_fields=['created_time']) if rating_grade != self.rating or visibility != self.visibility: - Rating.set_item_rating_by_user(self.item, rating_grade, self.owner, visibility) + rating_content = Rating.rate_item_by_user(self.item, self.owner, rating_grade, visibility) self.rating = rating_grade + if rating_content and created_time: + rating_content.created_time = created_time + rating_content.save(update_fields=['created_time']) diff --git a/journal/tests.py b/journal/tests.py index 45995f30..b15f08ee 100644 --- a/journal/tests.py +++ b/journal/tests.py @@ -35,38 +35,38 @@ class ShelfTest(TestCase): self.assertEqual(user.shelf_set.all().count(), 33) book1 = Edition.objects.create(title="Hyperion") book2 = Edition.objects.create(title="Andymion") - q1 = shelf_manager.get_shelf(ItemCategory.Book, ShelfType.WISHED) - q2 = shelf_manager.get_shelf(ItemCategory.Book, ShelfType.STARTED) + q1 = shelf_manager.get_shelf(ItemCategory.Book, ShelfType.WISHLIST) + q2 = shelf_manager.get_shelf(ItemCategory.Book, ShelfType.PROGRESS) self.assertIsNotNone(q1) self.assertIsNotNone(q2) self.assertEqual(q1.members.all().count(), 0) self.assertEqual(q2.members.all().count(), 0) - shelf_manager.move_item(book1, ShelfType.WISHED) - shelf_manager.move_item(book2, ShelfType.WISHED) + shelf_manager.move_item(book1, ShelfType.WISHLIST) + shelf_manager.move_item(book2, ShelfType.WISHLIST) self.assertEqual(q1.members.all().count(), 2) - shelf_manager.move_item(book1, ShelfType.STARTED) + shelf_manager.move_item(book1, ShelfType.PROGRESS) self.assertEqual(q1.members.all().count(), 1) self.assertEqual(q2.members.all().count(), 1) - shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 1}) + shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={'progress': 1}) self.assertEqual(q1.members.all().count(), 1) self.assertEqual(q2.members.all().count(), 1) log = shelf_manager.get_log_for_item(book1) self.assertEqual(log.count(), 3) - shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 1}) + shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={'progress': 1}) log = shelf_manager.get_log_for_item(book1) self.assertEqual(log.count(), 3) - shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 10}) + shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={'progress': 10}) log = shelf_manager.get_log_for_item(book1) self.assertEqual(log.count(), 4) - shelf_manager.move_item(book1, ShelfType.STARTED) + shelf_manager.move_item(book1, ShelfType.PROGRESS) log = shelf_manager.get_log_for_item(book1) self.assertEqual(log.count(), 4) self.assertEqual(log.last().metadata, {'progress': 10}) - shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 90}) + shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={'progress': 90}) log = shelf_manager.get_log_for_item(book1) self.assertEqual(log.count(), 5) self.assertEqual(Mark(user, book1).visibility, 0) - shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 90}, visibility=1) + shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={'progress': 90}, visibility=1) self.assertEqual(Mark(user, book1).visibility, 1) self.assertEqual(shelf_manager.get_log_for_item(book1).count(), 5) @@ -81,6 +81,15 @@ class TagTest(TestCase): self.user3 = User.objects.create(mastodon_site="site2", username="name3") pass + def test_user_tag(self): + t1 = 'sci-fi' + t2 = 'private' + t3 = 'public' + TagManager.tag_item_by_user(self.book1, self.user2, [t1, t3]) + self.assertEqual(self.book1.tags.sort(), [t1, t3].sort()) + TagManager.tag_item_by_user(self.book1, self.user2, [t2, t3]) + self.assertEqual(self.book1.tags.sort(), [t3, t2].sort()) + def test_tag(self): t1 = 'sci-fi' t2 = 'private' @@ -88,6 +97,7 @@ class TagTest(TestCase): TagManager.add_tag_by_user(self.book1, t3, self.user2) TagManager.add_tag_by_user(self.book1, t1, self.user1) TagManager.add_tag_by_user(self.book1, t1, self.user2) + self.assertEqual(self.book1.tags, [t1, t3]) TagManager.add_tag_by_user(self.book1, t2, self.user1, default_visibility=2) self.assertEqual(self.book1.tags, [t1, t3]) TagManager.add_tag_by_user(self.book1, t3, self.user1) @@ -119,10 +129,10 @@ class MarkTest(TestCase): self.assertEqual(mark.visibility, None) self.assertEqual(mark.review, None) self.assertEqual(mark.tags, []) - mark.update(ShelfType.WISHED, 'a gentle comment', 9, 1) + mark.update(ShelfType.WISHLIST, 'a gentle comment', 9, 1) mark = Mark(self.user1, self.book1) - self.assertEqual(mark.shelf_type, ShelfType.WISHED) + self.assertEqual(mark.shelf_type, ShelfType.WISHLIST) self.assertEqual(mark.shelf_label, '想读') self.assertEqual(mark.text, 'a gentle comment') self.assertEqual(mark.rating, 9) @@ -134,6 +144,6 @@ class MarkTest(TestCase): mark = Mark(self.user1, self.book1) self.assertEqual(mark.review, review) - self.user1.tag_manager.add_item_tags(self.book1, [' Sci-Fi ', ' fic ']) + TagManager.tag_item_by_user(self.book1, self.user1, [' Sci-Fi ', ' fic ']) mark = Mark(self.user1, self.book1) self.assertEqual(mark.tags, ['sci-fi', 'fic']) diff --git a/legacy/management/commands/migrate_catalog.py b/legacy/management/commands/migrate_catalog.py index df4d78c5..653bd8d1 100644 --- a/legacy/management/commands/migrate_catalog.py +++ b/legacy/management/commands/migrate_catalog.py @@ -147,7 +147,7 @@ model_link = { class Command(BaseCommand): - help = 'Migrate legacy books' + help = 'Migrate legacy catalog' def add_arguments(self, parser): parser.add_argument('--book', dest='types', action='append_const', const=Legacy_Book) @@ -168,7 +168,7 @@ class Command(BaseCommand): LinkModel = model_link[typ] if options['clearlink']: LinkModel.objects.all().delete() - qs = typ.objects.all().order_by('id') # if h == 0 else c.objects.filter(edited_time__gt=timezone.now() - timedelta(hours=h)) + qs = typ.objects.all().order_by('id') if options['id']: if options['maxid']: qs = qs.filter(id__gte=int(options['id']), id__lte=int(options['maxid'])) @@ -208,7 +208,7 @@ class Command(BaseCommand): links.append(LinkModel(old_id=entity.id, new_uid=item.uid)) # pprint.pp(site.get_item()) except Exception as e: - print(f'Convert failed for {entity}: {e}') + print(f'Convert failed for {typ} {entity.id}: {e}') if options['failstop']: raise(e) # return diff --git a/legacy/management/commands/migrate_journal.py b/legacy/management/commands/migrate_journal.py new file mode 100644 index 00000000..379863ec --- /dev/null +++ b/legacy/management/commands/migrate_journal.py @@ -0,0 +1,109 @@ +from books.models import Book as Legacy_Book +from movies.models import Movie as Legacy_Movie +from music.models import Album as Legacy_Album +from games.models import Game as Legacy_Game +from common.models import MarkStatusEnum +from books.models import BookMark +from movies.models import MovieMark +from music.models import AlbumMark +from games.models import GameMark +from catalog.common import * +from catalog.models import * +from catalog.sites import * +from journal.models import * +# from social import models as social_models +from django.core.management.base import BaseCommand +from django.core.paginator import Paginator +import pprint +from tqdm import tqdm +from django.db.models import Q, Count, Sum +from django.utils import dateparse, timezone +import re +from legacy.models import * +from users.models import User +from django.db import DatabaseError, transaction + +BATCH_SIZE = 1000 + + +model_link = { + BookMark: BookLink, + MovieMark: MovieLink, + AlbumMark: AlbumLink, + GameMark: GameLink, +} + +shelf_map = { + MarkStatusEnum.WISH: ShelfType.WISHLIST, + MarkStatusEnum.DO: ShelfType.PROGRESS, + MarkStatusEnum.COLLECT: ShelfType.COMPLETE, +} + +tag_map = { + BookMark: 'bookmark_tags', + MovieMark: 'moviemark_tags', + AlbumMark: 'albummark_tags', + GameMark: 'gamemark_tags', +} + + +class Command(BaseCommand): + help = 'Migrate legacy marks to user journal' + + def add_arguments(self, parser): + parser.add_argument('--book', dest='types', action='append_const', const=BookMark) + parser.add_argument('--movie', dest='types', action='append_const', const=MovieMark) + parser.add_argument('--album', dest='types', action='append_const', const=AlbumMark) + parser.add_argument('--game', dest='types', action='append_const', const=GameMark) + parser.add_argument('--id', help='id to convert; or, if using with --max-id, the min id') + parser.add_argument('--maxid', help='max id to convert') + parser.add_argument('--failstop', help='stop on fail', action='store_true') + parser.add_argument('--initshelf', help='initialize shelves for users, then exit', action='store_true') + parser.add_argument('--clear', help='clear all user pieces, then exit', action='store_true') + + def handle(self, *args, **options): + if options['initshelf']: + print("Initialize shelves") + with transaction.atomic(): + for user in tqdm(User.objects.filter(is_active=True)): + user.shelf_manager.initialize() + return + if options['clear']: + print("Deleting all migrated user pieces") + Piece.objects.all().delete() + return + types = options['types'] or [GameMark, AlbumMark, MovieMark, BookMark] + for typ in types: + print(typ) + LinkModel = model_link[typ] + tag_field = tag_map[typ] + qs = typ.objects.all().filter(owner__is_active=True).order_by('id') + if options['id']: + if options['maxid']: + qs = qs.filter(id__gte=int(options['id']), id__lte=int(options['maxid'])) + else: + qs = qs.filter(id=int(options['id'])) + + pg = Paginator(qs, BATCH_SIZE) + for p in tqdm(pg.page_range): + with transaction.atomic(): + for entity in pg.get_page(p).object_list: + try: + item_link = LinkModel.objects.get(old_id=entity.item.id) + item = Item.objects.get(uid=item_link.new_uid) + mark = Mark(entity.owner, item) + mark.update( + shelf_type=shelf_map[entity.status], + comment_text=entity.text, + rating_grade=entity.rating, + visibility=entity.visibility, + metadata={'shared_link': entity.shared_link}, + created_time=entity.created_time + ) + tags = [t.content for t in getattr(entity, tag_field).all()] + TagManager.tag_item_by_user(item, entity.owner, tags) + except Exception as e: + print(f'Convert failed for {typ} {entity.id}: {e}') + if options['failstop']: + raise(e) + self.stdout.write(self.style.SUCCESS(f'Done.')) diff --git a/social/models.py b/social/models.py index 4e35f774..9e0fac8f 100644 --- a/social/models.py +++ b/social/models.py @@ -14,6 +14,7 @@ import logging from functools import cached_property from django.db.models.signals import post_save, post_delete, pre_delete from django.db.models import Q +from django.conf import settings _logger = logging.getLogger(__name__) @@ -144,8 +145,9 @@ class DataSignalManager: @staticmethod def add_handler_for_model(model): - post_save.connect(DataSignalManager.save_handler, sender=model) - pre_delete.connect(DataSignalManager.delete_handler, sender=model) + if not settings.DISABLE_SOCIAL: + post_save.connect(DataSignalManager.save_handler, sender=model) + pre_delete.connect(DataSignalManager.delete_handler, sender=model) @staticmethod def register(processor): diff --git a/social/tests.py b/social/tests.py index 025d9502..ba46913b 100644 --- a/social/tests.py +++ b/social/tests.py @@ -21,17 +21,17 @@ class SocialTest(TestCase): self.assertEqual(len(timeline), 0) # 1 activity after adding first book to shelf - self.alice.shelf_manager.move_item(self.book1, ShelfType.WISHED, visibility=1) + self.alice.shelf_manager.move_item(self.book1, ShelfType.WISHLIST, visibility=1) timeline = self.alice.activity_manager.get_viewable_activities() self.assertEqual(len(timeline), 1) # 2 activities after adding second book to shelf - self.alice.shelf_manager.move_item(self.book2, ShelfType.WISHED) + self.alice.shelf_manager.move_item(self.book2, ShelfType.WISHLIST) timeline = self.alice.activity_manager.get_viewable_activities() self.assertEqual(len(timeline), 2) # 2 activities after change first mark - self.alice.shelf_manager.move_item(self.book1, ShelfType.STARTED) + self.alice.shelf_manager.move_item(self.book1, ShelfType.PROGRESS) timeline = self.alice.activity_manager.get_viewable_activities() self.assertEqual(len(timeline), 2) @@ -47,7 +47,7 @@ class SocialTest(TestCase): self.assertEqual(len(timeline2), 2) # alice:3 bob:2 after alice adding second book to shelf as private - self.alice.shelf_manager.move_item(self.movie, ShelfType.WISHED, visibility=2) + self.alice.shelf_manager.move_item(self.movie, ShelfType.WISHLIST, visibility=2) timeline = self.alice.activity_manager.get_viewable_activities() self.assertEqual(len(timeline), 3) timeline2 = self.bob.activity_manager.get_viewable_activities()