Itmens/lib.itmens
Itmens/lib.itmens
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions3

new data model: migrate journal

Browse source
This commit is contained in:
Your Name 2022-12-17 16:18:16 -05:00
parent 2a853024cd
commit 58509b650d
7 changed files with 210 additions and 55 deletions

2
doc/social.md
View file

@ -42,7 +42,7 @@ These are list of activities should be either shown in the site or delivered as
- `Add` / `Remove` an *Item* to / from a *List*:
+ add / remove *Item* to / from a user *Collection*
+ mark *Item* as wished / started / done, which are essentially add to / remove from user's predefined *Collection*
+ mark *Item* as wishlist / progress / complete, which are essentially add to / remove from user's predefined *Collection*
- `Create` / `Update` / `Delete` a user *Collection*
- `Create` / `Update` / `Delete` a *Content* with an `Object Link` to *Item*
+ `Create` / `Update` / `Delete` a *Comment* or *Review*

96
journal/models.py
View file

@ -16,9 +16,11 @@ from functools import cached_property
from django.db.models import Count, Avg
import django.dispatch
import math
import uuid
class Piece(PolymorphicModel, UserOwnedObjectMixin):
uid = models.UUIDField(default=uuid.uuid4, editable=False, db_index=True)
owner = models.ForeignKey(User, on_delete=models.PROTECT)
visibility = models.PositiveSmallIntegerField(default=0) # 0: Public / 1: Follower only / 2: Self only
created_time = models.DateTimeField(auto_now_add=True)
@ -47,13 +49,13 @@ class Comment(Content):
@staticmethod
def comment_item_by_user(item, user, text, visibility=0):
comment = Comment.objects.filter(owner=user, item=item).first()
if text is None:
if not text:
if comment is not None:
comment.delete()
comment = None
elif comment is None:
comment = Comment.objects.create(owner=user, item=item, text=text, visibility=visibility)
else:
elif comment.text != text or comment.visibility != visibility:
comment.text = text
comment.visibility = visibility
comment.save()
@ -99,16 +101,21 @@ class Rating(Content):
return stat['count']
@staticmethod
def set_item_rating_by_user(item, rating_grade, user, visibility=0):
if rating_grade is not None and (rating_grade < 1 or rating_grade > 10):
def rate_item_by_user(item, user, rating_grade, visibility=0):
if not rating_grade and (rating_grade < 1 or rating_grade > 10):
raise ValueError(f'Invalid rating grade: {rating_grade}')
rating = Rating.objects.filter(owner=user, item=item).first()
if not rating:
if not rating_grade:
if rating:
rating.delete()
rating = None
elif rating is None:
rating = Rating.objects.create(owner=user, item=item, grade=rating_grade, visibility=visibility)
else:
elif rating.grade != rating_grade or rating.visibility != visibility:
rating.visibility = visibility
rating.grade = rating_grade
rating.save()
return rating
@staticmethod
def get_item_rating_by_user(item, user):
@ -173,8 +180,8 @@ class List(Piece):
def remove_item(self, item):
member = self.members.all().filter(item=item).first()
list_remove.send(sender=self.__class__, instance=self, item=item, member=member)
if member:
list_remove.send(sender=self.__class__, instance=self, item=item, member=member)
member.delete()
def move_up_item(self, item):
@ -228,29 +235,29 @@ Shelf
class ShelfType(models.TextChoices):
WISHED = ('wished', '未开始')
STARTED = ('started', '进行中')
DONE = ('done', '完成')
WISHLIST = ('wishlist', '未开始')
PROGRESS = ('progress', '进行中')
COMPLETE = ('complete', '完成')
# DISCARDED = ('discarded', '放弃')
ShelfTypeNames = [
[ItemCategory.Book, ShelfType.WISHED, _('想读')],
[ItemCategory.Book, ShelfType.STARTED, _('在读')],
[ItemCategory.Book, ShelfType.DONE, _('读过')],
[ItemCategory.Movie, ShelfType.WISHED, _('想看')],
[ItemCategory.Movie, ShelfType.STARTED, _('在看')],
[ItemCategory.Movie, ShelfType.DONE, _('看过')],
[ItemCategory.TV, ShelfType.WISHED, _('想看')],
[ItemCategory.TV, ShelfType.STARTED, _('在看')],
[ItemCategory.TV, ShelfType.DONE, _('看过')],
[ItemCategory.Music, ShelfType.WISHED, _('想听')],
[ItemCategory.Music, ShelfType.STARTED, _('在听')],
[ItemCategory.Music, ShelfType.DONE, _('听过')],
[ItemCategory.Game, ShelfType.WISHED, _('想玩')],
[ItemCategory.Game, ShelfType.STARTED, _('在玩')],
[ItemCategory.Game, ShelfType.DONE, _('玩过')],
[ItemCategory.Collection, ShelfType.WISHED, _('关注')],
[ItemCategory.Book, ShelfType.WISHLIST, _('想读')],
[ItemCategory.Book, ShelfType.PROGRESS, _('在读')],
[ItemCategory.Book, ShelfType.COMPLETE, _('读过')],
[ItemCategory.Movie, ShelfType.WISHLIST, _('想看')],
[ItemCategory.Movie, ShelfType.PROGRESS, _('在看')],
[ItemCategory.Movie, ShelfType.COMPLETE, _('看过')],
[ItemCategory.TV, ShelfType.WISHLIST, _('想看')],
[ItemCategory.TV, ShelfType.PROGRESS, _('在看')],
[ItemCategory.TV, ShelfType.COMPLETE, _('看过')],
[ItemCategory.Music, ShelfType.WISHLIST, _('想听')],
[ItemCategory.Music, ShelfType.PROGRESS, _('在听')],
[ItemCategory.Music, ShelfType.COMPLETE, _('听过')],
[ItemCategory.Game, ShelfType.WISHLIST, _('想玩')],
[ItemCategory.Game, ShelfType.PROGRESS, _('在玩')],
[ItemCategory.Game, ShelfType.COMPLETE, _('玩过')],
[ItemCategory.Collection, ShelfType.WISHLIST, _('关注')],
# TODO add more combinations
]
@ -327,6 +334,7 @@ class ShelfManager:
# metadata=None means no change
if not item:
raise ValueError('empty item')
new_shelfmember = None
last_shelfmember = self._shelf_member_for_item(item)
last_shelf = last_shelfmember._shelf if last_shelfmember else None
last_metadata = last_shelfmember.metadata if last_shelfmember else None
@ -338,10 +346,11 @@ class ShelfManager:
if last_shelf:
last_shelf.remove_item(item)
if shelf:
shelf.append_item(item, visibility=visibility, metadata=metadata or {})
new_shelfmember = shelf.append_item(item, visibility=visibility, metadata=metadata or {})
elif last_shelf is None:
raise ValueError('empty shelf')
else:
new_shelfmember = last_shelfmember
if metadata is not None and metadata != last_metadata: # change metadata
changed = True
last_shelfmember.metadata = metadata
@ -354,6 +363,7 @@ class ShelfManager:
if metadata is None:
metadata = last_metadata or {}
ShelfLogEntry.objects.create(owner=self.owner, shelf=shelf, item=item, metadata=metadata)
return new_shelfmember
def get_log(self):
return ShelfLogEntry.objects.filter(owner=self.owner).order_by('timestamp')
@ -443,6 +453,19 @@ class TagManager:
tags = user.tag_set.all().values('title').annotate(frequency=Count('members')).order_by('-frequency')
return list(map(lambda t: t['title'], tags))
@staticmethod
def tag_item_by_user(item, user, tag_titles, default_visibility=0):
titles = set([Tag.cleanup_title(tag_title) for tag_title in tag_titles])
current_titles = set([m._tag.title for m in TagMember.objects.filter(owner=user, item=item)])
for title in titles - current_titles:
tag = Tag.objects.filter(owner=user, title=title).first()
if not tag:
tag = Tag.objects.create(owner=user, title=title, visibility=default_visibility)
tag.append_item(item)
for title in current_titles - titles:
tag = Tag.objects.filter(owner=user, title=title).first()
tag.remove_item(item)
@staticmethod
def add_tag_by_user(item, tag_title, user, default_visibility=0):
title = Tag.cleanup_title(tag_title)
@ -526,12 +549,23 @@ class Mark:
def review(self):
return Review.objects.filter(owner=self.owner, item=self.item).first()
def update(self, shelf_type, comment_text, rating_grade, visibility):
def update(self, shelf_type, comment_text, rating_grade, visibility, metadata=None, created_time=None):
if shelf_type != self.shelf_type or visibility != self.visibility:
self.owner.shelf_manager.move_item(self.item, shelf_type, visibility=visibility)
del self.shelfmember
self.shelfmember = self.owner.shelf_manager.move_item(self.item, shelf_type, visibility=visibility)
if self.shelfmember and (created_time or metadata is not None):
if created_time:
self.shelfmember.created_time = created_time
if metadata is not None:
self.shelfmember.metadata = metadata
self.shelfmember.save()
if comment_text != self.text or visibility != self.visibility:
self.comment = Comment.comment_item_by_user(self.item, self.owner, comment_text, visibility)
if self.comment and created_time:
self.comment.created_time = created_time
self.comment.save(update_fields=['created_time'])
if rating_grade != self.rating or visibility != self.visibility:
Rating.set_item_rating_by_user(self.item, rating_grade, self.owner, visibility)
rating_content = Rating.rate_item_by_user(self.item, self.owner, rating_grade, visibility)
self.rating = rating_grade
if rating_content and created_time:
rating_content.created_time = created_time
rating_content.save(update_fields=['created_time'])

38
journal/tests.py
View file

@ -35,38 +35,38 @@ class ShelfTest(TestCase):
self.assertEqual(user.shelf_set.all().count(), 33)
book1 = Edition.objects.create(title="Hyperion")
book2 = Edition.objects.create(title="Andymion")
q1 = shelf_manager.get_shelf(ItemCategory.Book, ShelfType.WISHED)
q2 = shelf_manager.get_shelf(ItemCategory.Book, ShelfType.STARTED)
q1 = shelf_manager.get_shelf(ItemCategory.Book, ShelfType.WISHLIST)
q2 = shelf_manager.get_shelf(ItemCategory.Book, ShelfType.PROGRESS)
self.assertIsNotNone(q1)
self.assertIsNotNone(q2)
self.assertEqual(q1.members.all().count(), 0)
self.assertEqual(q2.members.all().count(), 0)
shelf_manager.move_item(book1, ShelfType.WISHED)
shelf_manager.move_item(book2, ShelfType.WISHED)
shelf_manager.move_item(book1, ShelfType.WISHLIST)
shelf_manager.move_item(book2, ShelfType.WISHLIST)
self.assertEqual(q1.members.all().count(), 2)
shelf_manager.move_item(book1, ShelfType.STARTED)
shelf_manager.move_item(book1, ShelfType.PROGRESS)
self.assertEqual(q1.members.all().count(), 1)
self.assertEqual(q2.members.all().count(), 1)
shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 1})
shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={'progress': 1})
self.assertEqual(q1.members.all().count(), 1)
self.assertEqual(q2.members.all().count(), 1)
log = shelf_manager.get_log_for_item(book1)
self.assertEqual(log.count(), 3)
shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 1})
shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={'progress': 1})
log = shelf_manager.get_log_for_item(book1)
self.assertEqual(log.count(), 3)
shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 10})
shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={'progress': 10})
log = shelf_manager.get_log_for_item(book1)
self.assertEqual(log.count(), 4)
shelf_manager.move_item(book1, ShelfType.STARTED)
shelf_manager.move_item(book1, ShelfType.PROGRESS)
log = shelf_manager.get_log_for_item(book1)
self.assertEqual(log.count(), 4)
self.assertEqual(log.last().metadata, {'progress': 10})
shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 90})
shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={'progress': 90})
log = shelf_manager.get_log_for_item(book1)
self.assertEqual(log.count(), 5)
self.assertEqual(Mark(user, book1).visibility, 0)
shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 90}, visibility=1)
shelf_manager.move_item(book1, ShelfType.PROGRESS, metadata={'progress': 90}, visibility=1)
self.assertEqual(Mark(user, book1).visibility, 1)
self.assertEqual(shelf_manager.get_log_for_item(book1).count(), 5)
@ -81,6 +81,15 @@ class TagTest(TestCase):
self.user3 = User.objects.create(mastodon_site="site2", username="name3")
pass
def test_user_tag(self):
t1 = 'sci-fi'
t2 = 'private'
t3 = 'public'
TagManager.tag_item_by_user(self.book1, self.user2, [t1, t3])
self.assertEqual(self.book1.tags.sort(), [t1, t3].sort())
TagManager.tag_item_by_user(self.book1, self.user2, [t2, t3])
self.assertEqual(self.book1.tags.sort(), [t3, t2].sort())
def test_tag(self):
t1 = 'sci-fi'
t2 = 'private'
@ -88,6 +97,7 @@ class TagTest(TestCase):
TagManager.add_tag_by_user(self.book1, t3, self.user2)
TagManager.add_tag_by_user(self.book1, t1, self.user1)
TagManager.add_tag_by_user(self.book1, t1, self.user2)
self.assertEqual(self.book1.tags, [t1, t3])
TagManager.add_tag_by_user(self.book1, t2, self.user1, default_visibility=2)
self.assertEqual(self.book1.tags, [t1, t3])
TagManager.add_tag_by_user(self.book1, t3, self.user1)
@ -119,10 +129,10 @@ class MarkTest(TestCase):
self.assertEqual(mark.visibility, None)
self.assertEqual(mark.review, None)
self.assertEqual(mark.tags, [])
mark.update(ShelfType.WISHED, 'a gentle comment', 9, 1)
mark.update(ShelfType.WISHLIST, 'a gentle comment', 9, 1)
mark = Mark(self.user1, self.book1)
self.assertEqual(mark.shelf_type, ShelfType.WISHED)
self.assertEqual(mark.shelf_type, ShelfType.WISHLIST)
self.assertEqual(mark.shelf_label, '想读')
self.assertEqual(mark.text, 'a gentle comment')
self.assertEqual(mark.rating, 9)
@ -134,6 +144,6 @@ class MarkTest(TestCase):
mark = Mark(self.user1, self.book1)
self.assertEqual(mark.review, review)
self.user1.tag_manager.add_item_tags(self.book1, [' Sci-Fi ', ' fic '])
TagManager.tag_item_by_user(self.book1, self.user1, [' Sci-Fi ', ' fic '])
mark = Mark(self.user1, self.book1)
self.assertEqual(mark.tags, ['sci-fi', 'fic'])

6
legacy/management/commands/migrate_catalog.py
View file

@ -147,7 +147,7 @@ model_link = {
class Command(BaseCommand):
help = 'Migrate legacy books'
help = 'Migrate legacy catalog'
def add_arguments(self, parser):
parser.add_argument('--book', dest='types', action='append_const', const=Legacy_Book)
@ -168,7 +168,7 @@ class Command(BaseCommand):
LinkModel = model_link[typ]
if options['clearlink']:
LinkModel.objects.all().delete()
qs = typ.objects.all().order_by('id') # if h == 0 else c.objects.filter(edited_time__gt=timezone.now() - timedelta(hours=h))
qs = typ.objects.all().order_by('id')
if options['id']:
if options['maxid']:
qs = qs.filter(id__gte=int(options['id']), id__lte=int(options['maxid']))
@ -208,7 +208,7 @@ class Command(BaseCommand):
links.append(LinkModel(old_id=entity.id, new_uid=item.uid))
# pprint.pp(site.get_item())
except Exception as e:
print(f'Convert failed for {entity}: {e}')
print(f'Convert failed for {typ} {entity.id}: {e}')
if options['failstop']:
raise(e)
# return

109
legacy/management/commands/migrate_journal.py Normal file
View file

@ -0,0 +1,109 @@
from books.models import Book as Legacy_Book
from movies.models import Movie as Legacy_Movie
from music.models import Album as Legacy_Album
from games.models import Game as Legacy_Game
from common.models import MarkStatusEnum
from books.models import BookMark
from movies.models import MovieMark
from music.models import AlbumMark
from games.models import GameMark
from catalog.common import *
from catalog.models import *
from catalog.sites import *
from journal.models import *
# from social import models as social_models
from django.core.management.base import BaseCommand
from django.core.paginator import Paginator
import pprint
from tqdm import tqdm
from django.db.models import Q, Count, Sum
from django.utils import dateparse, timezone
import re
from legacy.models import *
from users.models import User
from django.db import DatabaseError, transaction
BATCH_SIZE = 1000
model_link = {
BookMark: BookLink,
MovieMark: MovieLink,
AlbumMark: AlbumLink,
GameMark: GameLink,
}
shelf_map = {
MarkStatusEnum.WISH: ShelfType.WISHLIST,
MarkStatusEnum.DO: ShelfType.PROGRESS,
MarkStatusEnum.COLLECT: ShelfType.COMPLETE,
}
tag_map = {
BookMark: 'bookmark_tags',
MovieMark: 'moviemark_tags',
AlbumMark: 'albummark_tags',
GameMark: 'gamemark_tags',
}
class Command(BaseCommand):
help = 'Migrate legacy marks to user journal'
def add_arguments(self, parser):
parser.add_argument('--book', dest='types', action='append_const', const=BookMark)
parser.add_argument('--movie', dest='types', action='append_const', const=MovieMark)
parser.add_argument('--album', dest='types', action='append_const', const=AlbumMark)
parser.add_argument('--game', dest='types', action='append_const', const=GameMark)
parser.add_argument('--id', help='id to convert; or, if using with --max-id, the min id')
parser.add_argument('--maxid', help='max id to convert')
parser.add_argument('--failstop', help='stop on fail', action='store_true')
parser.add_argument('--initshelf', help='initialize shelves for users, then exit', action='store_true')
parser.add_argument('--clear', help='clear all user pieces, then exit', action='store_true')
def handle(self, *args, **options):
if options['initshelf']:
print("Initialize shelves")
with transaction.atomic():
for user in tqdm(User.objects.filter(is_active=True)):
user.shelf_manager.initialize()
return
if options['clear']:
print("Deleting all migrated user pieces")
Piece.objects.all().delete()
return
types = options['types'] or [GameMark, AlbumMark, MovieMark, BookMark]
for typ in types:
print(typ)
LinkModel = model_link[typ]
tag_field = tag_map[typ]
qs = typ.objects.all().filter(owner__is_active=True).order_by('id')
if options['id']:
if options['maxid']:
qs = qs.filter(id__gte=int(options['id']), id__lte=int(options['maxid']))
else:
qs = qs.filter(id=int(options['id']))
pg = Paginator(qs, BATCH_SIZE)
for p in tqdm(pg.page_range):
with transaction.atomic():
for entity in pg.get_page(p).object_list:
try:
item_link = LinkModel.objects.get(old_id=entity.item.id)
item = Item.objects.get(uid=item_link.new_uid)
mark = Mark(entity.owner, item)
mark.update(
shelf_type=shelf_map[entity.status],
comment_text=entity.text,
rating_grade=entity.rating,
visibility=entity.visibility,
metadata={'shared_link': entity.shared_link},
created_time=entity.created_time
)
tags = [t.content for t in getattr(entity, tag_field).all()]
TagManager.tag_item_by_user(item, entity.owner, tags)
except Exception as e:
print(f'Convert failed for {typ} {entity.id}: {e}')
if options['failstop']:
raise(e)
self.stdout.write(self.style.SUCCESS(f'Done.'))

6
social/models.py
View file

@ -14,6 +14,7 @@ import logging
from functools import cached_property
from django.db.models.signals import post_save, post_delete, pre_delete
from django.db.models import Q
from django.conf import settings
_logger = logging.getLogger(__name__)
@ -144,8 +145,9 @@ class DataSignalManager:
@staticmethod
def add_handler_for_model(model):
post_save.connect(DataSignalManager.save_handler, sender=model)
pre_delete.connect(DataSignalManager.delete_handler, sender=model)
if not settings.DISABLE_SOCIAL:
post_save.connect(DataSignalManager.save_handler, sender=model)
pre_delete.connect(DataSignalManager.delete_handler, sender=model)
@staticmethod
def register(processor):

8
social/tests.py
View file

@ -21,17 +21,17 @@ class SocialTest(TestCase):
self.assertEqual(len(timeline), 0)
# 1 activity after adding first book to shelf
self.alice.shelf_manager.move_item(self.book1, ShelfType.WISHED, visibility=1)
self.alice.shelf_manager.move_item(self.book1, ShelfType.WISHLIST, visibility=1)
timeline = self.alice.activity_manager.get_viewable_activities()
self.assertEqual(len(timeline), 1)
# 2 activities after adding second book to shelf
self.alice.shelf_manager.move_item(self.book2, ShelfType.WISHED)
self.alice.shelf_manager.move_item(self.book2, ShelfType.WISHLIST)
timeline = self.alice.activity_manager.get_viewable_activities()
self.assertEqual(len(timeline), 2)
# 2 activities after change first mark
self.alice.shelf_manager.move_item(self.book1, ShelfType.STARTED)
self.alice.shelf_manager.move_item(self.book1, ShelfType.PROGRESS)
timeline = self.alice.activity_manager.get_viewable_activities()
self.assertEqual(len(timeline), 2)
@ -47,7 +47,7 @@ class SocialTest(TestCase):
self.assertEqual(len(timeline2), 2)
# alice:3 bob:2 after alice adding second book to shelf as private
self.alice.shelf_manager.move_item(self.movie, ShelfType.WISHED, visibility=2)
self.alice.shelf_manager.move_item(self.movie, ShelfType.WISHLIST, visibility=2)
timeline = self.alice.activity_manager.get_viewable_activities()
self.assertEqual(len(timeline), 3)
timeline2 = self.bob.activity_manager.get_viewable_activities()