more doc and test

2022-12-13 18:12:43 +00:00 · 2022-12-13 18:12:43 +00:00 · ab86a6f73e
commit ab86a6f73e
parent ad696d8377
11 changed files with 410 additions and 164 deletions
--- a/boofilsic/settings.py
+++ b/boofilsic/settings.py
@ -118,7 +118,7 @@ if DEBUG:
                'client_encoding': 'UTF8',
                # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT,
            }
-        }    
+        }
    }
 else:
    DATABASES = {
@ -132,7 +132,7 @@ else:
                'client_encoding': 'UTF8',
                # 'isolation_level': psycopg2.extensions.ISOLATION_LEVEL_DEFAULT,
            }
-        }    
+        }
    }

 # Customized auth backend, glue OAuth2 and Django User model together
@ -173,7 +173,7 @@ if not DEBUG:
                'format': '{levelname} {asctime} {name}:{lineno} {message}',
                'style': '{',
            },
-        },    
+        },
        'handlers': {
            'file': {
                'level': 'INFO',
@ -248,7 +248,7 @@ MASTODON_ALLOW_ANY_SITE = False
 MASTODON_TIMEOUT = 30

 MASTODON_CLIENT_SCOPE = 'read write follow'
-#use the following if it's a new site
+# use the following if it's a new site
 #MASTODON_CLIENT_SCOPE = 'read:accounts read:follows read:search read:blocks read:mutes write:statuses write:media'

 MASTODON_LEGACY_CLIENT_SCOPE = 'read write follow'
@ -366,3 +366,5 @@ ENABLE_NEW_MODEL = os.getenv('new_data_model')
 if ENABLE_NEW_MODEL:
    INSTALLED_APPS.append('polymorphic')
    INSTALLED_APPS.append('catalog.apps.CatalogConfig')
+    INSTALLED_APPS.append('journal.apps.JournalConfig')
+    INSTALLED_APPS.append('social.apps.SocialConfig')
--- a/catalog/common/mixins.py
+++ b/catalog/common/mixins.py
@ -0,0 +1,22 @@
+class SoftDeleteMixin:
+    """
+    SoftDeleteMixin
+
+    Model must add this:
+    is_deleted = models.BooleanField(default=False, db_index=True)
+
+    Model may override this:
+    def clear(self):
+        pass
+    """
+
+    def clear(self):
+        pass
+
+    def delete(self, using=None, soft=True, *args, **kwargs):
+        if soft:
+            self.clear()
+            self.is_deleted = True
+            self.save(using=using)
+        else:
+            return super().delete(using=using, *args, **kwargs)
--- a/catalog/common/models.py
+++ b/catalog/common/models.py
@ -9,6 +9,7 @@ from django.utils.baseconv import base62
 from simple_history.models import HistoricalRecords
 import uuid
 from .utils import DEFAULT_ITEM_COVER, item_cover_path
+from .mixins import SoftDeleteMixin
 # from django.conf import settings


@ -155,31 +156,6 @@ class LookupIdDescriptor(object):  # TODO make it mixin of Field
 #     return sid[0] in IdType.values()


-class SoftDeleteMixin:
-    """
-    SoftDeleteMixin
-
-    Model must add this:
-    is_deleted = models.BooleanField(default=False, db_index=True)
-
-    Model may override this:
-    def clear(self):
-        pass
-    """
-
-    def clear(self):
-        pass
-
-    def delete(self, using=None, soft=True, *args, **kwargs):
-        print('SOFT')
-        if soft:
-            self.clear()
-            self.is_deleted = True
-            self.save(using=using)
-        else:
-            return super().delete(using=using, *args, **kwargs)
-
-
 class Item(PolymorphicModel, SoftDeleteMixin):
    url_path = None  # subclass must specify this
    category = None  # subclass must specify this
--- a/doc/catalog.md
+++ b/doc/catalog.md
@ -0,0 +1,103 @@
+Catalog
+=======
+
+Data Models
+-----------
+all types of catalog items inherits from `Item` which stores as multi-table django model. 
+one `Item` may have multiple `ExternalResource`s, each represents one page on an external site
+
+```mermaid
+classDiagram
+    class Item {
+        <<abstract>>
+    }
+    Item <|-- Album
+    class Album {
+        +String barcode
+        +String Douban_ID
+        +String Spotify_ID
+    }
+    Item <|-- Game
+    class Game {
+        +String Steam_ID
+    }
+    Item <|-- Podcast
+    class Podcast {
+        +String feed_url
+        +String Apple_ID
+    }
+    Item <|-- Performance
+    Item <|-- Work
+    class Work {
+        +String Douban_Work_ID
+        +String Goodreads_Work_ID
+    }
+    Item <|-- Edition
+    Item <|-- Series
+    
+    Series *-- Work
+    Work *-- Edition
+    
+    class Series {
+        +String Goodreads_Series_ID
+    }
+    class Work {
+        +String Douban_ID
+        +String Goodreads_ID
+    }
+    class Edition{
+        +String ISBN
+        +String Douban_ID
+        +String Goodreads_ID
+        +String GoogleBooks_ID
+    }
+
+    Item <|-- Movie
+    Item <|-- TVShow
+    Item <|-- TVSeason
+    Item <|-- TVEpisode
+    TVShow *-- TVSeason
+    TVSeason *-- TVEpisode
+    
+    class TVShow{
+        +String IMDB_ID
+        +String TMDB_ID
+    }
+    class TVSeason{
+        +String Douban_ID
+        +String TMDB_ID
+    }
+    class TVEpisode{
+        +String IMDB_ID
+        +String TMDB_ID
+    }
+    class Movie{
+        +String Douban_ID
+        +String IMDB_ID
+        +String TMDB_ID
+    }
+
+    Item <|-- Collection
+
+    ExternalResource --* Item
+    class ExternalResource {
+        +enum site
+        +url: string
+    }
+```
+
+Add a new site
+--------------
+ - add a new item to `IdType` enum in `catalog/common/models.py`
+ - add a new file in `catalog/sites/`, a new class inherits `AbstractSite`, with:
+    * `ID_TYPE`
+    * `URL_PATTERNS`
+    * `WIKI_PROPERTY_ID` (not used now)
+    * `DEFAULT_MODEL` (unless specified in `scrape()` return val)
+    * a `classmethod` `id_to_url()`
+    * a method `scrape()` returns a `ResourceContent` object
+    * ...
+
+    see existing files in `catalog/sites/` for more examples
+ - add an import in `catalog/sites/__init__.py`
+
--- a/doc/development.md
+++ b/doc/development.md
@ -0,0 +1,64 @@
+Development
+===========
+
+*this doc is based on new data models work which is a work in progress*
+
+First, a working version of local NeoDB instance has to be established, see [install guide](GUIDE.md).
+
+Since new data model is still under development, most pieces are off by default, add `new_data_model=1` to your shell env and run migrations before start working on these new models
+
+```
+export new_data_model=1
+python3 manage.py makemigrations 
+python3 manage.py migrate
+```
+
+It's recommended to create the test database from freshly created database:
+```
+CREATE DATABASE test_neodb WITH TEMPLATE neodb;
+```
+Alternatively `python3 manage.py test` can create test databases every time test runs, but it's slow and buggy bc test run initial migration scripts slightly differently.
+
+Run Test
+--------
+Now to verify everything works, run tests with `python3 manage.py test --keepdb`
+```
+$ python3 manage.py test --keepdb
+
+Using existing test database for alias 'default'...
+System check identified no issues (2 silenced).
+........................................................
+----------------------------------------------------------------------
+Ran 56 tests in 1.100s
+
+OK
+Preserving test database for alias 'default'...
+```
+
+
+Data Models
+-----------
+main django apps for NeoDB:
+ - `users` manages user in typical django fashion
+ - `mastodon` this leverages [Mastodon API](https://docs.joinmastodon.org/client/intro/) and [Twitter API](https://developer.twitter.com/en/docs/twitter-api) for user login and data sync
+ - `catalog` manages different types of items user may review, and scrapers to fetch from external resources, see [catalog.md](catalog.md) for more details
+ - `journal` manages user created content(review/ratings) and lists(collection/shelf/tag), see [journal.md](journal.md) for more details
+ - `social` manages timeline for local users and ActivityStreams for remote servers, see [social.md](social.md) for more details
+
+These apps are legacy: books, music, movies, games, collections, they will be removed soon.
+
+
+ActivityPub
+-----------
+
+TBA
+
+References:
+ - https://www.w3.org/TR/activitypub/
+ - https://www.w3.org/TR/activitystreams-core/
+ - https://www.w3.org/TR/activitystreams-vocabulary/
+ - https://www.w3.org/TR/json-ld/
+ - https://codeberg.org/fediverse/fep/src/branch/main/feps/fep-e232.md
+ - https://socialhub.activitypub.rocks/t/guide-for-new-activitypub-implementers/479
+ - https://blog.joinmastodon.org/2018/06/how-to-implement-a-basic-activitypub-server/
+ - https://docs.joinmastodon.org/spec/activitypub/
--- a/journal/mixins.py
+++ b/journal/mixins.py
@ -0,0 +1,36 @@
+class UserOwnedObjectMixin:
+    """
+    UserOwnedObjectMixin
+
+    Models must add these:
+    owner = models.ForeignKey(User, on_delete=models.PROTECT)
+    visibility = models.PositiveSmallIntegerField(default=0)
+    """
+
+    def is_visible_to(self, viewer):
+        if not viewer.is_authenticated:
+            return self.visibility == 0
+        owner = self.owner
+        if owner == viewer:
+            return True
+        if not owner.is_active:
+            return False
+        if self.visibility == 2:
+            return False
+        if viewer.is_blocking(owner) or owner.is_blocking(viewer) or viewer.is_muting(owner):
+            return False
+        if self.visibility == 1:
+            return viewer.is_following(owner)
+        else:
+            return True
+
+    def is_editable_by(self, viewer):
+        return True if viewer.is_staff or viewer.is_superuser or viewer == self.owner else False
+
+    @classmethod
+    def get_available(cls, entity, request_user, following_only=False):
+        # e.g. SongMark.get_available(song, request.user)
+        query_kwargs = {entity.__class__.__name__.lower(): entity}
+        all_entities = cls.objects.filter(**query_kwargs).order_by("-created_time")  # get all marks for song
+        visible_entities = list(filter(lambda _entity: _entity.is_visible_to(request_user) and (_entity.owner.mastodon_username in request_user.mastodon_following if following_only else True), all_entities))
+        return visible_entities
--- a/journal/models.py
+++ b/journal/models.py
@ -1,7 +1,9 @@
 from django.db import models
 from polymorphic.models import PolymorphicModel
 from users.models import User
-from catalog.common.models import Item, ItemCategory, SoftDeleteMixin
+from catalog.common.models import Item, ItemCategory
+from catalog.common.mixins import SoftDeleteMixin
+from .mixins import UserOwnedObjectMixin
 from catalog.collection.models import Collection as CatalogCollection
 from decimal import *
 from enum import Enum
@ -13,44 +15,7 @@ from django.utils.translation import gettext_lazy as _
 from django.core.validators import RegexValidator
 from functools import cached_property
 from django.db.models import Count
-
-
-class UserOwnedObjectMixin:
-    """
-    UserOwnedObjectMixin
-
-    Models must add these:
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
-    visibility = models.PositiveSmallIntegerField(default=0)
-    """
-
-    def is_visible_to(self, viewer):
-        if not viewer.is_authenticated:
-            return self.visibility == 0
-        owner = self.owner
-        if owner == viewer:
-            return True
-        if not owner.is_active:
-            return False
-        if self.visibility == 2:
-            return False
-        if viewer.is_blocking(owner) or owner.is_blocking(viewer) or viewer.is_muting(owner):
-            return False
-        if self.visibility == 1:
-            return viewer.is_following(owner)
-        else:
-            return True
-
-    def is_editable_by(self, viewer):
-        return True if viewer.is_staff or viewer.is_superuser or viewer == self.owner else False
-
-    @classmethod
-    def get_available(cls, entity, request_user, following_only=False):
-        # e.g. SongMark.get_available(song, request.user)
-        query_kwargs = {entity.__class__.__name__.lower(): entity}
-        all_entities = cls.objects.filter(**query_kwargs).order_by("-created_time")  # get all marks for song
-        visible_entities = list(filter(lambda _entity: _entity.is_visible_to(request_user) and (_entity.owner.mastodon_username in request_user.mastodon_following if following_only else True), all_entities))
-        return visible_entities
+import django.dispatch


 class Piece(PolymorphicModel, UserOwnedObjectMixin):
@ -96,6 +61,9 @@ class Reply(Content):
 List (abstract class)
 """

+list_add = django.dispatch.Signal()
+list_remove = django.dispatch.Signal()
+

 class List(Piece):
    class Meta:
@ -129,11 +97,13 @@ class List(Piece):
            ml = self.ordered_members
            p = {'_' + self.__class__.__name__.lower(): self}
            p.update(params)
-            i = self.MEMBER_CLASS.objects.create(owner=self.owner, position=ml.last().position + 1 if ml.count() else 1, item=item, **p)
-            return i
+            member = self.MEMBER_CLASS.objects.create(owner=self.owner, position=ml.last().position + 1 if ml.count() else 1, item=item, **p)
+            list_add.send(sender=self.__class__, instance=self, item=item, member=member)
+            return member

    def remove_item(self, item):
        member = self.members.all().filter(item=item).first()
+        list_remove.send(sender=self.__class__, instance=self, item=item, member=member)
        if member:
            member.delete()

@ -178,85 +148,88 @@ class ListMember(Piece):
    class Meta:
        abstract = True

+    def __str__(self):
+        return f'{self.id}:{self.position} ({self.item})'
+

 """
-Queue
+Shelf
 """


-class QueueType(models.TextChoices):
+class ShelfType(models.TextChoices):
    WISHED = ('wished', '未开始')
    STARTED = ('started', '进行中')
    DONE = ('done', '完成')
    # DISCARDED = ('discarded', '放弃')


-QueueTypeNames = [
-    [ItemCategory.Book, QueueType.WISHED, _('想读')],
-    [ItemCategory.Book, QueueType.STARTED, _('在读')],
-    [ItemCategory.Book, QueueType.DONE, _('读过')],
-    [ItemCategory.Movie, QueueType.WISHED, _('想看')],
-    [ItemCategory.Movie, QueueType.STARTED, _('在看')],
-    [ItemCategory.Movie, QueueType.DONE, _('看过')],
-    [ItemCategory.TV, QueueType.WISHED, _('想看')],
-    [ItemCategory.TV, QueueType.STARTED, _('在看')],
-    [ItemCategory.TV, QueueType.DONE, _('看过')],
-    [ItemCategory.Music, QueueType.WISHED, _('想听')],
-    [ItemCategory.Music, QueueType.STARTED, _('在听')],
-    [ItemCategory.Music, QueueType.DONE, _('听过')],
-    [ItemCategory.Game, QueueType.WISHED, _('想玩')],
-    [ItemCategory.Game, QueueType.STARTED, _('在玩')],
-    [ItemCategory.Game, QueueType.DONE, _('玩过')],
-    [ItemCategory.Collection, QueueType.WISHED, _('关注')],
+ShelfTypeNames = [
+    [ItemCategory.Book, ShelfType.WISHED, _('想读')],
+    [ItemCategory.Book, ShelfType.STARTED, _('在读')],
+    [ItemCategory.Book, ShelfType.DONE, _('读过')],
+    [ItemCategory.Movie, ShelfType.WISHED, _('想看')],
+    [ItemCategory.Movie, ShelfType.STARTED, _('在看')],
+    [ItemCategory.Movie, ShelfType.DONE, _('看过')],
+    [ItemCategory.TV, ShelfType.WISHED, _('想看')],
+    [ItemCategory.TV, ShelfType.STARTED, _('在看')],
+    [ItemCategory.TV, ShelfType.DONE, _('看过')],
+    [ItemCategory.Music, ShelfType.WISHED, _('想听')],
+    [ItemCategory.Music, ShelfType.STARTED, _('在听')],
+    [ItemCategory.Music, ShelfType.DONE, _('听过')],
+    [ItemCategory.Game, ShelfType.WISHED, _('想玩')],
+    [ItemCategory.Game, ShelfType.STARTED, _('在玩')],
+    [ItemCategory.Game, ShelfType.DONE, _('玩过')],
+    [ItemCategory.Collection, ShelfType.WISHED, _('关注')],
    # TODO add more combinations
 ]


-class QueueMember(ListMember):
-    _queue = models.ForeignKey('Queue', related_name='members', on_delete=models.CASCADE)
+class ShelfMember(ListMember):
+    _shelf = models.ForeignKey('Shelf', related_name='members', on_delete=models.CASCADE)


-class Queue(List):
+class Shelf(List):
    class Meta:
-        unique_together = [['_owner', 'item_category', 'queue_type']]
+        unique_together = [['_owner', 'item_category', 'shelf_type']]

-    MEMBER_CLASS = QueueMember
-    items = models.ManyToManyField(Item, through='QueueMember', related_name="+")
+    MEMBER_CLASS = ShelfMember
+    items = models.ManyToManyField(Item, through='ShelfMember', related_name="+")
    item_category = models.CharField(choices=ItemCategory.choices, max_length=100, null=False, blank=False)
-    queue_type = models.CharField(choices=QueueType.choices, max_length=100, null=False, blank=False)
+    shelf_type = models.CharField(choices=ShelfType.choices, max_length=100, null=False, blank=False)

    def __str__(self):
        return f'{self.id} {self.title}'

    @cached_property
-    def queue_type_name(self):
-        return next(iter([n[2] for n in iter(QueueTypeNames) if n[0] == self.item_category and n[1] == self.queue_type]), self.queue_type)
+    def shelf_type_name(self):
+        return next(iter([n[2] for n in iter(ShelfTypeNames) if n[0] == self.item_category and n[1] == self.shelf_type]), self.shelf_type)

    @cached_property
    def title(self):
-        q = _("{item_category} {queue_type_name} list").format(queue_type_name=self.queue_type_name, item_category=self.item_category)
-        return _("{user}'s {queue_name}").format(user=self.owner.mastodon_username, queue_name=q)
+        q = _("{item_category} {shelf_type_name} list").format(shelf_type_name=self.shelf_type_name, item_category=self.item_category)
+        return _("{user}'s {shelf_name}").format(user=self.owner.mastodon_username, shelf_name=q)


-class QueueLogEntry(models.Model):
+class ShelfLogEntry(models.Model):
    owner = models.ForeignKey(User, on_delete=models.PROTECT)
-    queue = models.ForeignKey(Queue, on_delete=models.PROTECT, related_name='entries', null=True)  # None means removed from any queue
+    shelf = models.ForeignKey(Shelf, on_delete=models.PROTECT, related_name='entries', null=True)  # None means removed from any shelf
    item = models.ForeignKey(Item, on_delete=models.PROTECT)
+    timestamp = models.DateTimeField(default=timezone.now)  # this may later be changed by user
    metadata = models.JSONField(default=dict)
    created_time = models.DateTimeField(auto_now_add=True)
    edited_time = models.DateTimeField(auto_now=True)
-    queued_time = models.DateTimeField(default=timezone.now)

    def __str__(self):
-        return f'{self.owner}:{self.queue}:{self.item}:{self.metadata}'
+        return f'{self.owner}:{self.shelf}:{self.item}:{self.metadata}'


-class QueueManager:
+class ShelfManager:
    """
-    QueueManager
+    ShelfManager

-    all queue operations should go thru this class so that QueueLogEntry can be properly populated
-    QueueLogEntry can later be modified if user wish to change history
+    all shelf operations should go thru this class so that ShelfLogEntry can be properly populated
+    ShelfLogEntry can later be modified if user wish to change history
    """

    def __init__(self, user):
@ -264,55 +237,56 @@ class QueueManager:

    def initialize(self):
        for ic in ItemCategory:
-            for qt in QueueType:
-                Queue.objects.create(owner=self.owner, item_category=ic, queue_type=qt)
+            for qt in ShelfType:
+                Shelf.objects.create(owner=self.owner, item_category=ic, shelf_type=qt)

-    def _queue_member_for_item(self, item):
-        return QueueMember.objects.filter(item=item, _queue__in=self.owner.queue_set.all()).first()
+    def _shelf_member_for_item(self, item):
+        return ShelfMember.objects.filter(item=item, _shelf__in=self.owner.shelf_set.all()).first()

-    def _queue_for_item_and_type(item, queue_type):
-        if not item or not queue_type:
+    def _shelf_for_item_and_type(item, shelf_type):
+        if not item or not shelf_type:
            return None
-        return self.owner.queue_set.all().filter(item_category=item.category, queue_type=queue_type)
+        return self.owner.shelf_set.all().filter(item_category=item.category, shelf_type=shelf_type)

-    def update_for_item(self, item, queue_type, metadata=None):
-        # None means no change for metadata, comment
+    def move_item(self, item, shelf_type, visibility=0, metadata=None):
+        # shelf_type=None means remove from current shelf
+        # metadata=None means no change
        if not item:
            raise ValueError('empty item')
-        lastqm = self._queue_member_for_item(item)
+        lastqm = self._shelf_member_for_item(item)
        lastqmm = lastqm.metadata if lastqm else None
-        lastq = lastqm._queue if lastqm else None
-        lastqt = lastq.queue_type if lastq else None
-        queue = self.get_queue(item.category, queue_type) if queue_type else None
-        if lastq != queue:
+        lastq = lastqm._shelf if lastqm else None
+        lastqt = lastq.shelf_type if lastq else None
+        shelf = self.get_shelf(item.category, shelf_type) if shelf_type else None
+        if lastq != shelf:
            if lastq:
                lastq.remove_item(item)
-            if queue:
-                queue.append_item(item, metadata=metadata or {})
+            if shelf:
+                shelf.append_item(item, visibility=visibility, metadata=metadata or {})
        elif metadata is not None:
            lastqm.metadata = metadata
            lastqm.save()
        elif lastqm:
            metadata = lastqm.metadata
-        if lastqt != queue_type or (lastqt and metadata != lastqmm):
-            QueueLogEntry.objects.create(owner=self.owner, queue=queue, item=item, metadata=metadata or {})
+        if lastqt != shelf_type or (lastqt and metadata != lastqmm):
+            ShelfLogEntry.objects.create(owner=self.owner, shelf=shelf, item=item, metadata=metadata or {})

    def get_log(self):
-        return QueueLogEntry.objects.filter(owner=self.owner)
+        return ShelfLogEntry.objects.filter(owner=self.owner).order_by('timestamp')

    def get_log_for_item(self, item):
-        return QueueLogEntry.objects.filter(owner=self.owner, item=item)
+        return ShelfLogEntry.objects.filter(owner=self.owner, item=item).order_by('timestamp')

-    def get_queue(self, item_category, queue_type):
-        return self.owner.queue_set.all().filter(item_category=item_category, queue_type=queue_type).first()
+    def get_shelf(self, item_category, shelf_type):
+        return self.owner.shelf_set.all().filter(item_category=item_category, shelf_type=shelf_type).first()

    @staticmethod
    def get_manager_for_user(user):
-        return QueueManager(user)
+        return ShelfManager(user)


-User.queue_manager = cached_property(QueueManager.get_manager_for_user)
-User.queue_manager.__set_name__(User, 'queue_manager')
+User.shelf_manager = cached_property(ShelfManager.get_manager_for_user)
+User.shelf_manager.__set_name__(User, 'shelf_manager')


 """
--- a/journal/tests.py
+++ b/journal/tests.py
@ -24,46 +24,46 @@ class CollectionTest(TestCase):
        self.assertEqual(list(collection.ordered_items), [self.book2, self.book1])


-class QueueTest(TestCase):
+class ShelfTest(TestCase):
    def setUp(self):
        pass

-    def test_queue(self):
+    def test_shelf(self):
        user = User.objects.create(mastodon_site="site", username="name")
-        queue_manager = QueueManager(user=user)
-        queue_manager.initialize()
-        self.assertEqual(user.queue_set.all().count(), 33)
+        shelf_manager = ShelfManager(user=user)
+        shelf_manager.initialize()
+        self.assertEqual(user.shelf_set.all().count(), 33)
        book1 = Edition.objects.create(title="Hyperion")
        book2 = Edition.objects.create(title="Andymion")
-        q1 = queue_manager.get_queue(ItemCategory.Book, QueueType.WISHED)
-        q2 = queue_manager.get_queue(ItemCategory.Book, QueueType.STARTED)
+        q1 = shelf_manager.get_shelf(ItemCategory.Book, ShelfType.WISHED)
+        q2 = shelf_manager.get_shelf(ItemCategory.Book, ShelfType.STARTED)
        self.assertIsNotNone(q1)
        self.assertIsNotNone(q2)
        self.assertEqual(q1.members.all().count(), 0)
        self.assertEqual(q2.members.all().count(), 0)
-        queue_manager.update_for_item(book1, QueueType.WISHED)
-        queue_manager.update_for_item(book2, QueueType.WISHED)
+        shelf_manager.move_item(book1, ShelfType.WISHED)
+        shelf_manager.move_item(book2, ShelfType.WISHED)
        self.assertEqual(q1.members.all().count(), 2)
-        queue_manager.update_for_item(book1, QueueType.STARTED)
+        shelf_manager.move_item(book1, ShelfType.STARTED)
        self.assertEqual(q1.members.all().count(), 1)
        self.assertEqual(q2.members.all().count(), 1)
-        queue_manager.update_for_item(book1, QueueType.STARTED, metadata={'progress': 1})
+        shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 1})
        self.assertEqual(q1.members.all().count(), 1)
        self.assertEqual(q2.members.all().count(), 1)
-        log = queue_manager.get_log_for_item(book1)
+        log = shelf_manager.get_log_for_item(book1)
        self.assertEqual(log.count(), 3)
-        queue_manager.update_for_item(book1, QueueType.STARTED, metadata={'progress': 1})
-        log = queue_manager.get_log_for_item(book1)
+        shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 1})
+        log = shelf_manager.get_log_for_item(book1)
        self.assertEqual(log.count(), 3)
-        queue_manager.update_for_item(book1, QueueType.STARTED, metadata={'progress': 10})
-        log = queue_manager.get_log_for_item(book1)
+        shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 10})
+        log = shelf_manager.get_log_for_item(book1)
        self.assertEqual(log.count(), 4)
-        queue_manager.update_for_item(book1, QueueType.STARTED)
-        log = queue_manager.get_log_for_item(book1)
+        shelf_manager.move_item(book1, ShelfType.STARTED)
+        log = shelf_manager.get_log_for_item(book1)
        self.assertEqual(log.count(), 4)
-        self.assertEqual(log.order_by('queued_time').last().metadata, {'progress': 10})
-        queue_manager.update_for_item(book1, QueueType.STARTED, metadata={'progress': 100})
-        log = queue_manager.get_log_for_item(book1)
+        self.assertEqual(log.last().metadata, {'progress': 10})
+        shelf_manager.move_item(book1, ShelfType.STARTED, metadata={'progress': 100})
+        log = shelf_manager.get_log_for_item(book1)
        self.assertEqual(log.count(), 5)


--- a/misc/dev-reset.sh
+++ b/misc/dev-reset.sh
@ -0,0 +1,31 @@
+#!/bin/sh
+# Reset databases and migrations, for development only
+
+[ -f manage.py ] || exit $1
+
+echo "\033[0;31mWARNING"
+while true; do
+    read -p "Do you wish to continue destroy all databases and migrations? (yes/no) " yn
+    case $yn in
+        [Yy]* ) break;;
+        [Nn]* ) exit;;
+    esac
+done
+
+psql $* postgres -c "DROP DATABASE IF EXISTS neodb;" || exit $?
+
+psql $* postgres -c "DROP DATABASE IF EXISTS test_neodb;" || exit $?
+
+psql $* postgres -c "CREATE DATABASE neodb ENCODING 'UTF8' LC_COLLATE='en_US.UTF-8' LC_CTYPE='en_US.UTF-8' TEMPLATE template0;" || exit $?
+
+psql $* neodb -c "CREATE EXTENSION hstore WITH SCHEMA public;" || exit $?
+
+find -type d -name migrations | xargs rm -rf
+
+python3 manage.py makemigrations mastodon users books movies games music sync management collection catalog journal social || exit $?
+
+python3 manage.py migrate || exit $?
+
+psql $* neodb -c "CREATE DATABASE test_neodb WITH TEMPLATE neodb;" || exit $?
+
+python3 manage.py check
--- a/social/models.py
+++ b/social/models.py
@ -43,6 +43,7 @@ class ActivityManager:

    def get_viewable_activities(self, before_time=None):
        q = Q(owner_id__in=self.owner.following, visibility__lt=2) | Q(owner=self.owner)
+        q = q & Q(is_viewable=True)
        if before_time:
            q = q & Q(created_time__lt=before_time)
        return Activity.objects.filter(q)
@ -72,16 +73,20 @@ class Activity(models.Model, UserOwnedObjectMixin):
    def action_class(self):
        return self.action_object.__class__.__name__

+    def __str__(self):
+        return f'{self.id}:{self.action_type}:{self.action_object}:{self.is_viewable}'
+

 class DefaultSignalProcessor():
    def __init__(self, action_object):
        self.action_object = action_object

    def activity_viewable(self, action_type):
-        return action_type == ActionType.Create and bool(getattr(self.action_object, 'attached_to', None))
+        return action_type == ActionType.Create and bool(getattr(self.action_object, 'attached_to', None) is None)

    def created(self):
-        return Activity.objects.create(owner=self.action_object.owner, visibility=self.action_object.visibility, action_object=self.action_object, action_type=ActionType.Create, is_viewable=self.activity_viewable(ActionType.Create))
+        activity = Activity.objects.create(owner=self.action_object.owner, visibility=self.action_object.visibility, action_object=self.action_object, action_type=ActionType.Create, is_viewable=self.activity_viewable(ActionType.Create))
+        return activity

    def updated(self):
        create_activity = Activity.objects.filter(owner=self.action_object.owner, action_object=self.action_object, action_type=ActionType.Create).first()
@ -92,8 +97,10 @@ class DefaultSignalProcessor():
    def deleted(self):
        create_activity = Activity.objects.filter(owner=self.action_object.owner, action_object=self.action_object, action_type=ActionType.Create).first()
        if create_activity:
-            create_activity.viewable = False
+            create_activity.is_viewable = False
            create_activity.save()
+        else:
+            _logger.warning(f'unable to find create activity for {self.action_object}')
        # FIXME action_object=self.action_object causing issues in test when hard delete, the bare minimum is to save id of the actual object that ActivityPub requires
        return Activity.objects.create(owner=self.action_object.owner, visibility=self.action_object.visibility, action_object=None, action_type=ActionType.Delete, is_viewable=self.activity_viewable(ActionType.Delete))

@ -147,7 +154,7 @@ class DataSignalManager:

@DataSignalManager.register
 class MarkProcessor(DefaultSignalProcessor):
-    model = QueueMember
+    model = ShelfMember


 # @DataSignalManager.register
--- a/social/tests.py
+++ b/social/tests.py
@ -9,15 +9,46 @@ class SocialTest(TestCase):
    def setUp(self):
        self.book1 = Edition.objects.create(title="Hyperion")
        self.book2 = Edition.objects.create(title="Andymion")
+        self.movie = Edition.objects.create(title="Fight Club")
        self.alice = User.objects.create(mastodon_site="MySpace", username="Alice")
-        self.alice.queue_manager.initialize()
+        self.alice.shelf_manager.initialize()
        self.bob = User.objects.create(mastodon_site="KKCity", username="Bob")
-        self.bob.queue_manager.initialize()
+        self.bob.shelf_manager.initialize()

    def test_timeline(self):
-        timeline = list(self.alice.activity_manager.get_viewable_activities())
-        self.assertEqual(timeline, [])
+        # alice see 0 activity in timeline in the beginning
+        timeline = self.alice.activity_manager.get_viewable_activities()
+        self.assertEqual(len(timeline), 0)

-        self.alice.queue_manager.update_for_item(self.book1, QueueType.WISHED)
-        timeline = list(self.alice.activity_manager.get_viewable_activities())
+        # 1 activity after adding first book to shelf
+        self.alice.shelf_manager.move_item(self.book1, ShelfType.WISHED, visibility=1)
+        timeline = self.alice.activity_manager.get_viewable_activities()
        self.assertEqual(len(timeline), 1)
+
+        # 2 activities after adding second book to shelf
+        self.alice.shelf_manager.move_item(self.book2, ShelfType.WISHED)
+        timeline = self.alice.activity_manager.get_viewable_activities()
+        self.assertEqual(len(timeline), 2)
+
+        # 2 activities after change first mark
+        self.alice.shelf_manager.move_item(self.book1, ShelfType.STARTED)
+        timeline = self.alice.activity_manager.get_viewable_activities()
+        self.assertEqual(len(timeline), 2)
+
+        # bon see 0 activity in timeline in the beginning
+        timeline2 = self.bob.activity_manager.get_viewable_activities()
+        self.assertEqual(len(timeline2), 0)
+
+        # bob follows alice, see 2 activities
+        self.bob.mastodon_following = ['Alice@MySpace']
+        self.alice.mastodon_follower = ['Bob@KKCity']
+        self.bob.following = self.bob.get_following_ids()
+        timeline2 = self.bob.activity_manager.get_viewable_activities()
+        self.assertEqual(len(timeline2), 2)
+
+        # alice:3 bob:2 after alice adding second book to shelf as private
+        self.alice.shelf_manager.move_item(self.movie, ShelfType.WISHED, visibility=2)
+        timeline = self.alice.activity_manager.get_viewable_activities()
+        self.assertEqual(len(timeline), 3)
+        timeline2 = self.bob.activity_manager.get_viewable_activities()
+        self.assertEqual(len(timeline2), 2)