write index to meilisearch

books/apps.py

@@ -3,3 +3,8 @@ from django.apps import AppConfig
 
 class BooksConfig(AppConfig):
     name = 'books'
+
+    def ready(self):
+        from common.index import Indexer
+        from .models import Book
+        Indexer.update_model_indexable(Book)

common/index.py

@@ -0,0 +1,97 @@
+import meilisearch
+from django.conf import settings
+from django.db.models.signals import post_save, post_delete
+
+
+# TODO
+# use post_save, post_delete
+# search result translate back to model
+INDEX_NAME = 'items'
+INDEX_SEARCHABLE_ATTRIBUTES = ['title', 'orig_title', 'other_title', 'subtitle', 'artist', 'author', 'translator', 'developer', 'brief', 'contents', 'track_list', 'pub_house', 'company', 'publisher', 'isbn', 'imdb_code', 'UPC', 'TMDB_ID', 'BANDCAMP_ALBUM_ID']
+INDEXABLE_DIRECT_TYPES = ['BigAutoField', 'BooleanField', 'CharField', 'DecimalField', 'PositiveIntegerField', 'PositiveSmallIntegerField', 'TextField', 'ArrayField']
+INDEXABLE_TIME_TYPES = ['DateTimeField']
+INDEXABLE_DICT_TYPES = ['JSONField']
+# NONINDEXABLE_TYPES = ['ForeignKey', 'FileField']
+
+
+def item_post_save_handler(sender, instance, **kwargs):
+    Indexer.replace_item(instance)
+
+
+def item_post_delete_handler(sender, instance, **kwargs):
+    Indexer.delete_item(instance)
+
+
+def tag_post_save_handler(sender, instance, **kwargs):
+    pass
+
+
+def tag_post_delete_handler(sender, instance, **kwargs):
+    pass
+
+
+class Indexer:
+    @classmethod
+    def instance(self):
+        return meilisearch.Client(settings.MEILISEARCH_SERVER, settings.MEILISEARCH_KEY).index(INDEX_NAME)
+        # TODO cache per process/request
+
+    @classmethod
+    def init(self):
+        meilisearch.Client(settings.MEILISEARCH_SERVER, settings.MEILISEARCH_KEY).create_index(INDEX_NAME, {'primaryKey': '_id'})
+        self.update_settings()
+
+    @classmethod
+    def update_settings(self):
+        self.instance().update_searchable_attributes(INDEX_SEARCHABLE_ATTRIBUTES)
+        self.instance().update_filterable_attributes(['_class', 'tags', 'genre', 'source_site'])
+        self.instance().update_settings({'displayedAttributes': ['_id', '_class', 'id', 'title', 'tags']})
+
+    @classmethod
+    def update_model_indexable(self, model):
+        model.indexable_fields = ['tags']
+        model.indexable_fields_time = []
+        model.indexable_fields_dict = []
+        for field in model._meta.get_fields():
+            type = field.get_internal_type()
+            if type in INDEXABLE_DIRECT_TYPES:
+                model.indexable_fields.append(field.name)
+            elif type in INDEXABLE_TIME_TYPES:
+                model.indexable_fields_time.append(field.name)
+            elif type in INDEXABLE_DICT_TYPES:
+                model.indexable_fields_dict.append(field.name)
+        post_save.connect(item_post_save_handler, sender=model)
+        post_delete.connect(item_post_delete_handler, sender=model)
+
+    @classmethod
+    def replace_item(self, obj):
+        pk = f'{obj.__class__.__name__}-{obj.id}'
+        item = {
+            '_id': pk,
+            '_class': obj.__class__.__name__,
+            # 'id': obj.id
+        }
+        for field in obj.__class__.indexable_fields:
+            item[field] = getattr(obj, field)
+        for field in obj.__class__.indexable_fields_time:
+            item[field] = getattr(obj, field).timestamp()
+        for field in obj.__class__.indexable_fields_dict:
+            d = getattr(obj, field)
+            if d.__class__ is dict:
+                item.update(d)
+        item = {k: v for k, v in item.items() if v}
+        # print(item)
+        self.instance().add_documents([item])
+
+    @classmethod
+    def delete_item(self, obj):
+        pk = f'{obj.__class__.__name__}-{obj.id}'
+        self.instance().delete_document(pk)
+
+    @classmethod
+    def patch_item(self, obj, fields):
+        pk = f'{obj.__class__.__name__}-{obj.id}'
+        data = {}
+        for f in fields:
+            data[f] = getattr(obj, f)
+        self.instance().update_documents(documents=[data], primary_key=[pk])

common/management/commands/init_index.py

@@ -0,0 +1,16 @@
+from django.core.management.base import BaseCommand
+from common.index import Indexer, INDEX_NAME
+from django.conf import settings
+
+
+class Command(BaseCommand):
+    help = 'Initialize the search index'
+
+    def handle(self, *args, **options):
+        print(f'Connecting to search server {settings.MEILISEARCH_SERVER} for index: {INDEX_NAME}')
+        try:
+            Indexer.init()
+            self.stdout.write(self.style.SUCCESS('Index created.'))
+        except Exception:
+            Indexer.update_settings()
+            self.stdout.write(self.style.SUCCESS('Index settings updated.'))

common/management/commands/reindex.py

@@ -0,0 +1,19 @@
+from django.core.management.base import BaseCommand
+from common.index import Indexer, INDEX_NAME
+from django.conf import settings
+from movies.models import Movie
+from books.models import Book
+from games.models import Game
+from music.models import Album, Song
+
+
+class Command(BaseCommand):
+    help = 'Regenerate the search index'
+
+    def handle(self, *args, **options):
+        print(f'Connecting to search server {settings.MEILISEARCH_SERVER} for index: {INDEX_NAME}')
+        self.stdout.write(self.style.SUCCESS('Index settings updated.'))
+        for c in [Movie, Book, Album, Song, Game]:
+            print(f'Re-indexing {c}')
+            for i in c.objects.all():
+                Indexer.replace_item(i)

common/models.py

@@ -56,7 +56,6 @@ class Entity(models.Model):
                 rating__lte=10), name='%(class)s_rating_upperbound'),
         ]
 
-
     def get_absolute_url(self):
         raise NotImplementedError("Subclass should implement this method")
 
@@ -137,6 +136,14 @@ class Entity(models.Model):
         """
         raise NotImplementedError("Subclass should implement this method.")
 
+    @property
+    def tag_list(self):
+        return self.get_tags_manager().values('content').annotate(frequency=Count('content')).order_by('-frequency')
+
+    @property
+    def tags(self):
+        return list(map(lambda t:t['content'], self.tag_list))
+
     @classmethod
     def get_category_mapping_dict(cls):
         category_mapping_dict = {}

games/apps.py

@@ -3,3 +3,8 @@ from django.apps import AppConfig
 
 class GamesConfig(AppConfig):
     name = 'games'
+
+    def ready(self):
+        from common.index import Indexer
+        from .models import Game
+        Indexer.update_model_indexable(Game)

movies/apps.py

@@ -3,3 +3,8 @@ from django.apps import AppConfig
 
 class MoviesConfig(AppConfig):
     name = 'movies'
+
+    def ready(self):
+        from common.index import Indexer
+        from .models import Movie
+        Indexer.update_model_indexable(Movie)

music/apps.py

@@ -3,3 +3,9 @@ from django.apps import AppConfig
 
 class MusicConfig(AppConfig):
     name = 'music'
+
+    def ready(self):
+        from common.index import Indexer
+        from .models import Album, Song
+        Indexer.update_model_indexable(Album)
+        Indexer.update_model_indexable(Song)