From a0de1ecbd0e57a31cf94d539af478dd2801cefcf Mon Sep 17 00:00:00 2001 From: Their Name Date: Thu, 30 Dec 2021 07:38:38 -0500 Subject: [PATCH] write index to meilisearch --- books/apps.py | 5 ++ common/index.py | 97 ++++++++++++++++++++++++ common/management/commands/init_index.py | 16 ++++ common/management/commands/reindex.py | 19 +++++ common/models.py | 9 ++- games/apps.py | 5 ++ movies/apps.py | 5 ++ music/apps.py | 6 ++ 8 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 common/index.py create mode 100644 common/management/commands/init_index.py create mode 100644 common/management/commands/reindex.py diff --git a/books/apps.py b/books/apps.py index f716137a..b03e2d23 100644 --- a/books/apps.py +++ b/books/apps.py @@ -3,3 +3,8 @@ from django.apps import AppConfig class BooksConfig(AppConfig): name = 'books' + + def ready(self): + from common.index import Indexer + from .models import Book + Indexer.update_model_indexable(Book) diff --git a/common/index.py b/common/index.py new file mode 100644 index 00000000..a31bc922 --- /dev/null +++ b/common/index.py @@ -0,0 +1,97 @@ +import meilisearch +from django.conf import settings +from django.db.models.signals import post_save, post_delete + + +# TODO +# use post_save, post_delete +# search result translate back to model +INDEX_NAME = 'items' +INDEX_SEARCHABLE_ATTRIBUTES = ['title', 'orig_title', 'other_title', 'subtitle', 'artist', 'author', 'translator', 'developer', 'brief', 'contents', 'track_list', 'pub_house', 'company', 'publisher', 'isbn', 'imdb_code', 'UPC', 'TMDB_ID', 'BANDCAMP_ALBUM_ID'] +INDEXABLE_DIRECT_TYPES = ['BigAutoField', 'BooleanField', 'CharField', 'DecimalField', 'PositiveIntegerField', 'PositiveSmallIntegerField', 'TextField', 'ArrayField'] +INDEXABLE_TIME_TYPES = ['DateTimeField'] +INDEXABLE_DICT_TYPES = ['JSONField'] +# NONINDEXABLE_TYPES = ['ForeignKey', 'FileField'] + + +def item_post_save_handler(sender, instance, **kwargs): + Indexer.replace_item(instance) + + +def item_post_delete_handler(sender, instance, **kwargs): + Indexer.delete_item(instance) + + +def tag_post_save_handler(sender, instance, **kwargs): + pass + + +def tag_post_delete_handler(sender, instance, **kwargs): + pass + + +class Indexer: + @classmethod + def instance(self): + return meilisearch.Client(settings.MEILISEARCH_SERVER, settings.MEILISEARCH_KEY).index(INDEX_NAME) + # TODO cache per process/request + + @classmethod + def init(self): + meilisearch.Client(settings.MEILISEARCH_SERVER, settings.MEILISEARCH_KEY).create_index(INDEX_NAME, {'primaryKey': '_id'}) + self.update_settings() + + @classmethod + def update_settings(self): + self.instance().update_searchable_attributes(INDEX_SEARCHABLE_ATTRIBUTES) + self.instance().update_filterable_attributes(['_class', 'tags', 'genre', 'source_site']) + self.instance().update_settings({'displayedAttributes': ['_id', '_class', 'id', 'title', 'tags']}) + + @classmethod + def update_model_indexable(self, model): + model.indexable_fields = ['tags'] + model.indexable_fields_time = [] + model.indexable_fields_dict = [] + for field in model._meta.get_fields(): + type = field.get_internal_type() + if type in INDEXABLE_DIRECT_TYPES: + model.indexable_fields.append(field.name) + elif type in INDEXABLE_TIME_TYPES: + model.indexable_fields_time.append(field.name) + elif type in INDEXABLE_DICT_TYPES: + model.indexable_fields_dict.append(field.name) + post_save.connect(item_post_save_handler, sender=model) + post_delete.connect(item_post_delete_handler, sender=model) + + @classmethod + def replace_item(self, obj): + pk = f'{obj.__class__.__name__}-{obj.id}' + item = { + '_id': pk, + '_class': obj.__class__.__name__, + # 'id': obj.id + } + for field in obj.__class__.indexable_fields: + item[field] = getattr(obj, field) + for field in obj.__class__.indexable_fields_time: + item[field] = getattr(obj, field).timestamp() + for field in obj.__class__.indexable_fields_dict: + d = getattr(obj, field) + if d.__class__ is dict: + item.update(d) + item = {k: v for k, v in item.items() if v} + # print(item) + self.instance().add_documents([item]) + + @classmethod + def delete_item(self, obj): + pk = f'{obj.__class__.__name__}-{obj.id}' + self.instance().delete_document(pk) + + @classmethod + def patch_item(self, obj, fields): + pk = f'{obj.__class__.__name__}-{obj.id}' + data = {} + for f in fields: + data[f] = getattr(obj, f) + self.instance().update_documents(documents=[data], primary_key=[pk]) diff --git a/common/management/commands/init_index.py b/common/management/commands/init_index.py new file mode 100644 index 00000000..64378af3 --- /dev/null +++ b/common/management/commands/init_index.py @@ -0,0 +1,16 @@ +from django.core.management.base import BaseCommand +from common.index import Indexer, INDEX_NAME +from django.conf import settings + + +class Command(BaseCommand): + help = 'Initialize the search index' + + def handle(self, *args, **options): + print(f'Connecting to search server {settings.MEILISEARCH_SERVER} for index: {INDEX_NAME}') + try: + Indexer.init() + self.stdout.write(self.style.SUCCESS('Index created.')) + except Exception: + Indexer.update_settings() + self.stdout.write(self.style.SUCCESS('Index settings updated.')) diff --git a/common/management/commands/reindex.py b/common/management/commands/reindex.py new file mode 100644 index 00000000..48cb5f65 --- /dev/null +++ b/common/management/commands/reindex.py @@ -0,0 +1,19 @@ +from django.core.management.base import BaseCommand +from common.index import Indexer, INDEX_NAME +from django.conf import settings +from movies.models import Movie +from books.models import Book +from games.models import Game +from music.models import Album, Song + + +class Command(BaseCommand): + help = 'Regenerate the search index' + + def handle(self, *args, **options): + print(f'Connecting to search server {settings.MEILISEARCH_SERVER} for index: {INDEX_NAME}') + self.stdout.write(self.style.SUCCESS('Index settings updated.')) + for c in [Movie, Book, Album, Song, Game]: + print(f'Re-indexing {c}') + for i in c.objects.all(): + Indexer.replace_item(i) diff --git a/common/models.py b/common/models.py index 7c0c80e3..0b1ec8d7 100644 --- a/common/models.py +++ b/common/models.py @@ -56,7 +56,6 @@ class Entity(models.Model): rating__lte=10), name='%(class)s_rating_upperbound'), ] - def get_absolute_url(self): raise NotImplementedError("Subclass should implement this method") @@ -137,6 +136,14 @@ class Entity(models.Model): """ raise NotImplementedError("Subclass should implement this method.") + @property + def tag_list(self): + return self.get_tags_manager().values('content').annotate(frequency=Count('content')).order_by('-frequency') + + @property + def tags(self): + return list(map(lambda t:t['content'], self.tag_list)) + @classmethod def get_category_mapping_dict(cls): category_mapping_dict = {} diff --git a/games/apps.py b/games/apps.py index b74f62c9..a204c094 100644 --- a/games/apps.py +++ b/games/apps.py @@ -3,3 +3,8 @@ from django.apps import AppConfig class GamesConfig(AppConfig): name = 'games' + + def ready(self): + from common.index import Indexer + from .models import Game + Indexer.update_model_indexable(Game) diff --git a/movies/apps.py b/movies/apps.py index bda16f08..3b025da4 100644 --- a/movies/apps.py +++ b/movies/apps.py @@ -3,3 +3,8 @@ from django.apps import AppConfig class MoviesConfig(AppConfig): name = 'movies' + + def ready(self): + from common.index import Indexer + from .models import Movie + Indexer.update_model_indexable(Movie) diff --git a/music/apps.py b/music/apps.py index d909c7fb..6fb97b37 100644 --- a/music/apps.py +++ b/music/apps.py @@ -3,3 +3,9 @@ from django.apps import AppConfig class MusicConfig(AppConfig): name = 'music' + + def ready(self): + from common.index import Indexer + from .models import Album, Song + Indexer.update_model_indexable(Album) + Indexer.update_model_indexable(Song)