diff --git a/common/index.py b/common/index.py index 63b607e7..44d73edf 100644 --- a/common/index.py +++ b/common/index.py @@ -48,6 +48,10 @@ class Indexer: self.instance().update_filterable_attributes(['_class', 'tags', 'genre', 'source_site']) self.instance().update_settings({'displayedAttributes': ['_id', '_class', 'id', 'title', 'tags']}) + @classmethod + def get_stats(self): + return self.instance().get_stats() + @classmethod def update_model_indexable(self, model): model.indexable_fields = ['tags'] @@ -68,7 +72,7 @@ class Indexer: post_delete.connect(item_post_delete_handler, sender=model) @classmethod - def replace_item(self, obj): + def obj_to_dict(self, obj): pk = f'{obj.__class__.__name__}-{obj.id}' item = { '_id': pk, @@ -86,8 +90,11 @@ class Indexer: if d.__class__ is dict: item.update(d) item = {k: v for k, v in item.items() if v} - # print(item) - self.instance().add_documents([item]) + return item + + @classmethod + def replace_item(self, obj): + self.instance().add_documents([self.obj_to_dict(obj)]) @classmethod def delete_item(self, obj): diff --git a/common/management/commands/reindex.py b/common/management/commands/reindex.py index 48cb5f65..b00af326 100644 --- a/common/management/commands/reindex.py +++ b/common/management/commands/reindex.py @@ -5,15 +5,34 @@ from movies.models import Movie from books.models import Book from games.models import Game from music.models import Album, Song +from django.core.paginator import Paginator +from tqdm import tqdm +from time import sleep +from datetime import timedelta +from django.utils import timezone + + +BATCH_SIZE = 1000 class Command(BaseCommand): help = 'Regenerate the search index' + def add_arguments(self, parser): + parser.add_argument('hours', type=int, help='Re-index items modified in last N hours, 0 to reindex all') + def handle(self, *args, **options): + h = int(options['hours']) print(f'Connecting to search server {settings.MEILISEARCH_SERVER} for index: {INDEX_NAME}') + Indexer.update_settings() self.stdout.write(self.style.SUCCESS('Index settings updated.')) - for c in [Movie, Book, Album, Song, Game]: + for c in [Game, Movie, Book, Album, Song]: print(f'Re-indexing {c}') - for i in c.objects.all(): - Indexer.replace_item(i) + qs = c.objects.all() if h == 0 else c.objects.filter(edited_time__gt=timezone.now() - timedelta(hours=h)) + pg = Paginator(qs.order_by('id'), BATCH_SIZE) + for p in tqdm(pg.page_range): + items = list(map(lambda o: Indexer.obj_to_dict(o), pg.get_page(p).object_list)) + if items: + Indexer.instance().update_documents(documents=items) + while Indexer.get_stats()['isIndexing']: + sleep(0.1)