From 274fe591a140b44c314390289514b003d2dc9e38 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 31 Dec 2022 18:55:27 -0500 Subject: [PATCH] new data model: fix reindex --- catalog/management/commands/index.py | 44 +++++++++++++++++++++++++-- catalog/search/typesense.py | 18 +++++++---- catalog/templates/album.html | 2 +- catalog/templates/game.html | 2 +- catalog/templates/sidebar_item.html | 2 +- catalog/views.py | 1 + journal/templates/list_item_base.html | 7 +++-- 7 files changed, 62 insertions(+), 14 deletions(-) diff --git a/catalog/management/commands/index.py b/catalog/management/commands/index.py index a7fdfbcb..a9dc84a5 100644 --- a/catalog/management/commands/index.py +++ b/catalog/management/commands/index.py @@ -2,6 +2,13 @@ from django.core.management.base import BaseCommand from django.conf import settings from catalog.models import * import pprint +from django.core.paginator import Paginator +from tqdm import tqdm +from time import sleep +from datetime import timedelta +from django.utils import timezone + +BATCH_SIZE = 1000 class Command(BaseCommand): @@ -13,26 +20,59 @@ class Command(BaseCommand): help="initialize index", action="store_true", ) + parser.add_argument( + "--update", + help="update index schema", + action="store_true", + ) parser.add_argument( "--stat", action="store_true", ) + parser.add_argument( + "--reindex", + action="store_true", + ) def init_index(self): - self.stdout.write(f"Connecting to search server") Indexer.init() self.stdout.write(self.style.SUCCESS("Index created.")) + def update_index(self): + Indexer.update_settings() + self.stdout.write(self.style.SUCCESS("Index updated.")) + def stat(self, *args, **options): - self.stdout.write(f"Connecting to search server") stats = Indexer.get_stats() pprint.pp(stats) + def reindex(self): + if Indexer.busy(): + self.stdout.write("Please wait for previous updates") + # Indexer.update_settings() + # self.stdout.write(self.style.SUCCESS('Index settings updated.')) + qs = ( + Item.objects.all() + ) # if h == 0 else c.objects.filter(edited_time__gt=timezone.now() - timedelta(hours=h)) + pg = Paginator(qs.order_by("id"), BATCH_SIZE) + for p in tqdm(pg.page_range): + items = list( + map(lambda o: Indexer.obj_to_dict(o), pg.get_page(p).object_list) + ) + if items: + Indexer.replace_batch(items) + while Indexer.busy(): + sleep(0.5) + def handle(self, *args, **options): if options["init"]: self.init_index() + elif options["update"]: + self.update_index() elif options["stat"]: self.stat() + elif options["reindex"]: + self.reindex() # else: # try: diff --git a/catalog/search/typesense.py b/catalog/search/typesense.py index c8481f46..13467f5a 100644 --- a/catalog/search/typesense.py +++ b/catalog/search/typesense.py @@ -75,8 +75,7 @@ class Indexer: return cls._instance @classmethod - def init(cls): - # cls.instance().collections[INDEX_NAME].delete() + def config(cls): # fields = [ # {"name": "_class", "type": "string", "facet": True}, # {"name": "source_site", "type": "string", "facet": True}, @@ -122,14 +121,20 @@ class Indexer: {"name": "tags", "optional": True, "locale": "zh", "type": "string[]"}, {"name": ".*", "optional": True, "locale": "zh", "type": "auto"}, ] + return { + "name": INDEX_NAME, + "fields": fields, + # "default_sorting_field": "rating_count", + } - cls.instance().collections.create({"name": INDEX_NAME, "fields": fields}) + @classmethod + def init(cls): + # cls.instance().collections[INDEX_NAME].delete() + cls.instance().collections.create(cls.config()) @classmethod def update_settings(cls): - # https://github.com/typesense/typesense/issues/96 - # FIXME - pass + cls.instance().collections[INDEX_NAME].update(cls.config()) @classmethod def get_stats(cls): @@ -227,6 +232,7 @@ class Indexer: "per_page": SEARCH_PAGE_SIZE, "query_by": ",".join(SEARCHABLE_ATTRIBUTES), "filter_by": filters, + "sort_by": "_text_match:desc,rating_count:desc" # 'facetsDistribution': ['_class'], # 'sort_by': None, } diff --git a/catalog/templates/album.html b/catalog/templates/album.html index a6d04396..f87161f4 100644 --- a/catalog/templates/album.html +++ b/catalog/templates/album.html @@ -16,7 +16,7 @@
{% if item.rating and item.rating_count >= 5 %} - {{ item.rating }} + {{ item.rating | floatformat:1 }} ({{ item.rating_count }}人评分) {% else %} {% trans '评分：评分人数不足' %} diff --git a/catalog/templates/game.html b/catalog/templates/game.html index 3ca075a6..abc5b518 100644 --- a/catalog/templates/game.html +++ b/catalog/templates/game.html @@ -16,7 +16,7 @@
{% if item.rating and item.rating_count >= 5 %} - {{ item.rating }} + {{ item.rating | floatformat:1 }} ({{ item.rating_count }}人评分) {% else %} {% trans '评分：评分人数不足' %} diff --git a/catalog/templates/sidebar_item.html b/catalog/templates/sidebar_item.html index c8249eef..ab52f96c 100644 --- a/catalog/templates/sidebar_item.html +++ b/catalog/templates/sidebar_item.html @@ -22,7 +22,7 @@
{% if item.pub_house %}{% trans '出版社：' %}{{ item.pub_house }}{% endif %}
{% if item.rating %} - {% trans '评分: ' %} + {% trans '评分: ' %} {{ item.rating | floatformat:1 }} {% endif %}
diff --git a/catalog/views.py b/catalog/views.py index b4e2deea..21f69018 100644 --- a/catalog/views.py +++ b/catalog/views.py @@ -267,5 +267,6 @@ def search(request): PAGE_LINK_NUMBER, page_number, result.num_pages ), "categories": ["book", "movie", "music", "game"], + "hide_category": category is not None, }, ) diff --git a/journal/templates/list_item_base.html b/journal/templates/list_item_base.html index 9f800b26..3bc8c2c2 100644 --- a/journal/templates/list_item_base.html +++ b/journal/templates/list_item_base.html @@ -36,15 +36,16 @@ {% else %} {{ item.title }} {% endif %} + {% if item.year %}({{ item.year }}){% endif %} - {% if not request.GET.c and not hide_category %} - [{{item.category.label}}] - {% endif %} {% for res in item.external_resources.all %} {{ res.site_name.label }} {% endfor %} + {% if not hide_category %} + [{{item.category.label}}] + {% endif %}
{% if item.rating %}