diff --git a/catalog/apps.py b/catalog/apps.py index aff10ed7..d5052fc8 100644 --- a/catalog/apps.py +++ b/catalog/apps.py @@ -10,3 +10,6 @@ class CatalogConfig(AppConfig): from catalog import models from catalog import sites from journal import models as journal_models + from catalog.models import init_catalog_search_models + + init_catalog_search_models() diff --git a/catalog/management/commands/index.py b/catalog/management/commands/index.py new file mode 100644 index 00000000..a7fdfbcb --- /dev/null +++ b/catalog/management/commands/index.py @@ -0,0 +1,43 @@ +from django.core.management.base import BaseCommand +from django.conf import settings +from catalog.models import * +import pprint + + +class Command(BaseCommand): + help = "Manage the search index" + + def add_arguments(self, parser): + parser.add_argument( + "--init", + help="initialize index", + action="store_true", + ) + parser.add_argument( + "--stat", + action="store_true", + ) + + def init_index(self): + self.stdout.write(f"Connecting to search server") + Indexer.init() + self.stdout.write(self.style.SUCCESS("Index created.")) + + def stat(self, *args, **options): + self.stdout.write(f"Connecting to search server") + stats = Indexer.get_stats() + pprint.pp(stats) + + def handle(self, *args, **options): + if options["init"]: + self.init_index() + elif options["stat"]: + self.stat() + # else: + + # try: + # Indexer.init() + # self.stdout.write(self.style.SUCCESS('Index created.')) + # except Exception: + # Indexer.update_settings() + # self.stdout.write(self.style.SUCCESS('Index settings updated.')) diff --git a/catalog/models.py b/catalog/models.py index 75b13ec5..212b3789 100644 --- a/catalog/models.py +++ b/catalog/models.py @@ -8,6 +8,19 @@ from .podcast.models import Podcast from .performance.models import Performance from .collection.models import Collection as CatalogCollection from django.contrib.contenttypes.models import ContentType +from django.conf import settings + + +if settings.SEARCH_BACKEND == "MEILISEARCH": + from .search.meilisearch import Indexer +elif settings.SEARCH_BACKEND == "TYPESENSE": + from .search.typesense import Indexer +else: + + class Indexer: + @classmethod + def update_model_indexable(cls, model): + pass # class Exhibition(Item): @@ -54,3 +67,13 @@ def all_categories(): else: _CATEGORY_LIST[c].append(cls) return _CATEGORY_LIST + + +def init_catalog_search_models(): + Indexer.update_model_indexable(Edition) + Indexer.update_model_indexable(Work) + Indexer.update_model_indexable(Movie) + Indexer.update_model_indexable(TVShow) + Indexer.update_model_indexable(TVSeason) + Indexer.update_model_indexable(Album) + Indexer.update_model_indexable(Game) diff --git a/catalog/search/typesense.py b/catalog/search/typesense.py new file mode 100644 index 00000000..c8481f46 --- /dev/null +++ b/catalog/search/typesense.py @@ -0,0 +1,257 @@ +import types +import logging +import typesense +from typesense.exceptions import ObjectNotFound +from django.conf import settings +from django.db.models.signals import post_save, post_delete + + +INDEX_NAME = "catalog" +SEARCHABLE_ATTRIBUTES = [ + "title", + "orig_title", + "other_title", + "subtitle", + "artist", + "author", + "translator", + "developer", + "director", + "actor", + "playwright", + "pub_house", + "company", + "publisher", + "isbn", + "imdb_code", +] +FILTERABLE_ATTRIBUTES = ["category", "tags", "class_name"] +INDEXABLE_DIRECT_TYPES = [ + "BigAutoField", + "BooleanField", + "CharField", + "PositiveIntegerField", + "PositiveSmallIntegerField", + "TextField", + "ArrayField", +] +INDEXABLE_TIME_TYPES = ["DateTimeField"] +INDEXABLE_DICT_TYPES = ["JSONField"] +INDEXABLE_FLOAT_TYPES = ["DecimalField"] +SORTING_ATTRIBUTE = None +# NONINDEXABLE_TYPES = ['ForeignKey', 'FileField',] +SEARCH_PAGE_SIZE = 20 + + +logger = logging.getLogger(__name__) + + +def item_post_save_handler(sender, instance, created, **kwargs): + if not created and settings.SEARCH_INDEX_NEW_ONLY: + return + Indexer.replace_item(instance) + + +def item_post_delete_handler(sender, instance, **kwargs): + Indexer.delete_item(instance) + + +def tag_post_save_handler(sender, instance, **kwargs): + pass + + +def tag_post_delete_handler(sender, instance, **kwargs): + pass + + +class Indexer: + class_map = {} + _instance = None + + @classmethod + def instance(cls): + if cls._instance is None: + cls._instance = typesense.Client(settings.TYPESENSE_CONNECTION) + return cls._instance + + @classmethod + def init(cls): + # cls.instance().collections[INDEX_NAME].delete() + # fields = [ + # {"name": "_class", "type": "string", "facet": True}, + # {"name": "source_site", "type": "string", "facet": True}, + # {"name": ".*", "type": "auto", "locale": "zh"}, + # ] + # use dumb schema below before typesense fix a bug + fields = [ + {"name": "id", "type": "string"}, + {"name": "category", "type": "string", "facet": True}, + {"name": "class_name", "type": "string", "facet": True}, + {"name": "rating_count", "optional": True, "type": "int32", "facet": True}, + {"name": "isbn", "optional": True, "type": "string"}, + {"name": "imdb_code", "optional": True, "type": "string"}, + {"name": "author", "optional": True, "locale": "zh", "type": "string[]"}, + {"name": "orig_title", "optional": True, "locale": "zh", "type": "string"}, + {"name": "pub_house", "optional": True, "locale": "zh", "type": "string"}, + {"name": "title", "optional": True, "locale": "zh", "type": "string"}, + { + "name": "translator", + "optional": True, + "locale": "zh", + "type": "string[]", + }, + {"name": "subtitle", "optional": True, "locale": "zh", "type": "string"}, + {"name": "artist", "optional": True, "locale": "zh", "type": "string[]"}, + {"name": "company", "optional": True, "locale": "zh", "type": "string[]"}, + {"name": "developer", "optional": True, "locale": "zh", "type": "string[]"}, + { + "name": "other_title", + "optional": True, + "locale": "zh", + "type": "string[]", + }, + {"name": "publisher", "optional": True, "locale": "zh", "type": "string[]"}, + {"name": "actor", "optional": True, "locale": "zh", "type": "string[]"}, + {"name": "director", "optional": True, "locale": "zh", "type": "string[]"}, + { + "name": "playwright", + "optional": True, + "locale": "zh", + "type": "string[]", + }, + {"name": "tags", "optional": True, "locale": "zh", "type": "string[]"}, + {"name": ".*", "optional": True, "locale": "zh", "type": "auto"}, + ] + + cls.instance().collections.create({"name": INDEX_NAME, "fields": fields}) + + @classmethod + def update_settings(cls): + # https://github.com/typesense/typesense/issues/96 + # FIXME + pass + + @classmethod + def get_stats(cls): + return cls.instance().collections[INDEX_NAME].retrieve() + + @classmethod + def busy(cls): + return False + + @classmethod + def update_model_indexable(cls, model): + cls.class_map[model.__name__.lower()] = model + model.indexable_fields = ["tags"] + model.indexable_fields_time = [] + model.indexable_fields_dict = [] + model.indexable_fields_float = [] + for field in model._meta.get_fields(): + type = field.get_internal_type() + if type in INDEXABLE_DIRECT_TYPES: + model.indexable_fields.append(field.name) + elif type in INDEXABLE_TIME_TYPES: + model.indexable_fields_time.append(field.name) + elif type in INDEXABLE_DICT_TYPES: + model.indexable_fields_dict.append(field.name) + elif type in INDEXABLE_FLOAT_TYPES: + model.indexable_fields_float.append(field.name) + post_save.connect(item_post_save_handler, sender=model) + post_delete.connect(item_post_delete_handler, sender=model) + + @classmethod + def obj_to_dict(cls, obj): + item = {} + for field in obj.__class__.indexable_fields: + item[field] = getattr(obj, field) + for field in obj.__class__.indexable_fields_time: + item[field] = getattr(obj, field).timestamp() + for field in obj.__class__.indexable_fields_float: + item[field] = float(getattr(obj, field)) if getattr(obj, field) else None + for field in obj.__class__.indexable_fields_dict: + d = getattr(obj, field) + if d.__class__ is dict: + item.update(d) + + item["id"] = obj.uuid + item["category"] = obj.category + item["class_name"] = obj.class_name + item = { + k: v + for k, v in item.items() + if v + and (k in SEARCHABLE_ATTRIBUTES or k in FILTERABLE_ATTRIBUTES or k == "id") + } + # typesense requires primary key to be named 'id', type string + item["rating_count"] = obj.rating_count + + return item + + @classmethod + def replace_item(cls, obj): + try: + cls.instance().collections[INDEX_NAME].documents.upsert( + cls.obj_to_dict(obj), {"dirty_values": "coerce_or_drop"} + ) + except Exception as e: + logger.error(f"replace item error: \n{e}") + + @classmethod + def replace_batch(cls, objects): + try: + cls.instance().collections[INDEX_NAME].documents.import_( + objects, {"action": "upsert"} + ) + except Exception as e: + logger.error(f"replace batch error: \n{e}") + + @classmethod + def delete_item(cls, obj): + pk = f"{obj.__class__.__name__}-{obj.id}" + try: + cls.instance().collections[INDEX_NAME].documents[pk].delete() + except Exception as e: + logger.error(f"delete item error: \n{e}") + + @classmethod + def search(cls, q, page=1, category=None, tag=None, sort=None): + f = [] + if category: + f.append("category:= " + category) + if tag: + f.append(f"tags:= '{tag}'") + filters = " && ".join(f) + options = { + "q": q, + "page": page, + "per_page": SEARCH_PAGE_SIZE, + "query_by": ",".join(SEARCHABLE_ATTRIBUTES), + "filter_by": filters, + # 'facetsDistribution': ['_class'], + # 'sort_by': None, + } + results = types.SimpleNamespace() + + try: + r = cls.instance().collections[INDEX_NAME].documents.search(options) + results.items = list( + [ + x + for x in map(lambda i: cls.item_to_obj(i["document"]), r["hits"]) + if x is not None + ] + ) + results.num_pages = (r["found"] + SEARCH_PAGE_SIZE - 1) // SEARCH_PAGE_SIZE + except ObjectNotFound: + results.items = [] + results.num_pages = 1 + + return results + + @classmethod + def item_to_obj(cls, item): + try: + return cls.class_map[item["class_name"]].get_by_url(item["id"]) + except Exception as e: + logger.error(f"unable to load search result item from db:\n{item}") + return None diff --git a/catalog/templates/search_results.html b/catalog/templates/search_results.html new file mode 100644 index 00000000..1db59beb --- /dev/null +++ b/catalog/templates/search_results.html @@ -0,0 +1,245 @@ +{% load static %} +{% load i18n %} +{% load l10n %} +{% load humanize %} +{% load admin_url %} +{% load mastodon %} +{% load oauth_token %} +{% load truncate %} +{% load highlight %} +{% load thumb %} + + + + + + + {{ site_name }} - {% trans '搜索结果' %} + + + + + + + + + + +
+
+ {% include 'partial/_navbar.html' %} + +
+
+
+
+ +
+ {% if request.GET.q %} +
“{{ request.GET.q }}” {% trans '的搜索结果' %}
+ {% endif %} + + {% if request.GET.tag %} +
{% trans '含有标签' %} “{{ request.GET.tag }}” {% trans '的结果' %}
+ {% endif %} + +
    + {% for item in items %} + {% with "list_item_"|add:item.class_name|add:".html" as template %} + {% include template %} + {% endwith %} + {% empty %} +
  • + {% trans '无站内条目匹配' %} +
    • + {% endfor %} + {% if request.GET.q and user.is_authenticated %} +
  • + {% trans '正在实时搜索站外条目' %} +
    + +
    +
    • + {% endif %} +
+
+ +
+ +
+ +
+
+ +
+
+
+ {% trans '没有想要的结果？' %} +
+ {% if request.GET.c and request.GET.c in categories %} + + {% if request.GET.c|lower == 'book' %} + + + + + + {% elif request.GET.c|lower == 'movie' %} + + + + + + {% elif request.GET.c|lower == 'music' %} + + + + + + + + + {% elif request.GET.c|lower == 'game' %} + + + + + + {% endif %} + + {% else %} + + + + + + + + + + + + + + + + {% endif %} +
+ + +
+ +
+
+
+
+
+ {% include 'partial/_footer.html' %} +
+ + + + + + + + diff --git a/catalog/views.py b/catalog/views.py index c5bf44c5..b4e2deea 100644 --- a/catalog/views.py +++ b/catalog/views.py @@ -167,8 +167,6 @@ def fetch_task(url): def fetch_refresh(request, job_id): retry = request.GET job = Job.fetch(id=job_id, connection=django_rq.get_connection("fetch")) - print(job_id) - print(job) item_url = job.result if job else "-" # FIXME job.return_value() in rq 1.12 if item_url: if item_url == "-": @@ -250,9 +248,8 @@ def search(request): elif key not in keys: keys.append(key) items.append(i) - urls.append(i.source_url) - i.tag_list = i.all_tag_list[:TAG_NUMBER_ON_LIST] - + for res in i.external_resources.all(): + urls.append(res.url) if request.path.endswith(".json/"): return JsonResponse( { @@ -260,11 +257,10 @@ def search(request): "items": list(map(lambda i: i.get_json(), items)), } ) - request.session["search_dedupe_urls"] = urls return render( request, - "common/search_result.html", + "search_results.html", { "items": items, "pagination": PageLinksGenerator( diff --git a/common/views.py b/common/views.py index cafce536..bd4f1bb7 100644 --- a/common/views.py +++ b/common/views.py @@ -43,13 +43,13 @@ def home(request): @login_required def external_search(request): - category = request.GET.get("c", default='all').strip().lower() - if category == 'all': + category = request.GET.get("c", default="all").strip().lower() + if category == "all": category = None - keywords = request.GET.get("q", default='').strip() - page_number = int(request.GET.get('page', default=1)) + keywords = request.GET.get("q", default="").strip() + page_number = int(request.GET.get("page", default=1)) items = ExternalSources.search(category, keywords, page_number) if keywords else [] - dedupe_urls = request.session.get('search_dedupe_urls', []) + dedupe_urls = request.session.get("search_dedupe_urls", []) items = [i for i in items if i.source_url not in dedupe_urls] return render( @@ -57,19 +57,24 @@ def external_search(request): "common/external_search_result.html", { "external_items": items, - } + }, ) def search(request): + if settings.ENABLE_NEW_MODEL: + from catalog.views import search as new_search + + return new_search(request) + if settings.SEARCH_BACKEND is None: return search2(request) - category = request.GET.get("c", default='all').strip().lower() - if category == 'all': + category = request.GET.get("c", default="all").strip().lower() + if category == "all": category = None - keywords = request.GET.get("q", default='').strip() - tag = request.GET.get("tag", default='').strip() - p = request.GET.get('page', default='1') + keywords = request.GET.get("q", default="").strip() + tag = request.GET.get("tag", default="").strip() + p = request.GET.get("page", default="1") page_number = int(p) if p.isdigit() else 1 if not (keywords or tag): return render( @@ -77,7 +82,7 @@ def search(request): "common/search_result.html", { "items": None, - } + }, ) if request.user.is_authenticated: url_validator = URLValidator() @@ -87,13 +92,17 @@ def search(request): return jump_or_scrape(request, keywords) except ValidationError as e: pass - + result = Indexer.search(keywords, page=page_number, category=category, tag=tag) keys = [] items = [] urls = [] for i in result.items: - key = i.isbn if hasattr(i, 'isbn') else (i.imdb_code if hasattr(i, 'imdb_code') else None) + key = ( + i.isbn + if hasattr(i, "isbn") + else (i.imdb_code if hasattr(i, "imdb_code") else None) + ) if key is None: items.append(i) elif key not in keys: @@ -102,35 +111,39 @@ def search(request): urls.append(i.source_url) i.tag_list = i.all_tag_list[:TAG_NUMBER_ON_LIST] - if request.path.endswith('.json/'): - return JsonResponse({ - 'num_pages': result.num_pages, - 'items':list(map(lambda i:i.get_json(), items)) - }) + if request.path.endswith(".json/"): + return JsonResponse( + { + "num_pages": result.num_pages, + "items": list(map(lambda i: i.get_json(), items)), + } + ) - request.session['search_dedupe_urls'] = urls + request.session["search_dedupe_urls"] = urls return render( request, "common/search_result.html", { "items": items, - "pagination": PageLinksGenerator(PAGE_LINK_NUMBER, page_number, result.num_pages), - "categories": ['book', 'movie', 'music', 'game'], - } + "pagination": PageLinksGenerator( + PAGE_LINK_NUMBER, page_number, result.num_pages + ), + "categories": ["book", "movie", "music", "game"], + }, ) def search2(request): - if request.method == 'GET': + if request.method == "GET": # test if input serach string is empty or not excluding param ?c= - empty_querystring_criteria = {k: v for k, v in request.GET.items() if k != 'c'} + empty_querystring_criteria = {k: v for k, v in request.GET.items() if k != "c"} if not len(empty_querystring_criteria): return HttpResponseBadRequest() # test if user input an URL, if so jump to URL handling function url_validator = URLValidator() - input_string = request.GET.get('q', default='').strip() + input_string = request.GET.get("q", default="").strip() try: url_validator(input_string) # validation success @@ -139,13 +152,13 @@ def search2(request): pass # category, book/movie/music etc - category = request.GET.get("c", default='').strip().lower() + category = request.GET.get("c", default="").strip().lower() # keywords, seperated by blank space # it is better not to split the keywords - keywords = request.GET.get("q", default='').strip() - keywords = [keywords] if keywords else '' + keywords = request.GET.get("q", default="").strip() + keywords = [keywords] if keywords else "" # tag, when tag is provided there should be no keywords , for now - tag = request.GET.get("tag", default='') + tag = request.GET.get("tag", default="") # white space string, empty query if not (keywords or tag): @@ -154,14 +167,14 @@ def search2(request): "common/search_result.html", { "items": None, - } + }, ) def book_param_handler(**kwargs): # keywords - keywords = kwargs.get('keywords') + keywords = kwargs.get("keywords") # tag - tag = kwargs.get('tag') + tag = kwargs.get("tag") query_args = [] q = Q() @@ -181,29 +194,42 @@ def search2(request): # search by keywords similarity, n = 0, 0 for keyword in keywords: - similarity += 1/2 * SequenceMatcher(None, keyword, book.title).quick_ratio() - + 1/3 * SequenceMatcher(None, keyword, book.orig_title).quick_ratio() - + 1/6 * SequenceMatcher(None, keyword, book.subtitle).quick_ratio() + similarity += ( + 1 + / 2 + * SequenceMatcher(None, keyword, book.title).quick_ratio() + ) + +1 / 3 * SequenceMatcher( + None, keyword, book.orig_title + ).quick_ratio() + +1 / 6 * SequenceMatcher( + None, keyword, book.subtitle + ).quick_ratio() n += 1 book.similarity = similarity / n elif tag: # search by single tag - book.similarity = 0 if book.rating_number is None else book.rating_number + book.similarity = ( + 0 if book.rating_number is None else book.rating_number + ) else: book.similarity = 0 return book.similarity + if len(queryset) > 0: - ordered_queryset = sorted(queryset, key=calculate_similarity, reverse=True) + ordered_queryset = sorted( + queryset, key=calculate_similarity, reverse=True + ) else: ordered_queryset = list(queryset) return ordered_queryset def movie_param_handler(**kwargs): # keywords - keywords = kwargs.get('keywords') + keywords = kwargs.get("keywords") # tag - tag = kwargs.get('tag') + tag = kwargs.get("tag") query_args = [] q = Q() @@ -223,28 +249,41 @@ def search2(request): # search by name similarity, n = 0, 0 for keyword in keywords: - similarity += 1/2 * SequenceMatcher(None, keyword, movie.title).quick_ratio() - + 1/4 * SequenceMatcher(None, keyword, movie.orig_title).quick_ratio() - + 1/4 * SequenceMatcher(None, keyword, movie.other_title).quick_ratio() + similarity += ( + 1 + / 2 + * SequenceMatcher(None, keyword, movie.title).quick_ratio() + ) + +1 / 4 * SequenceMatcher( + None, keyword, movie.orig_title + ).quick_ratio() + +1 / 4 * SequenceMatcher( + None, keyword, movie.other_title + ).quick_ratio() n += 1 movie.similarity = similarity / n elif tag: # search by single tag - movie.similarity = 0 if movie.rating_number is None else movie.rating_number + movie.similarity = ( + 0 if movie.rating_number is None else movie.rating_number + ) else: movie.similarity = 0 return movie.similarity + if len(queryset) > 0: - ordered_queryset = sorted(queryset, key=calculate_similarity, reverse=True) + ordered_queryset = sorted( + queryset, key=calculate_similarity, reverse=True + ) else: ordered_queryset = list(queryset) return ordered_queryset def game_param_handler(**kwargs): # keywords - keywords = kwargs.get('keywords') + keywords = kwargs.get("keywords") # tag - tag = kwargs.get('tag') + tag = kwargs.get("tag") query_args = [] q = Q() @@ -263,33 +302,48 @@ def search2(request): def calculate_similarity(game): if keywords: # search by name - developer_dump = ' '.join(game.developer) - publisher_dump = ' '.join(game.publisher) + developer_dump = " ".join(game.developer) + publisher_dump = " ".join(game.publisher) similarity, n = 0, 0 for keyword in keywords: - similarity += 1/2 * SequenceMatcher(None, keyword, game.title).quick_ratio() - + 1/4 * SequenceMatcher(None, keyword, game.other_title).quick_ratio() - + 1/16 * SequenceMatcher(None, keyword, developer_dump).quick_ratio() - + 1/16 * SequenceMatcher(None, keyword, publisher_dump).quick_ratio() + similarity += ( + 1 + / 2 + * SequenceMatcher(None, keyword, game.title).quick_ratio() + ) + +1 / 4 * SequenceMatcher( + None, keyword, game.other_title + ).quick_ratio() + +1 / 16 * SequenceMatcher( + None, keyword, developer_dump + ).quick_ratio() + +1 / 16 * SequenceMatcher( + None, keyword, publisher_dump + ).quick_ratio() n += 1 game.similarity = similarity / n elif tag: # search by single tag - game.similarity = 0 if game.rating_number is None else game.rating_number + game.similarity = ( + 0 if game.rating_number is None else game.rating_number + ) else: game.similarity = 0 return game.similarity + if len(queryset) > 0: - ordered_queryset = sorted(queryset, key=calculate_similarity, reverse=True) + ordered_queryset = sorted( + queryset, key=calculate_similarity, reverse=True + ) else: ordered_queryset = list(queryset) return ordered_queryset def music_param_handler(**kwargs): # keywords - keywords = kwargs.get('keywords') + keywords = kwargs.get("keywords") # tag - tag = kwargs.get('tag') + tag = kwargs.get("tag") query_args = [] q = Q() @@ -321,25 +375,58 @@ def search2(request): if keywords: # search by name similarity, n = 0, 0 - artist_dump = ' '.join(music.artist) + artist_dump = " ".join(music.artist) for keyword in keywords: if music.__class__ == Album: - similarity += 1/2 * SequenceMatcher(None, keyword, music.title).quick_ratio() \ - + 1/2 * SequenceMatcher(None, keyword, artist_dump).quick_ratio() + similarity += ( + 1 + / 2 + * SequenceMatcher( + None, keyword, music.title + ).quick_ratio() + + 1 + / 2 + * SequenceMatcher( + None, keyword, artist_dump + ).quick_ratio() + ) elif music.__class__ == Song: - similarity += 1/2 * SequenceMatcher(None, keyword, music.title).quick_ratio() \ - + 1/6 * SequenceMatcher(None, keyword, artist_dump).quick_ratio() \ - + 1/6 * (SequenceMatcher(None, keyword, music.album.title).quick_ratio() if music.album is not None else 0) + similarity += ( + 1 + / 2 + * SequenceMatcher( + None, keyword, music.title + ).quick_ratio() + + 1 + / 6 + * SequenceMatcher( + None, keyword, artist_dump + ).quick_ratio() + + 1 + / 6 + * ( + SequenceMatcher( + None, keyword, music.album.title + ).quick_ratio() + if music.album is not None + else 0 + ) + ) n += 1 music.similarity = similarity / n elif tag: # search by single tag - music.similarity = 0 if music.rating_number is None else music.rating_number + music.similarity = ( + 0 if music.rating_number is None else music.rating_number + ) else: music.similarity = 0 return music.similarity + if len(queryset) > 0: - ordered_queryset = sorted(queryset, key=calculate_similarity, reverse=True) + ordered_queryset = sorted( + queryset, key=calculate_similarity, reverse=True + ) else: ordered_queryset = list(queryset) return ordered_queryset @@ -350,40 +437,40 @@ def search2(request): music_queryset = music_param_handler(**kwargs) game_queryset = game_param_handler(**kwargs) ordered_queryset = sorted( - book_queryset + movie_queryset + music_queryset + game_queryset, - key=operator.attrgetter('similarity'), - reverse=True + book_queryset + movie_queryset + music_queryset + game_queryset, + key=operator.attrgetter("similarity"), + reverse=True, ) return ordered_queryset param_handler = { - 'book': book_param_handler, - 'movie': movie_param_handler, - 'music': music_param_handler, - 'game': game_param_handler, - 'all': all_param_handler, - '': all_param_handler + "book": book_param_handler, + "movie": movie_param_handler, + "music": music_param_handler, + "game": game_param_handler, + "all": all_param_handler, + "": all_param_handler, } - categories = [k for k in param_handler.keys() if not k in ['all', '']] + categories = [k for k in param_handler.keys() if not k in ["all", ""]] try: - queryset = param_handler[category]( - keywords=keywords, - tag=tag - ) + queryset = param_handler[category](keywords=keywords, tag=tag) except KeyError as e: - queryset = param_handler['all']( - keywords=keywords, - tag=tag - ) + queryset = param_handler["all"](keywords=keywords, tag=tag) paginator = Paginator(queryset, ITEMS_PER_PAGE) - page_number = request.GET.get('page', default=1) + page_number = request.GET.get("page", default=1) items = paginator.get_page(page_number) - items.pagination = PageLinksGenerator(PAGE_LINK_NUMBER, page_number, paginator.num_pages) + items.pagination = PageLinksGenerator( + PAGE_LINK_NUMBER, page_number, paginator.num_pages + ) for item in items: - item.tag_list = item.get_tags_manager().values('content').annotate( - tag_frequency=Count('content')).order_by('-tag_frequency')[:TAG_NUMBER_ON_LIST] + item.tag_list = ( + item.get_tags_manager() + .values("content") + .annotate(tag_frequency=Count("content")) + .order_by("-tag_frequency")[:TAG_NUMBER_ON_LIST] + ) return render( request, @@ -391,7 +478,7 @@ def search2(request): { "items": items, "categories": categories, - } + }, ) else: @@ -415,21 +502,18 @@ def jump_or_scrape(request, url): scraper = get_scraper_by_url(url) if scraper is None: # invalid url - return render(request, 'common/error.html', {'msg': _("链接无效，查询失败")}) + return render(request, "common/error.html", {"msg": _("链接无效，查询失败")}) else: try: effective_url = scraper.get_effective_url(url) except ValueError: - return render(request, 'common/error.html', {'msg': _("链接无效，查询失败")}) + return render(request, "common/error.html", {"msg": _("链接无效，查询失败")}) try: # raise ObjectDoesNotExist entity = scraper.data_class.objects.get(source_url=effective_url) # if exists then jump to detail page - if request.path.endswith('.json/'): - return JsonResponse({ - 'num_pages': 1, - 'items': [entity.get_json()] - }) + if request.path.endswith(".json/"): + return JsonResponse({"num_pages": 1, "items": [entity.get_json()]}) return redirect(entity) except ObjectDoesNotExist: # scrape if not exists @@ -443,18 +527,27 @@ def jump_or_scrape(request, url): except Exception as e: logger.error(f"Scrape Failed URL: {url}\n{e}") if settings.DEBUG: - logger.error("Expections during saving scraped data:", exc_info=e) - return render(request, 'common/error.html', {'msg': _("爬取数据失败😫")}) + logger.error( + "Expections during saving scraped data:", exc_info=e + ) + return render(request, "common/error.html", {"msg": _("爬取数据失败😫")}) except Exception as e: logger.error(f"Scrape Failed URL: {url}\n{e}") if settings.DEBUG: logger.error("Expections during saving scraped data:", exc_info=e) - return render(request, 'common/error.html', {'msg': _("爬取数据失败😫")}) + return render(request, "common/error.html", {"msg": _("爬取数据失败😫")}) return redirect(form.instance) def go_relogin(request): - return render(request, 'common/error.html', { - 'url': reverse("users:connect") + '?domain=' + request.user.mastodon_site, - 'msg': _("信息已保存，但是未能分享到联邦网络"), - 'secondary_msg': _("可能是你在联邦网络(Mastodon/Pleroma/...)的登录状态过期了，正在跳转到联邦网络重新登录😼")}) + return render( + request, + "common/error.html", + { + "url": reverse("users:connect") + "?domain=" + request.user.mastodon_site, + "msg": _("信息已保存，但是未能分享到联邦网络"), + "secondary_msg": _( + "可能是你在联邦网络(Mastodon/Pleroma/...)的登录状态过期了，正在跳转到联邦网络重新登录😼" + ), + }, + )