From 6f58627d9f176c0c95a2447df7bb9655de5fbccd Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 31 Dec 2022 11:06:23 -0500 Subject: [PATCH] search no longer block --- catalog/common/sites.py | 8 +- catalog/templates/fetch_failed.html | 3 + catalog/templates/fetch_pending.html | 82 +++++++++++++++ catalog/templates/fetch_refresh.html | 1 + catalog/urls.py | 2 + catalog/views.py | 143 ++++++++++++++++++++++++++- 6 files changed, 233 insertions(+), 6 deletions(-) create mode 100644 catalog/templates/fetch_failed.html create mode 100644 catalog/templates/fetch_pending.html create mode 100644 catalog/templates/fetch_refresh.html diff --git a/catalog/common/sites.py b/catalog/common/sites.py index 08f6d545..0b33f0ee 100644 --- a/catalog/common/sites.py +++ b/catalog/common/sites.py @@ -21,8 +21,8 @@ _logger = logging.getLogger(__name__) class ResourceContent: lookup_ids: dict = field(default_factory=dict) metadata: dict = field(default_factory=dict) - cover_image: bytes | None = None - cover_image_extention: str | None = None + cover_image: bytes = None + cover_image_extention: str = None def dict(self): return {"metadata": self.metadata, "lookup_ids": self.lookup_ids} @@ -122,7 +122,7 @@ class AbstractSite: auto_link=True, preloaded_content=None, ignore_existing_content=False, - ) -> ExternalResource | None: + ) -> ExternalResource: """ Returns an ExternalResource in scraped state if possible @@ -196,7 +196,7 @@ class SiteManager: return SiteManager.registry[typ]() if typ in SiteManager.registry else None @staticmethod - def get_site_by_url(url: str) -> AbstractSite | None: + def get_site_by_url(url: str) -> AbstractSite: if not url: return None cls = next( diff --git a/catalog/templates/fetch_failed.html b/catalog/templates/fetch_failed.html new file mode 100644 index 00000000..d88da088 --- /dev/null +++ b/catalog/templates/fetch_failed.html @@ -0,0 +1,3 @@ +

+ 无法加载条目。部分网站可能删除条目或隐藏条目为仅登录可见，欢迎在本站手工添加这些条目。 +

\ No newline at end of file diff --git a/catalog/templates/fetch_pending.html b/catalog/templates/fetch_pending.html new file mode 100644 index 00000000..8376c6e3 --- /dev/null +++ b/catalog/templates/fetch_pending.html @@ -0,0 +1,82 @@ +{% load static %} +{% load i18n %} +{% load l10n %} +{% load humanize %} +{% load admin_url %} +{% load mastodon %} +{% load oauth_token %} +{% load truncate %} +{% load highlight %} +{% load thumb %} + + + + + + + {{ site_name }} - {% trans '搜索结果' %} + + + + + + + + + + +
+
+ {% include 'partial/_navbar.html' %} + +
+
+
+
+
+ {% trans '正在连线' %}{{ site.SITE_NAME }} +
+
+ +
+
+ +
+ +
+ +
+
+ +
+
+
+
+
+ {% include 'partial/_footer.html' %} +
+ + + + + + + + diff --git a/catalog/templates/fetch_refresh.html b/catalog/templates/fetch_refresh.html new file mode 100644 index 00000000..dec5c305 --- /dev/null +++ b/catalog/templates/fetch_refresh.html @@ -0,0 +1 @@ +
\ No newline at end of file diff --git a/catalog/urls.py b/catalog/urls.py index 66dde9c2..4001c66a 100644 --- a/catalog/urls.py +++ b/catalog/urls.py @@ -43,5 +43,7 @@ urlpatterns = [ mark_list, name="mark_list", ), + path("search2/", search, name="search"), + path("fetch_refresh/", fetch_refresh, name="fetch_refresh"), path("api/", api.urls), ] diff --git a/catalog/views.py b/catalog/views.py index 37fb0882..c5bf44c5 100644 --- a/catalog/views.py +++ b/catalog/views.py @@ -1,13 +1,21 @@ +import uuid import logging from django.shortcuts import render, get_object_or_404, redirect, reverse from django.contrib.auth.decorators import login_required, permission_required from django.utils.translation import gettext_lazy as _ -from django.http import HttpResponseBadRequest, HttpResponseServerError, HttpResponse -from django.core.exceptions import ObjectDoesNotExist, PermissionDenied +from django.http import ( + HttpResponseBadRequest, + HttpResponseServerError, + HttpResponse, + HttpResponseRedirect, +) +from django.core.exceptions import BadRequest, ObjectDoesNotExist, PermissionDenied from django.db import IntegrityError, transaction from django.db.models import Count from django.utils import timezone from django.core.paginator import Paginator +from polymorphic.base import django +from catalog.common.sites import AbstractSite, SiteManager from mastodon import mastodon_request_included from mastodon.models import MastodonApplication from mastodon.api import share_mark, share_review @@ -20,6 +28,8 @@ from journal.models import query_visible, query_following from common.utils import PageLinksGenerator from common.views import PAGE_LINK_NUMBER from journal.models import ShelfTypeNames +import django_rq +from rq.job import Job _logger = logging.getLogger(__name__) @@ -28,6 +38,14 @@ NUM_REVIEWS_ON_ITEM_PAGE = 5 NUM_REVIEWS_ON_LIST_PAGE = 20 +class HTTPResponseHXRedirect(HttpResponseRedirect): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self["HX-Redirect"] = self["Location"] + + status_code = 200 + + def retrieve_by_uuid(request, item_uid): item = get_object_or_404(Item, uid=item_uid) return redirect(item.url) @@ -134,3 +152,124 @@ def review_list(request, item_path, item_uuid): "item": item, }, ) + + +def fetch_task(url): + try: + site = SiteManager.get_site_by_url(url) + site.get_resource_ready() + item = site.get_item() + return item.url if item else "-" + except Exception: + return "-" + + +def fetch_refresh(request, job_id): + retry = request.GET + job = Job.fetch(id=job_id, connection=django_rq.get_connection("fetch")) + print(job_id) + print(job) + item_url = job.result if job else "-" # FIXME job.return_value() in rq 1.12 + if item_url: + if item_url == "-": + return render(request, "fetch_failed.html") + else: + return HTTPResponseHXRedirect(item_url) + else: + retry = int(request.GET.get("retry", 0)) + 1 + if retry > 10: + return render(request, "fetch_failed.html") + else: + return render( + request, + "fetch_refresh.html", + {"job_id": job_id, "retry": retry, "delay": retry * 2}, + ) + + +def fetch(request, url, site: AbstractSite = None): + if not site: + site = SiteManager.get_site_by_url(keywords) + if not site: + return HttpResponseBadRequest() + item = site.get_item() + if item: + return redirect(item.url) + job_id = uuid.uuid4().hex + django_rq.get_queue("fetch").enqueue(fetch_task, url, job_id=job_id) + return render( + request, + "fetch_pending.html", + { + "site": site, + "job_id": job_id, + }, + ) + + +def search(request): + category = request.GET.get("c", default="all").strip().lower() + if category == "all": + category = None + keywords = request.GET.get("q", default="").strip() + tag = request.GET.get("tag", default="").strip() + p = request.GET.get("page", default="1") + page_number = int(p) if p.isdigit() else 1 + if not (keywords or tag): + return render( + request, + "common/search_result.html", + { + "items": None, + }, + ) + + if request.user.is_authenticated and keywords.find("://") > 0: + site = SiteManager.get_site_by_url(keywords) + if site: + return fetch(request, keywords, site) + if settings.SEARCH_BACKEND is None: + # return limited results if no SEARCH_BACKEND + result = { + "items": Items.objects.filter(title__like=f"%{keywords}%")[:10], + "num_pages": 1, + } + else: + result = Indexer.search(keywords, page=page_number, category=category, tag=tag) + keys = [] + items = [] + urls = [] + for i in result.items: + key = ( + i.isbn + if hasattr(i, "isbn") + else (i.imdb_code if hasattr(i, "imdb_code") else None) + ) + if key is None: + items.append(i) + elif key not in keys: + keys.append(key) + items.append(i) + urls.append(i.source_url) + i.tag_list = i.all_tag_list[:TAG_NUMBER_ON_LIST] + + if request.path.endswith(".json/"): + return JsonResponse( + { + "num_pages": result.num_pages, + "items": list(map(lambda i: i.get_json(), items)), + } + ) + + request.session["search_dedupe_urls"] = urls + return render( + request, + "common/search_result.html", + { + "items": items, + "pagination": PageLinksGenerator( + PAGE_LINK_NUMBER, page_number, result.num_pages + ), + "categories": ["book", "movie", "music", "game"], + }, + )