From c9ea3e92fdda40081a9095a712269133c5429202 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 15 Feb 2023 16:22:32 -0500 Subject: [PATCH] queue api fetch --- catalog/api.py | 23 ++++++++++++----------- catalog/book/models.py | 1 + catalog/movie/models.py | 1 + catalog/music/models.py | 1 + catalog/search/views.py | 17 +++++++++++++++-- catalog/tv/models.py | 1 + common/api.py | 7 ++++++- 7 files changed, 37 insertions(+), 14 deletions(-) diff --git a/catalog/api.py b/catalog/api.py index d6a8c0f6..f9c968da 100644 --- a/catalog/api.py +++ b/catalog/api.py @@ -3,32 +3,33 @@ from .common import * from .sites import * from ninja import Schema from django.http import Http404 -from common.api import api +from common.api import api, Result +from .search.views import enqueue_fetch class SearchResult(Schema): - code: int items: list[ItemSchema] -@api.post("/catalog/search", response=SearchResult) +@api.post("/catalog/search", response={200: SearchResult, 400: Result}) def search_item(request, query: str, category: ItemCategory | None = None): query = query.strip() if not query: - return {"code": -1, "items": []} + return 200, {"message": "Invalid query"} result = Indexer.search(query, page=1, category=category) - return {"code": 0, "items": result.items} + return 200, {"items": result.items} -@api.post("/catalog/fetch", response=ItemSchema) +@api.post("/catalog/fetch", response={200: ItemSchema, 202: Result}) def fetch_item(request, url: str): site = SiteManager.get_site_by_url(url) if not site: - return Http404() - resource = site.get_resource_ready() - if not resource: - return Http404() - return site.get_item() + raise Http404(url) + item = site.get_item() + if item: + return 200, item + enqueue_fetch(url, False) + return 202, {"message": "Fetch in progress"} @api.get("/book/{uuid}/", response=EditionSchema) diff --git a/catalog/book/models.py b/catalog/book/models.py index 0e1c9c6b..416ef5a5 100644 --- a/catalog/book/models.py +++ b/catalog/book/models.py @@ -41,6 +41,7 @@ class EditionInSchema(ItemInSchema): class EditionSchema(EditionInSchema, BaseSchema): + isbn: str | None = None pass diff --git a/catalog/movie/models.py b/catalog/movie/models.py index 399907cf..f02d501b 100644 --- a/catalog/movie/models.py +++ b/catalog/movie/models.py @@ -18,6 +18,7 @@ class MovieInSchema(ItemInSchema): class MovieSchema(MovieInSchema, BaseSchema): + imdb: str | None = None pass diff --git a/catalog/music/models.py b/catalog/music/models.py index 80e435e4..297294d8 100644 --- a/catalog/music/models.py +++ b/catalog/music/models.py @@ -15,6 +15,7 @@ class AlbumInSchema(ItemInSchema): class AlbumSchema(AlbumInSchema, BaseSchema): + barcode: str | None = None pass diff --git a/catalog/search/views.py b/catalog/search/views.py index 2b841b9a..2ee372e1 100644 --- a/catalog/search/views.py +++ b/catalog/search/views.py @@ -15,6 +15,7 @@ import django_rq from rq.job import Job from .external import ExternalSources from django.core.cache import cache +import hashlib _logger = logging.getLogger(__name__) @@ -52,6 +53,19 @@ def fetch_refresh(request, job_id): ) +def enqueue_fetch(url, is_refetch): + job_id = "fetch_" + hashlib.md5(url.encode()).hexdigest() + in_progress = False + try: + job = Job.fetch(id=job_id, connection=django_rq.get_connection("fetch")) + in_progress = job.get_status() in ["queued", "started"] + except: + in_progress = False + if not in_progress: + django_rq.get_queue("fetch").enqueue(fetch_task, url, is_refetch, job_id=job_id) + return job_id + + def fetch(request, url, is_refetch: bool = False, site: AbstractSite = None): if not site: site = SiteManager.get_site_by_url(url) @@ -60,8 +74,7 @@ def fetch(request, url, is_refetch: bool = False, site: AbstractSite = None): item = site.get_item() if item and not is_refetch: return redirect(item.url) - job_id = uuid.uuid4().hex - django_rq.get_queue("fetch").enqueue(fetch_task, url, is_refetch, job_id=job_id) + job_id = enqueue_fetch(url, is_refetch) return render( request, "fetch_pending.html", diff --git a/catalog/tv/models.py b/catalog/tv/models.py index dd59f5a8..5766fdfb 100644 --- a/catalog/tv/models.py +++ b/catalog/tv/models.py @@ -47,6 +47,7 @@ class TVShowInSchema(ItemInSchema): class TVShowSchema(TVShowInSchema, BaseSchema): + imdb: str | None = None # seasons: list['TVSeason'] pass diff --git a/common/api.py b/common/api.py index 653be17e..1a9ee1ae 100644 --- a/common/api.py +++ b/common/api.py @@ -1,6 +1,11 @@ -from ninja import NinjaAPI +from ninja import NinjaAPI, Schema from django.conf import settings + +class Result(Schema): + message: str + + api = NinjaAPI( title=settings.SITE_INFO["site_name"], version="1.0.0",