queue api fetch

catalog/api.py

@@ -3,32 +3,33 @@ from .common import *
 from .sites import *
 from ninja import Schema
 from django.http import Http404
-from common.api import api
+from common.api import api, Result
+from .search.views import enqueue_fetch
 
 
 class SearchResult(Schema):
-    code: int
     items: list[ItemSchema]
 
 
-@api.post("/catalog/search", response=SearchResult)
+@api.post("/catalog/search", response={200: SearchResult, 400: Result})
 def search_item(request, query: str, category: ItemCategory | None = None):
     query = query.strip()
     if not query:
-        return {"code": -1, "items": []}
+        return 200, {"message": "Invalid query"}
     result = Indexer.search(query, page=1, category=category)
-    return {"code": 0, "items": result.items}
+    return 200, {"items": result.items}
 
 
-@api.post("/catalog/fetch", response=ItemSchema)
+@api.post("/catalog/fetch", response={200: ItemSchema, 202: Result})
 def fetch_item(request, url: str):
     site = SiteManager.get_site_by_url(url)
     if not site:
-        return Http404()
-    resource = site.get_resource_ready()
-    if not resource:
-        return Http404()
-    return site.get_item()
+        raise Http404(url)
+    item = site.get_item()
+    if item:
+        return 200, item
+    enqueue_fetch(url, False)
+    return 202, {"message": "Fetch in progress"}
 
 
 @api.get("/book/{uuid}/", response=EditionSchema)

catalog/book/models.py

@@ -41,6 +41,7 @@ class EditionInSchema(ItemInSchema):
 
 
 class EditionSchema(EditionInSchema, BaseSchema):
+    isbn: str | None = None
     pass
 
 

catalog/movie/models.py

@@ -18,6 +18,7 @@ class MovieInSchema(ItemInSchema):
 
 
 class MovieSchema(MovieInSchema, BaseSchema):
+    imdb: str | None = None
     pass
 
 

catalog/music/models.py

@@ -15,6 +15,7 @@ class AlbumInSchema(ItemInSchema):
 
 
 class AlbumSchema(AlbumInSchema, BaseSchema):
+    barcode: str | None = None
     pass
 
 

catalog/search/views.py

@@ -15,6 +15,7 @@ import django_rq
 from rq.job import Job
 from .external import ExternalSources
 from django.core.cache import cache
+import hashlib
 
 _logger = logging.getLogger(__name__)
 
@@ -52,6 +53,19 @@ def fetch_refresh(request, job_id):
             )
 
 
+def enqueue_fetch(url, is_refetch):
+    job_id = "fetch_" + hashlib.md5(url.encode()).hexdigest()
+    in_progress = False
+    try:
+        job = Job.fetch(id=job_id, connection=django_rq.get_connection("fetch"))
+        in_progress = job.get_status() in ["queued", "started"]
+    except:
+        in_progress = False
+    if not in_progress:
+        django_rq.get_queue("fetch").enqueue(fetch_task, url, is_refetch, job_id=job_id)
+    return job_id
+
+
 def fetch(request, url, is_refetch: bool = False, site: AbstractSite = None):
     if not site:
         site = SiteManager.get_site_by_url(url)
@@ -60,8 +74,7 @@ def fetch(request, url, is_refetch: bool = False, site: AbstractSite = None):
     item = site.get_item()
     if item and not is_refetch:
         return redirect(item.url)
-    job_id = uuid.uuid4().hex
-    django_rq.get_queue("fetch").enqueue(fetch_task, url, is_refetch, job_id=job_id)
+    job_id = enqueue_fetch(url, is_refetch)
     return render(
         request,
         "fetch_pending.html",

catalog/tv/models.py

@@ -47,6 +47,7 @@ class TVShowInSchema(ItemInSchema):
 
 
 class TVShowSchema(TVShowInSchema, BaseSchema):
+    imdb: str | None = None
     # seasons: list['TVSeason']
     pass
 

common/api.py

@@ -1,6 +1,11 @@
-from ninja import NinjaAPI
+from ninja import NinjaAPI, Schema
 from django.conf import settings
 
+
+class Result(Schema):
+    message: str
+
+
 api = NinjaAPI(
     title=settings.SITE_INFO["site_name"],
     version="1.0.0",