rate limit fetch

catalog/api.py

@@ -6,7 +6,7 @@ from common.api import *
 from .models import *
 from .common import *
 from .sites import *
-from .search.models import enqueue_fetch, query_index
+from .search.models import enqueue_fetch, query_index, get_fetch_lock
 
 
 class SearchResult(Schema):
@@ -70,7 +70,8 @@ def fetch_item(request, url: str):
     item = site.get_item()
     if item:
         return 200, item
-    enqueue_fetch(url, False)
+    if get_fetch_lock():
+        enqueue_fetch(url, False)
     return 202, {"message": "Fetch in progress"}
 
 

catalog/search/models.py

@@ -9,6 +9,7 @@ from django.core.cache import cache
 import hashlib
 from .typesense import Indexer as TypeSenseIndexer
 from auditlog.context import set_actor
+from django.core.cache import cache
 
 # from .meilisearch import Indexer as MeiliSearchIndexer
 
@@ -98,6 +99,17 @@ def query_index(keywords, category=None, tag=None, page=1, prepare_external=True
     return items, result.num_pages, result.count, duplicated_items
 
 
+_fetch_lock_key = "_fetch_lock"
+_fetch_lock_ttl = 2
+
+
+def get_fetch_lock():
+    if cache.get(_fetch_lock_key):
+        return False
+    cache.set(_fetch_lock_key, 1, timeout=_fetch_lock_ttl)
+    return True
+
+
 def enqueue_fetch(url, is_refetch, user=None):
     job_id = "fetch_" + hashlib.md5(url.encode()).hexdigest()
     in_progress = False

catalog/search/views.py

@@ -16,7 +16,7 @@ from rq.job import Job
 from .external import ExternalSources
 from django.core.cache import cache
 import hashlib
-from .models import query_index, enqueue_fetch
+from .models import get_fetch_lock, query_index, enqueue_fetch
 
 _logger = logging.getLogger(__name__)
 
@@ -68,7 +68,9 @@ def fetch(request, url, is_refetch: bool = False, site: AbstractSite | None = No
                 "!refetch": [url, None],
             }
         )
-    job_id = enqueue_fetch(url, is_refetch, request.user)
+    job_id = None
+    if is_refetch or get_fetch_lock():
+        job_id = enqueue_fetch(url, is_refetch, request.user)
     return render(
         request,
         "fetch_pending.html",
@@ -98,7 +100,7 @@ def search(request):
             },
         )
 
-    if request.user.is_authenticated and keywords.find("://") > 0:
+    if keywords.find("://") > 0:
         site = SiteManager.get_site_by_url(keywords)
         if site:
             return fetch(request, keywords, False, site)

catalog/templates/fetch_pending.html

@@ -21,12 +21,16 @@
     <main>
       <div class="grid__main">
         <article>
-          <h5>正在从{{ site.SITE_NAME.label }}获取</h5>
-          <div hx-get="{% url 'catalog:fetch_refresh' job_id %}"
-               hx-trigger="load delay:2s"
-               hx-swap="outerHTML">
-            <i class="fa-solid fa-compact-disc fa-spin loading"></i>
-          </div>
+          {% if job_id %}
+            <h5>正在从{{ site.SITE_NAME.label }}获取</h5>
+            <div hx-get="{% url 'catalog:fetch_refresh' job_id %}"
+                 hx-trigger="load delay:2s"
+                 hx-swap="outerHTML">
+              <i class="fa-solid fa-compact-disc fa-spin loading"></i>
+            </div>
+          {% else %}
+            <h5>获取系统繁忙，请稍等几秒钟再搜索</h5>
+          {% endif %}
         </article>
       </div>
       <aside class="grid__aside bottom">

users/models.py

@@ -18,7 +18,7 @@ from django.db.models import Q, F, Value
 from django.db.models.functions import Concat
 from django.templatetags.static import static
 import hashlib
-
+from loguru import logger
 
 RESERVED_USERNAMES = [
     "connect",
@@ -394,7 +394,7 @@ class User(AbstractUser):
             self.merge_relationships()
             updated = True
         elif code == 401:
-            print(f"401 {self}")
+            logger.error(f"401 {self}")
             self.mastodon_token = ""
         return updated