From d8d07c3777a1447c4b190bda9af8174df94aa620 Mon Sep 17 00:00:00 2001 From: mein Name Date: Fri, 28 Feb 2025 06:03:33 -0500 Subject: [PATCH] use cached 302 if present, in web request --- catalog/apis.py | 2 +- catalog/common/sites.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/catalog/apis.py b/catalog/apis.py index 8585a3fc..fb1f5bcd 100644 --- a/catalog/apis.py +++ b/catalog/apis.py @@ -125,7 +125,7 @@ def fetch_item(request, url: str): Some site may take ~90 seconds to fetch. If not getting the item after 120 seconds, please stop and consider the URL is not available. """ - site = SiteManager.get_site_by_url(url) + site = SiteManager.get_site_by_url(url, detect_redirection=False) if not site: return 404, {"message": "URL not supported"} item = site.get_item() diff --git a/catalog/common/sites.py b/catalog/common/sites.py index b683b090..6c8c81ab 100644 --- a/catalog/common/sites.py +++ b/catalog/common/sites.py @@ -309,13 +309,15 @@ class SiteManager: raise ValueError(f"Site for {typ} not found") @staticmethod - def get_redirected_url(url: str) -> str: + def get_redirected_url(url: str, allow_head: bool = True) -> str: k = "_redir_" + md5(url.encode()).hexdigest() u = cache.get(k, default=None) if u == "": return url elif u: return u + elif not allow_head: + return url try: u = requests.head(url, allow_redirects=True, timeout=2).url except requests.RequestException: @@ -351,7 +353,7 @@ class SiteManager: strict_query=False, ): return None - u = SiteManager.get_redirected_url(url) if detect_redirection else url + u = SiteManager.get_redirected_url(url, allow_head=detect_redirection) cls = SiteManager.get_class_by_url(u) if cls is None and u != url: cls = SiteManager.get_fallback_class_by_url(url)