From 10ef657baf4c8cfa7f5c785fab0581af80382382 Mon Sep 17 00:00:00 2001
From: Your Name <you@example.com>
Date: Thu, 18 Jul 2024 09:49:25 -0400
Subject: [PATCH] reduce error log by censored content from douban

---
 catalog/common/downloaders.py | 1 +
 journal/importers/douban.py   | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/catalog/common/downloaders.py b/catalog/common/downloaders.py
index d0e9503a..e1c55f64 100644
--- a/catalog/common/downloaders.py
+++ b/catalog/common/downloaders.py
@@ -107,6 +107,7 @@ class DownloadError(Exception):
     def __init__(self, downloader, msg=None):
         self.url = downloader.url
         self.logs = downloader.logs
+        self.response_type = downloader.response_type
         if downloader.response_type == RESPONSE_INVALID_CONTENT:
             error = "Invalid Response"
         elif downloader.response_type == RESPONSE_NETWORK_ERROR:
diff --git a/journal/importers/douban.py b/journal/importers/douban.py
index 5feab816..b89a8059 100644
--- a/journal/importers/douban.py
+++ b/journal/importers/douban.py
@@ -328,6 +328,12 @@ class DoubanImporter:
             else:
                 # logger.info(f"matched {url}")
                 print(".", end="", flush=True)
+        except DownloadError as e:
+            if e.response_type == RESPONSE_CENSORSHIP:
+                # avoid flood error log since there are too many
+                logger.warning(f"fetching error: {url}", extra={"exception": e})
+            else:
+                logger.error(f"fetching error: {url}", extra={"exception": e})
         except Exception as e:
             logger.error(f"fetching error: {url}", extra={"exception": e})
         if item is None: