From 10ef657baf4c8cfa7f5c785fab0581af80382382 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 18 Jul 2024 09:49:25 -0400 Subject: [PATCH] reduce error log by censored content from douban --- catalog/common/downloaders.py | 1 + journal/importers/douban.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/catalog/common/downloaders.py b/catalog/common/downloaders.py index d0e9503a..e1c55f64 100644 --- a/catalog/common/downloaders.py +++ b/catalog/common/downloaders.py @@ -107,6 +107,7 @@ class DownloadError(Exception): def __init__(self, downloader, msg=None): self.url = downloader.url self.logs = downloader.logs + self.response_type = downloader.response_type if downloader.response_type == RESPONSE_INVALID_CONTENT: error = "Invalid Response" elif downloader.response_type == RESPONSE_NETWORK_ERROR: diff --git a/journal/importers/douban.py b/journal/importers/douban.py index 5feab816..b89a8059 100644 --- a/journal/importers/douban.py +++ b/journal/importers/douban.py @@ -328,6 +328,12 @@ class DoubanImporter: else: # logger.info(f"matched {url}") print(".", end="", flush=True) + except DownloadError as e: + if e.response_type == RESPONSE_CENSORSHIP: + # avoid flood error log since there are too many + logger.warning(f"fetching error: {url}", extra={"exception": e}) + else: + logger.error(f"fetching error: {url}", extra={"exception": e}) except Exception as e: logger.error(f"fetching error: {url}", extra={"exception": e}) if item is None: