Itmens/lib.itmens
Itmens/lib.itmens
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

reduce error log by censored content from douban

Browse source
This commit is contained in:
Your Name 2024-07-18 09:49:25 -04:00 • committed by Henri Dickson
parent 4e995a970a
commit 10ef657baf
2 changed files with 7 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
catalog/common/downloaders.py
View file

@ -107,6 +107,7 @@ class DownloadError(Exception):
def __init__(self, downloader, msg=None):
self.url = downloader.url
self.logs = downloader.logs
self.response_type = downloader.response_type
if downloader.response_type == RESPONSE_INVALID_CONTENT:
error = "Invalid Response"
elif downloader.response_type == RESPONSE_NETWORK_ERROR:

6
journal/importers/douban.py
View file

@ -328,6 +328,12 @@ class DoubanImporter:
else:
# logger.info(f"matched {url}")
print(".", end="", flush=True)
except DownloadError as e:
if e.response_type == RESPONSE_CENSORSHIP:
# avoid flood error log since there are too many
logger.warning(f"fetching error: {url}", extra={"exception": e})
else:
logger.error(f"fetching error: {url}", extra={"exception": e})
except Exception as e:
logger.error(f"fetching error: {url}", extra={"exception": e})
if item is None: