Itmens/lib.itmens
Itmens/lib.itmens
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 3

local proxy

Browse source
This commit is contained in:
Your Name 2022-06-21 19:42:00 -04:00
parent 3525f4b7d1
commit b84ebd3383
1 changed files with 11 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
common/scrapers/douban.py
View file

@ -55,7 +55,12 @@ class DoubanScrapperMixin:
content = None
last_error = 'censorship'
error = error + 'Not found or hidden by Douban'
elif r.status_code == 204:
content = None
last_error = 'censorship'
error = error + 'Not found or hidden by Douban'
else:
content = None
last_error = 'network'
error = error + str(r.status_code)
@ -109,10 +114,7 @@ class DoubanScrapperMixin:
def latest():
nonlocal r, error, content
if settings.LOCAL_PROXY is not None:
error = error + '\nLocal: '
get(f'{settings.LOCAL_PROXY}?url={url}')
elif settings.SCRAPESTACK_KEY is not None:
if settings.SCRAPESTACK_KEY is not None:
error = error + '\nScrapeStack: '
get(f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}')
elif settings.SCRAPERAPI_KEY is not None:
@ -126,9 +128,13 @@ class DoubanScrapperMixin:
error = error + '\nProxyCrawl: '
get(f'https://api.proxycrawl.com/?token={settings.PROXYCRAWL_KEY}&url={url}')
check_content()
if last_error == 'censorship' and settings.LOCAL_PROXY is not None:
error = error + '\nLocal: '
get(f'{settings.LOCAL_PROXY}?url={url}')
check_content()
latest()
if content is None and settings.LOCAL_PROXY is None:
if content is None:
wayback_cdx()
if content is None: