Itmens/lib.itmens
Itmens/lib.itmens
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

allow disable proxy

Browse source
This commit is contained in:
Your Name 2021-09-10 22:57:40 -04:00
parent 18df8ee3d7
commit 3395d7b300
1 changed files with 8 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
common/scraper.py
View file

@ -156,8 +156,8 @@ class AbstractScraper:
'http': proxy_url,
'https': proxy_url,
}
# if settings.DEBUG:
# proxies = None
if settings.LUMINATI_USERNAME is None:
proxies = None
r = requests.get(url, proxies=proxies,
headers=headers, timeout=TIMEOUT)
@ -179,8 +179,8 @@ class AbstractScraper:
'http': proxy_url,
'https': proxy_url,
}
# if settings.DEBUG:
# proxies = None
if settings.LUMINATI_USERNAME is None:
proxies = None
if url:
img_response = requests.get(
url,
@ -224,7 +224,10 @@ class DoubanScrapperMixin:
def download_page(cls, url, headers):
url = cls.get_effective_url(url)
scraper_api_endpoint = f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={url}'
if settings.SCRAPERAPI_KEY is None:
scraper_api_endpoint = url
else:
scraper_api_endpoint = f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={url}'
r = requests.get(scraper_api_endpoint, timeout=TIMEOUT)