From a6e24d630319be59363bc80daf496229da547944 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 30 Nov 2021 20:58:32 -0500 Subject: [PATCH] scrapestack --- boofilsic/settings.py | 2 ++ common/scraper.py | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/boofilsic/settings.py b/boofilsic/settings.py index 28d38aef..132b9bdc 100644 --- a/boofilsic/settings.py +++ b/boofilsic/settings.py @@ -247,6 +247,8 @@ LUMINATI_PASSWORD = 'nsb7te9bw0ney' # ScraperAPI api key SCRAPERAPI_KEY = 'wnb3794v675b8w475h0e8hr7tyge' +PROXYCRAWL_KEY = None +SCRAPESTACK_KEY = None # Spotify credentials SPOTIFY_CREDENTIAL = "NzYzNkYTE6MGQ0ODY0NTY2Y2b3n645sdfgAyY2I1ljYjg3Nzc0MjIwODQ0ZWE=" diff --git a/common/scraper.py b/common/scraper.py index cb5d32b0..15bb7331 100644 --- a/common/scraper.py +++ b/common/scraper.py @@ -310,7 +310,10 @@ class DoubanScrapperMixin: def latest(): nonlocal r, error, content - if settings.SCRAPERAPI_KEY is None: + if settings.SCRAPESTACK_KEY is not None: + error = error + '\nScrapeStack: ' + get(f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}', 30) + elif settings.SCRAPERAPI_KEY is None: error = error + '\nDirect: ' get(url, 30) else: @@ -338,7 +341,9 @@ class DoubanScrapperMixin: ext = None dl_url = url - if settings.SCRAPERAPI_KEY is not None: + if settings.SCRAPESTACK_KEY is not None: + dl_url = f'http://api.scrapestack.com/scrape?access_key={settings.SCRAPESTACK_KEY}&url={url}' + elif settings.SCRAPERAPI_KEY is not None: dl_url = f'http://api.scraperapi.com?api_key={settings.SCRAPERAPI_KEY}&url={url}' try: