diff --git a/catalog/common/sites.py b/catalog/common/sites.py index 7708863f..16e3dd11 100644 --- a/catalog/common/sites.py +++ b/catalog/common/sites.py @@ -107,8 +107,8 @@ class AbstractSite: data = ResourceContent() return data - def scrape_additional_data(self): - pass + def scrape_additional_data(self) -> bool: + return False @staticmethod def query_str(content, query: str) -> str: diff --git a/catalog/jobs/podcast.py b/catalog/jobs/podcast.py index f21e960a..571425a3 100644 --- a/catalog/jobs/podcast.py +++ b/catalog/jobs/podcast.py @@ -26,8 +26,11 @@ class PodcastUpdater(BaseJob): logger.info(f"updating {p}") c = p.episodes.count() site = RSS(p.feed_url) - site.scrape_additional_data() - c2 = p.episodes.count() - logger.info(f"updated {p}, {c2 - c} new episodes.") - count += c2 - c + r = site.scrape_additional_data() + if r: + c2 = p.episodes.count() + logger.info(f"updated {p}, {c2 - c} new episodes.") + count += c2 - c + else: + logger.warning(f"failed to update {p}") logger.info(f"Podcasts update finished, {count} new episodes total.") diff --git a/catalog/sites/rss.py b/catalog/sites/rss.py index 0207cbb9..e37429f6 100644 --- a/catalog/sites/rss.py +++ b/catalog/sites/rss.py @@ -111,13 +111,14 @@ class RSS(AbstractSite): return pd def scrape_additional_data(self): - item = self.get_item() - if not item: - logger.error(f"item for RSS {self.url} not found") - return feed = self.parse_feed_from_url(self.url) if not feed: - return + logger.warning(f"unable to parse RSS {self.url}") + return False + item = self.get_item() + if not item: + logger.warning(f"item for RSS {self.url} not found") + return False for episode in feed["episodes"]: PodcastEpisode.objects.get_or_create( program=item, @@ -139,3 +140,4 @@ class RSS(AbstractSite): "link": episode.get("link"), }, ) + return True