diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7d5db5e2..e735419e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -62,6 +62,7 @@ jobs: NEODB_SITE_NAME: test NEODB_SITE_DOMAIN: test.domain NEODB_SECRET_KEY: test + SPOTIFY_API_KEY: TEST run: | python manage.py compilemessages -i .venv -l zh_Hans python manage.py test diff --git a/boofilsic/settings.py b/boofilsic/settings.py index 3f74a9cb..00f9ceba 100644 --- a/boofilsic/settings.py +++ b/boofilsic/settings.py @@ -100,7 +100,7 @@ env = environ.FileAwareEnv( # INTEGRATED TAKAHE CONFIGURATION TAKAHE_DB_URL=(str, "postgres://takahe:takahepass@127.0.0.1:5432/takahe"), # Spotify - https://developer.spotify.com/ - SPOTIFY_API_KEY=(str, "TESTONLY"), + SPOTIFY_API_KEY=(str, ""), # Set to empty string to enable scraping without API key # The Movie Database (TMDB) - https://developer.themoviedb.org/ TMDB_API_V3_KEY=(str, "TESTONLY"), # Google Books - https://developers.google.com/books/docs/v1/using - not used at the moment diff --git a/catalog/music/tests.py b/catalog/music/tests.py index d489fe7a..6ff7634f 100644 --- a/catalog/music/tests.py +++ b/catalog/music/tests.py @@ -37,6 +37,13 @@ class SpotifyTestCase(TestCase): # self.assertIsNotNone(site) # self.assertEqual(site.id_value, t_id_value2) + @use_local_response + def test_scrape_web(self): + t_url = "https://open.spotify.com/album/65KwtzkJXw7oT819NFWmEP" + site = SiteManager.get_site_by_url(t_url) + r = site.scrape_web() + self.assertEqual(r.metadata["localized_title"][0]["text"], "The Race For Space") + @use_local_response def test_scrape(self): t_url = "https://open.spotify.com/album/65KwtzkJXw7oT819NFWmEP" diff --git a/catalog/sites/spotify.py b/catalog/sites/spotify.py index b03b038f..8745976e 100644 --- a/catalog/sites/spotify.py +++ b/catalog/sites/spotify.py @@ -40,7 +40,34 @@ class Spotify(AbstractSite): def id_to_url(cls, id_value): return f"https://open.spotify.com/album/{id_value}" + def scrape_web(self): + content = BasicDownloader(self.url).download().html() + txt: str = content.xpath("//script[@type='application/ld+json']/text()")[0] # type:ignore + schema_data = json.loads(txt) + title = schema_data["name"] + localized_title = [{"lang": detect_language(title), "text": title}] + localized_desc = [] + release_date = schema_data.get("datePublished", None) + artist = [] + genre = [] + omebed_url = f"https://open.spotify.com/oembed?url={self.url}" + omebed_json = BasicDownloader(omebed_url).download().json() + image_url = omebed_json.get("thumbnail_url", None) + pd = ResourceContent( + metadata={ + "localized_title": localized_title, + "localized_description": localized_desc, + "artist": artist, + "genre": genre, + "release_date": release_date, + "cover_image_url": image_url, + } + ) + return pd + def scrape(self): + if not settings.SPOTIFY_CREDENTIAL: + return self.scrape_web() api_url = f"https://api.spotify.com/v1/albums/{self.id_value}" headers = { "Authorization": f"Bearer {get_spotify_token()}", @@ -113,6 +140,8 @@ class Spotify(AbstractSite): async def search_task( cls, q: str, page: int, category: str, page_size: int ) -> list[ExternalSearchResultItem]: + if not settings.SPOTIFY_CREDENTIAL: + return [] if category not in ["music", "all"]: return [] results = [] diff --git a/test_data/https___open_spotify_com_album_65KwtzkJXw7oT819NFWmEP b/test_data/https___open_spotify_com_album_65KwtzkJXw7oT819NFWmEP new file mode 100644 index 00000000..d10f48ee --- /dev/null +++ b/test_data/https___open_spotify_com_album_65KwtzkJXw7oT819NFWmEP @@ -0,0 +1 @@ +Unsupported browser
Logo

Unsupported browser

Spotify is unavailable on this browser. For the best listening experience update your browser or download the Spotify app.

Get the AppLearn more
\ No newline at end of file diff --git a/test_data/https___open_spotify_com_oembed_url_https___open_spotify_com_album_65KwtzkJXw7oT819NFWmEP b/test_data/https___open_spotify_com_oembed_url_https___open_spotify_com_album_65KwtzkJXw7oT819NFWmEP new file mode 100644 index 00000000..9559aeef --- /dev/null +++ b/test_data/https___open_spotify_com_oembed_url_https___open_spotify_com_album_65KwtzkJXw7oT819NFWmEP @@ -0,0 +1 @@ +{"html":"","iframe_url":"https://open.spotify.com/embed/album/65KwtzkJXw7oT819NFWmEP?utm_source=oembed","width":456,"height":352,"version":"1.0","provider_name":"Spotify","provider_url":"https://spotify.com","type":"rich","title":"The Race For Space","thumbnail_url":"https://image-cdn-ak.spotifycdn.com/image/ab67616d00001e02123ebfc7ca99a9bb6342cd36","thumbnail_width":300,"thumbnail_height":300} \ No newline at end of file