Merge branch 'main' of https://github.com/neodb-social/neodb

.github/workflows/tests.yml

@@ -62,6 +62,7 @@ jobs:
         NEODB_SITE_NAME: test
         NEODB_SITE_DOMAIN: test.domain
         NEODB_SECRET_KEY: test
+        SPOTIFY_API_KEY: TEST
       run: |
         python manage.py compilemessages -i .venv -l zh_Hans
         python manage.py test

boofilsic/settings.py

@@ -100,7 +100,7 @@ env = environ.FileAwareEnv(
     # INTEGRATED TAKAHE CONFIGURATION
     TAKAHE_DB_URL=(str, "postgres://takahe:takahepass@127.0.0.1:5432/takahe"),
     # Spotify - https://developer.spotify.com/
-    SPOTIFY_API_KEY=(str, "TESTONLY"),
+    SPOTIFY_API_KEY=(str, ""),  # Set to empty string to enable scraping without API key
     # The Movie Database (TMDB) - https://developer.themoviedb.org/
     TMDB_API_V3_KEY=(str, "TESTONLY"),
     # Google Books - https://developers.google.com/books/docs/v1/using - not used at the moment

catalog/music/tests.py

@@ -37,6 +37,13 @@ class SpotifyTestCase(TestCase):
         # self.assertIsNotNone(site)
         # self.assertEqual(site.id_value, t_id_value2)
 
+    @use_local_response
+    def test_scrape_web(self):
+        t_url = "https://open.spotify.com/album/65KwtzkJXw7oT819NFWmEP"
+        site = SiteManager.get_site_by_url(t_url)
+        r = site.scrape_web()
+        self.assertEqual(r.metadata["localized_title"][0]["text"], "The Race For Space")
+
     @use_local_response
     def test_scrape(self):
         t_url = "https://open.spotify.com/album/65KwtzkJXw7oT819NFWmEP"

catalog/sites/spotify.py

@@ -40,7 +40,34 @@ class Spotify(AbstractSite):
     def id_to_url(cls, id_value):
         return f"https://open.spotify.com/album/{id_value}"
 
+    def scrape_web(self):
+        content = BasicDownloader(self.url).download().html()
+        txt: str = content.xpath("//script[@type='application/ld+json']/text()")[0]  # type:ignore
+        schema_data = json.loads(txt)
+        title = schema_data["name"]
+        localized_title = [{"lang": detect_language(title), "text": title}]
+        localized_desc = []
+        release_date = schema_data.get("datePublished", None)
+        artist = []
+        genre = []
+        omebed_url = f"https://open.spotify.com/oembed?url={self.url}"
+        omebed_json = BasicDownloader(omebed_url).download().json()
+        image_url = omebed_json.get("thumbnail_url", None)
+        pd = ResourceContent(
+            metadata={
+                "localized_title": localized_title,
+                "localized_description": localized_desc,
+                "artist": artist,
+                "genre": genre,
+                "release_date": release_date,
+                "cover_image_url": image_url,
+            }
+        )
+        return pd
+
     def scrape(self):
+        if not settings.SPOTIFY_CREDENTIAL:
+            return self.scrape_web()
         api_url = f"https://api.spotify.com/v1/albums/{self.id_value}"
         headers = {
             "Authorization": f"Bearer {get_spotify_token()}",
@@ -113,6 +140,8 @@ class Spotify(AbstractSite):
     async def search_task(
         cls, q: str, page: int, category: str, page_size: int
     ) -> list[ExternalSearchResultItem]:
+        if not settings.SPOTIFY_CREDENTIAL:
+            return []
         if category not in ["music", "all"]:
             return []
         results = []

test_data/https___open_spotify_com_oembed_url_https___open_spotify_com_album_65KwtzkJXw7oT819NFWmEP

@@ -0,0 +1 @@
+{"html":"<iframe style=\"border-radius: 12px\" width=\"100%\" height=\"352\" title=\"Spotify Embed: The Race For Space\" frameborder=\"0\" allowfullscreen allow=\"autoplay; clipboard-write; encrypted-media; fullscreen; picture-in-picture\" loading=\"lazy\" src=\"https://open.spotify.com/embed/album/65KwtzkJXw7oT819NFWmEP?utm_source=oembed\"></iframe>","iframe_url":"https://open.spotify.com/embed/album/65KwtzkJXw7oT819NFWmEP?utm_source=oembed","width":456,"height":352,"version":"1.0","provider_name":"Spotify","provider_url":"https://spotify.com","type":"rich","title":"The Race For Space","thumbnail_url":"https://image-cdn-ak.spotifycdn.com/image/ab67616d00001e02123ebfc7ca99a9bb6342cd36","thumbnail_width":300,"thumbnail_height":300}