Merge branch 'main' of https://github.com/neodb-social/neodb
Some checks are pending
code check / lint (3.12) (push) Waiting to run
code check / type-checker (3.12) (push) Waiting to run
Mirror to Codeberg / to_codeberg (push) Waiting to run
unit test / django (3.12) (push) Waiting to run

This commit is contained in:
gesang 2025-03-12 09:12:17 +01:00
commit 6dbb872411
Signed by: gesang
GPG key ID: 6CE35141D31CEAFB
6 changed files with 40 additions and 1 deletions

View file

@ -62,6 +62,7 @@ jobs:
NEODB_SITE_NAME: test NEODB_SITE_NAME: test
NEODB_SITE_DOMAIN: test.domain NEODB_SITE_DOMAIN: test.domain
NEODB_SECRET_KEY: test NEODB_SECRET_KEY: test
SPOTIFY_API_KEY: TEST
run: | run: |
python manage.py compilemessages -i .venv -l zh_Hans python manage.py compilemessages -i .venv -l zh_Hans
python manage.py test python manage.py test

View file

@ -100,7 +100,7 @@ env = environ.FileAwareEnv(
# INTEGRATED TAKAHE CONFIGURATION # INTEGRATED TAKAHE CONFIGURATION
TAKAHE_DB_URL=(str, "postgres://takahe:takahepass@127.0.0.1:5432/takahe"), TAKAHE_DB_URL=(str, "postgres://takahe:takahepass@127.0.0.1:5432/takahe"),
# Spotify - https://developer.spotify.com/ # Spotify - https://developer.spotify.com/
SPOTIFY_API_KEY=(str, "TESTONLY"), SPOTIFY_API_KEY=(str, ""), # Set to empty string to enable scraping without API key
# The Movie Database (TMDB) - https://developer.themoviedb.org/ # The Movie Database (TMDB) - https://developer.themoviedb.org/
TMDB_API_V3_KEY=(str, "TESTONLY"), TMDB_API_V3_KEY=(str, "TESTONLY"),
# Google Books - https://developers.google.com/books/docs/v1/using - not used at the moment # Google Books - https://developers.google.com/books/docs/v1/using - not used at the moment

View file

@ -37,6 +37,13 @@ class SpotifyTestCase(TestCase):
# self.assertIsNotNone(site) # self.assertIsNotNone(site)
# self.assertEqual(site.id_value, t_id_value2) # self.assertEqual(site.id_value, t_id_value2)
@use_local_response
def test_scrape_web(self):
t_url = "https://open.spotify.com/album/65KwtzkJXw7oT819NFWmEP"
site = SiteManager.get_site_by_url(t_url)
r = site.scrape_web()
self.assertEqual(r.metadata["localized_title"][0]["text"], "The Race For Space")
@use_local_response @use_local_response
def test_scrape(self): def test_scrape(self):
t_url = "https://open.spotify.com/album/65KwtzkJXw7oT819NFWmEP" t_url = "https://open.spotify.com/album/65KwtzkJXw7oT819NFWmEP"

View file

@ -40,7 +40,34 @@ class Spotify(AbstractSite):
def id_to_url(cls, id_value): def id_to_url(cls, id_value):
return f"https://open.spotify.com/album/{id_value}" return f"https://open.spotify.com/album/{id_value}"
def scrape_web(self):
content = BasicDownloader(self.url).download().html()
txt: str = content.xpath("//script[@type='application/ld+json']/text()")[0] # type:ignore
schema_data = json.loads(txt)
title = schema_data["name"]
localized_title = [{"lang": detect_language(title), "text": title}]
localized_desc = []
release_date = schema_data.get("datePublished", None)
artist = []
genre = []
omebed_url = f"https://open.spotify.com/oembed?url={self.url}"
omebed_json = BasicDownloader(omebed_url).download().json()
image_url = omebed_json.get("thumbnail_url", None)
pd = ResourceContent(
metadata={
"localized_title": localized_title,
"localized_description": localized_desc,
"artist": artist,
"genre": genre,
"release_date": release_date,
"cover_image_url": image_url,
}
)
return pd
def scrape(self): def scrape(self):
if not settings.SPOTIFY_CREDENTIAL:
return self.scrape_web()
api_url = f"https://api.spotify.com/v1/albums/{self.id_value}" api_url = f"https://api.spotify.com/v1/albums/{self.id_value}"
headers = { headers = {
"Authorization": f"Bearer {get_spotify_token()}", "Authorization": f"Bearer {get_spotify_token()}",
@ -113,6 +140,8 @@ class Spotify(AbstractSite):
async def search_task( async def search_task(
cls, q: str, page: int, category: str, page_size: int cls, q: str, page: int, category: str, page_size: int
) -> list[ExternalSearchResultItem]: ) -> list[ExternalSearchResultItem]:
if not settings.SPOTIFY_CREDENTIAL:
return []
if category not in ["music", "all"]: if category not in ["music", "all"]:
return [] return []
results = [] results = []

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1 @@
{"html":"<iframe style=\"border-radius: 12px\" width=\"100%\" height=\"352\" title=\"Spotify Embed: The Race For Space\" frameborder=\"0\" allowfullscreen allow=\"autoplay; clipboard-write; encrypted-media; fullscreen; picture-in-picture\" loading=\"lazy\" src=\"https://open.spotify.com/embed/album/65KwtzkJXw7oT819NFWmEP?utm_source=oembed\"></iframe>","iframe_url":"https://open.spotify.com/embed/album/65KwtzkJXw7oT819NFWmEP?utm_source=oembed","width":456,"height":352,"version":"1.0","provider_name":"Spotify","provider_url":"https://spotify.com","type":"rich","title":"The Race For Space","thumbnail_url":"https://image-cdn-ak.spotifycdn.com/image/ab67616d00001e02123ebfc7ca99a9bb6342cd36","thumbnail_width":300,"thumbnail_height":300}