Itmens/lib.itmens
Itmens/lib.itmens
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix discogs url regex; normalize upc/gtin across douban/spotify/discogs

Browse source
This commit is contained in:
Your Name 2023-02-03 16:33:58 -05:00 • committed by Henri Dickson
parent 40a3fa4e43
commit 0a3a371701
8 changed files with 226 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
catalog/common/models.py
View file

@ -44,7 +44,7 @@ class IdType(models.TextChoices):
ISSN = "issn", _("ISSN")
CUBN = "cubn", _("统一书号")
ISRC = "isrc", _("ISRC") # only for songs
GTIN = "gtin", _("GTIN UPC EAN码") # ISBN is separate
GTIN = "gtin", _("GTIN UPC EAN码") # GTIN-13, ISBN is separate
RSS = "rss", _("RSS Feed URL")
IMDB = "imdb", _("IMDb")
TMDB_TV = "tmdb_tv", _("TMDB剧集")

22
catalog/music/tests.py
View file

@ -1,6 +1,16 @@
from django.test import TestCase
from catalog.common import *
from catalog.models import *
from catalog.music.utils import *
class BasicMusicTest(TestCase):
def test_gtin(self):
self.assertIsNone(upc_to_gtin_13("018771208112X"))
self.assertIsNone(upc_to_gtin_13("999018771208112"))
self.assertEqual(upc_to_gtin_13("018771208112"), "0018771208112")
self.assertEqual(upc_to_gtin_13("00042281006722"), "0042281006722")
self.assertEqual(upc_to_gtin_13("0042281006722"), "0042281006722")
class SpotifyTestCase(TestCase):
@ -60,6 +70,14 @@ class MultiMusicSitesTestCase(TestCase):
p2 = SiteManager.get_site_by_url(url2).get_resource_ready()
self.assertEqual(p1.item.id, p2.item.id)
@use_local_response
def test_albums_discogs(self):
url1 = "https://www.discogs.com/release/13574140"
url2 = "https://open.spotify.com/album/0I8vpSE1bSmysN2PhmHoQg"
p1 = SiteManager.get_site_by_url(url1).get_resource_ready()
p2 = SiteManager.get_site_by_url(url2).get_resource_ready()
self.assertEqual(p1.item.id, p2.item.id)
class BandcampTestCase(TestCase):
def test_parse(self):
@ -98,6 +116,8 @@ class DiscogsReleaseTestCase(TestCase):
site = SiteManager.get_site_by_url(t_url)
self.assertEqual(site.url, t_url_2)
self.assertEqual(site.id_value, t_id_value)
site = SiteManager.get_site_by_url(t_url_2)
self.assertIsNotNone(site)
@use_local_response
def test_scrape(self):
@ -109,7 +129,7 @@ class DiscogsReleaseTestCase(TestCase):
self.assertEqual(site.resource.metadata["title"], "The Never Story")
self.assertEqual(site.resource.metadata["artist"], ["J.I.D"])
self.assertIsInstance(site.resource.item, Album)
self.assertEqual(site.resource.item.barcode, "602445804689")
self.assertEqual(site.resource.item.barcode, "0602445804689")
class DiscogsMasterTestCase(TestCase):

20
catalog/music/utils.py Normal file
View file

@ -0,0 +1,20 @@
import re
def upc_to_gtin_13(upc: str):
"""
Convert UPC-A to GTIN-13, return None if validation failed
may add or remove padding 0s from different source
"""
s = upc.strip() if upc else ""
if not re.match(r"^\d+$", s):
return None
if len(s) < 13:
s = s.zfill(13)
elif len(s) > 13:
if re.match(r"^0+$", s[0 : len(s) - 13]):
s = s[len(s) - 13 :]
else:
return None
return s

17
catalog/sites/discogs.py
View file

@ -4,6 +4,7 @@ Discogs.
from django.conf import settings
from catalog.common import *
from catalog.models import *
from catalog.music.utils import upc_to_gtin_13
from .douban import *
import json
import logging
@ -17,12 +18,12 @@ _logger = logging.getLogger(__name__)
class DiscogsRelease(AbstractSite):
SITE_NAME = SiteName.Discogs
ID_TYPE = IdType.Discogs_Release
URL_PATTERNS = [r"https://www\.discogs\.com/release/(\d+)-.+"]
URL_PATTERNS = [r"https://www\.discogs\.com/release/(\d+)[^\d]*"]
WIKI_PROPERTY_ID = "?"
DEFAULT_MODEL = Album
@classmethod
def id_to_url(self, id_value):
def id_to_url(cls, id_value):
return f"https://www.discogs.com/release/{id_value}"
def scrape(self):
@ -31,7 +32,9 @@ class DiscogsRelease(AbstractSite):
artist = [artist.get("name") for artist in release.get("artists")]
genre = release.get("genres")
track_list = [track.get("title") for track in release.get("tracklist")]
company = [company.get("name") for company in release.get("companies")]
company = list(
set([company.get("name") for company in release.get("companies")])
)
media, disc_count = None, None
formats = release.get("formats")
@ -44,7 +47,9 @@ class DiscogsRelease(AbstractSite):
if identifiers:
for i in identifiers:
if i["type"] == "Barcode":
barcode = i["value"].replace(" ", "").replace("-", "")
barcode = upc_to_gtin_13(
i["value"].replace(" ", "").replace("-", "")
)
image_url = None
if len(release.get("images")) > 0:
image_url = release["images"][0].get("uri")
@ -79,12 +84,12 @@ class DiscogsRelease(AbstractSite):
class DiscogsMaster(AbstractSite):
SITE_NAME = SiteName.Discogs
ID_TYPE = IdType.Discogs_Master
URL_PATTERNS = [r"https://www\.discogs\.com/master/(\d+)-.+"]
URL_PATTERNS = [r"https://www\.discogs\.com/master/(\d+)[^\d]*"]
WIKI_PROPERTY_ID = "?"
DEFAULT_MODEL = Album
@classmethod
def id_to_url(self, id_value):
def id_to_url(cls, id_value):
return f"https://www.discogs.com/master/{id_value}"
def scrape(self):

5
catalog/sites/douban_music.py
View file

@ -1,5 +1,6 @@
from catalog.common import *
from catalog.models import *
from catalog.music.utils import upc_to_gtin_13
from .douban import DoubanDownloader
import dateparser
import logging
@ -20,7 +21,7 @@ class DoubanMusic(AbstractSite):
DEFAULT_MODEL = Album
@classmethod
def id_to_url(self, id_value):
def id_to_url(cls, id_value):
return "https://music.douban.com/subject/" + id_value + "/"
def scrape(self):
@ -114,7 +115,7 @@ class DoubanMusic(AbstractSite):
"//div[@id='info']//span[text()='条形码:']/following-sibling::text()[1]"
)
if other_elem:
gtin = other_elem[0].strip()
gtin = upc_to_gtin_13(other_elem[0].strip())
other_elem = content.xpath(
"//div[@id='info']//span[text()='碟片数:']/following-sibling::text()[1]"
)

9
catalog/sites/spotify.py
View file

@ -4,6 +4,7 @@ Spotify
from django.conf import settings
from catalog.common import *
from catalog.models import *
from catalog.music.utils import upc_to_gtin_13
from .douban import *
import time
import datetime
@ -28,11 +29,11 @@ class Spotify(AbstractSite):
DEFAULT_MODEL = Album
@classmethod
def id_to_url(self, id_value):
def id_to_url(cls, id_value):
return f"https://open.spotify.com/album/{id_value}"
def scrape(self):
api_url = "https://api.spotify.com/v1/albums/" + self.id_value
api_url = f"https://api.spotify.com/v1/albums/{self.id_value}"
headers = {"Authorization": f"Bearer {get_spotify_token()}"}
res_data = BasicDownloader(api_url, headers=headers).download().json()
artist = []
@ -70,9 +71,9 @@ class Spotify(AbstractSite):
gtin = None
if res_data["external_ids"].get("upc"):
gtin = res_data["external_ids"].get("upc")
gtin = upc_to_gtin_13(res_data["external_ids"].get("upc"))
if res_data["external_ids"].get("ean"):
gtin = res_data["external_ids"].get("ean")
gtin = upc_to_gtin_13(res_data["external_ids"].get("ean"))
isrc = None
if res_data["external_ids"].get("isrc"):
isrc = res_data["external_ids"].get("isrc")

1
test_data/https___api_discogs_com_releases_13574140 Normal file
View file

File diff suppressed because one or more lines are too long

164
test_data/https___api_spotify_com_v1_albums_0I8vpSE1bSmysN2PhmHoQg Normal file
View file

@ -0,0 +1,164 @@
{
"album_type" : "album",
"artists" : [ {
"external_urls" : {
"spotify" : "https://open.spotify.com/artist/0F3Aew9DSd6fb6192K1K0Y"
},
"href" : "https://api.spotify.com/v1/artists/0F3Aew9DSd6fb6192K1K0Y",
"id" : "0F3Aew9DSd6fb6192K1K0Y",
"name" : "Keith Jarrett",
"type" : "artist",
"uri" : "spotify:artist:0F3Aew9DSd6fb6192K1K0Y"
} ],
"available_markets" : [ "AD", "AE", "AG", "AL", "AM", "AO", "AR", "AT", "AU", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BN", "BO", "BR", "BS", "BT", "BW", "BY", "BZ", "CA", "CD", "CG", "CH", "CI", "CL", "CM", "CO", "CR", "CV", "CW", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "ES", "FI", "FJ", "FM", "FR", "GA", "GB", "GD", "GE", "GH", "GM", "GN", "GQ", "GR", "GT", "GW", "GY", "HK", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IN", "IQ", "IS", "IT", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KR", "KW", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MG", "MH", "MK", "ML", "MN", "MO", "MR", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NE", "NG", "NI", "NL", "NO", "NP", "NR", "NZ", "OM", "PA", "PE", "PG", "PH", "PK", "PL", "PS", "PT", "PW", "PY", "QA", "RO", "RS", "RW", "SA", "SB", "SC", "SE", "SG", "SI", "SK", "SL", "SM", "SN", "SR", "ST", "SV", "SZ", "TD", "TG", "TH", "TJ", "TL", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "US", "UY", "UZ", "VC", "VE", "VN", "VU", "WS", "XK", "ZA", "ZM", "ZW" ],
"copyrights" : [ {
"text" : "© 1975 ECM Records GmbH, under exclusive license to Deutsche Grammophon GmbH, Berlin",
"type" : "C"
}, {
"text" : "℗ 1975 ECM Records GmbH, under exclusive license to Deutsche Grammophon GmbH, Berlin",
"type" : "P"
} ],
"external_ids" : {
"upc" : "00042281006722"
},
"external_urls" : {
"spotify" : "https://open.spotify.com/album/0I8vpSE1bSmysN2PhmHoQg"
},
"genres" : [ ],
"href" : "https://api.spotify.com/v1/albums/0I8vpSE1bSmysN2PhmHoQg",
"id" : "0I8vpSE1bSmysN2PhmHoQg",
"images" : [ {
"height" : 640,
"url" : "https://i.scdn.co/image/ab67616d0000b27364d8292b7825888183e4da28",
"width" : 640
}, {
"height" : 300,
"url" : "https://i.scdn.co/image/ab67616d00001e0264d8292b7825888183e4da28",
"width" : 300
}, {
"height" : 64,
"url" : "https://i.scdn.co/image/ab67616d0000485164d8292b7825888183e4da28",
"width" : 64
} ],
"label" : "ECM Records",
"name" : "The Köln Concert",
"popularity" : 48,
"release_date" : "1975-11-30",
"release_date_precision" : "day",
"total_tracks" : 4,
"tracks" : {
"href" : "https://api.spotify.com/v1/albums/0I8vpSE1bSmysN2PhmHoQg/tracks?offset=0&limit=50",
"items" : [ {
"artists" : [ {
"external_urls" : {
"spotify" : "https://open.spotify.com/artist/0F3Aew9DSd6fb6192K1K0Y"
},
"href" : "https://api.spotify.com/v1/artists/0F3Aew9DSd6fb6192K1K0Y",
"id" : "0F3Aew9DSd6fb6192K1K0Y",
"name" : "Keith Jarrett",
"type" : "artist",
"uri" : "spotify:artist:0F3Aew9DSd6fb6192K1K0Y"
} ],
"available_markets" : [ "AD", "AE", "AG", "AL", "AM", "AO", "AR", "AT", "AU", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BN", "BO", "BR", "BS", "BT", "BW", "BY", "BZ", "CA", "CD", "CG", "CH", "CI", "CL", "CM", "CO", "CR", "CV", "CW", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "ES", "ET", "FI", "FJ", "FM", "FR", "GA", "GB", "GD", "GE", "GH", "GM", "GN", "GQ", "GR", "GT", "GW", "GY", "HK", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IN", "IQ", "IS", "IT", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KR", "KW", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MG", "MH", "MK", "ML", "MN", "MO", "MR", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NE", "NG", "NI", "NL", "NO", "NP", "NR", "NZ", "OM", "PA", "PE", "PG", "PH", "PK", "PL", "PS", "PT", "PW", "PY", "QA", "RO", "RS", "RW", "SA", "SB", "SC", "SE", "SG", "SI", "SK", "SL", "SM", "SN", "SR", "ST", "SV", "SZ", "TD", "TG", "TH", "TJ", "TL", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "US", "UY", "UZ", "VC", "VE", "VN", "VU", "WS", "XK", "ZA", "ZM", "ZW" ],
"disc_number" : 1,
"duration_ms" : 1561600,
"explicit" : false,
"external_urls" : {
"spotify" : "https://open.spotify.com/track/0T4KV1pj8as2xvdHZAP5ae"
},
"href" : "https://api.spotify.com/v1/tracks/0T4KV1pj8as2xvdHZAP5ae",
"id" : "0T4KV1pj8as2xvdHZAP5ae",
"is_local" : false,
"name" : "Köln, January 24, 1975, Pt. I - Live",
"preview_url" : null,
"track_number" : 1,
"type" : "track",
"uri" : "spotify:track:0T4KV1pj8as2xvdHZAP5ae"
}, {
"artists" : [ {
"external_urls" : {
"spotify" : "https://open.spotify.com/artist/0F3Aew9DSd6fb6192K1K0Y"
},
"href" : "https://api.spotify.com/v1/artists/0F3Aew9DSd6fb6192K1K0Y",
"id" : "0F3Aew9DSd6fb6192K1K0Y",
"name" : "Keith Jarrett",
"type" : "artist",
"uri" : "spotify:artist:0F3Aew9DSd6fb6192K1K0Y"
} ],
"available_markets" : [ "AD", "AE", "AG", "AL", "AM", "AO", "AR", "AT", "AU", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BN", "BO", "BR", "BS", "BT", "BW", "BY", "BZ", "CA", "CD", "CG", "CH", "CI", "CL", "CM", "CO", "CR", "CV", "CW", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "ES", "ET", "FI", "FJ", "FM", "FR", "GA", "GB", "GD", "GE", "GH", "GM", "GN", "GQ", "GR", "GT", "GW", "GY", "HK", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IN", "IQ", "IS", "IT", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KR", "KW", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MG", "MH", "MK", "ML", "MN", "MO", "MR", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NE", "NG", "NI", "NL", "NO", "NP", "NR", "NZ", "OM", "PA", "PE", "PG", "PH", "PK", "PL", "PS", "PT", "PW", "PY", "QA", "RO", "RS", "RW", "SA", "SB", "SC", "SE", "SG", "SI", "SK", "SL", "SM", "SN", "SR", "ST", "SV", "SZ", "TD", "TG", "TH", "TJ", "TL", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "US", "UY", "UZ", "VC", "VE", "VN", "VU", "WS", "XK", "ZA", "ZM", "ZW" ],
"disc_number" : 1,
"duration_ms" : 894733,
"explicit" : false,
"external_urls" : {
"spotify" : "https://open.spotify.com/track/3hrt1fLgEZIZPONUbcDY0c"
},
"href" : "https://api.spotify.com/v1/tracks/3hrt1fLgEZIZPONUbcDY0c",
"id" : "3hrt1fLgEZIZPONUbcDY0c",
"is_local" : false,
"name" : "Köln, January 24, 1975, Pt. II A - Live",
"preview_url" : null,
"track_number" : 2,
"type" : "track",
"uri" : "spotify:track:3hrt1fLgEZIZPONUbcDY0c"
}, {
"artists" : [ {
"external_urls" : {
"spotify" : "https://open.spotify.com/artist/0F3Aew9DSd6fb6192K1K0Y"
},
"href" : "https://api.spotify.com/v1/artists/0F3Aew9DSd6fb6192K1K0Y",
"id" : "0F3Aew9DSd6fb6192K1K0Y",
"name" : "Keith Jarrett",
"type" : "artist",
"uri" : "spotify:artist:0F3Aew9DSd6fb6192K1K0Y"
} ],
"available_markets" : [ "AD", "AE", "AG", "AL", "AM", "AO", "AR", "AT", "AU", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BN", "BO", "BR", "BS", "BT", "BW", "BY", "BZ", "CA", "CD", "CG", "CH", "CI", "CL", "CM", "CO", "CR", "CV", "CW", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "ES", "ET", "FI", "FJ", "FM", "FR", "GA", "GB", "GD", "GE", "GH", "GM", "GN", "GQ", "GR", "GT", "GW", "GY", "HK", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IN", "IQ", "IS", "IT", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KR", "KW", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MG", "MH", "MK", "ML", "MN", "MO", "MR", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NE", "NG", "NI", "NL", "NO", "NP", "NR", "NZ", "OM", "PA", "PE", "PG", "PH", "PK", "PL", "PS", "PT", "PW", "PY", "QA", "RO", "RS", "RW", "SA", "SB", "SC", "SE", "SG", "SI", "SK", "SL", "SM", "SN", "SR", "ST", "SV", "SZ", "TD", "TG", "TH", "TJ", "TL", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "US", "UY", "UZ", "VC", "VE", "VN", "VU", "WS", "XK", "ZA", "ZM", "ZW" ],
"disc_number" : 1,
"duration_ms" : 1094493,
"explicit" : false,
"external_urls" : {
"spotify" : "https://open.spotify.com/track/5E2TkoLzEpqvxwWzkgh0kT"
},
"href" : "https://api.spotify.com/v1/tracks/5E2TkoLzEpqvxwWzkgh0kT",
"id" : "5E2TkoLzEpqvxwWzkgh0kT",
"is_local" : false,
"name" : "Köln, January 24, 1975, Pt. II B - Live",
"preview_url" : null,
"track_number" : 3,
"type" : "track",
"uri" : "spotify:track:5E2TkoLzEpqvxwWzkgh0kT"
}, {
"artists" : [ {
"external_urls" : {
"spotify" : "https://open.spotify.com/artist/0F3Aew9DSd6fb6192K1K0Y"
},
"href" : "https://api.spotify.com/v1/artists/0F3Aew9DSd6fb6192K1K0Y",
"id" : "0F3Aew9DSd6fb6192K1K0Y",
"name" : "Keith Jarrett",
"type" : "artist",
"uri" : "spotify:artist:0F3Aew9DSd6fb6192K1K0Y"
} ],
"available_markets" : [ "AD", "AE", "AG", "AL", "AM", "AO", "AR", "AT", "AU", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BN", "BO", "BR", "BS", "BT", "BW", "BY", "BZ", "CA", "CD", "CG", "CH", "CI", "CL", "CM", "CO", "CR", "CV", "CW", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "ES", "ET", "FI", "FJ", "FM", "FR", "GA", "GB", "GD", "GE", "GH", "GM", "GN", "GQ", "GR", "GT", "GW", "GY", "HK", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IN", "IQ", "IS", "IT", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KR", "KW", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MG", "MH", "MK", "ML", "MN", "MO", "MR", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NE", "NG", "NI", "NL", "NO", "NP", "NR", "NZ", "OM", "PA", "PE", "PG", "PH", "PK", "PL", "PS", "PT", "PW", "PY", "QA", "RO", "RS", "RW", "SA", "SB", "SC", "SE", "SG", "SI", "SK", "SL", "SM", "SN", "SR", "ST", "SV", "SZ", "TD", "TG", "TH", "TJ", "TL", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "US", "UY", "UZ", "VC", "VE", "VN", "VU", "WS", "XK", "ZA", "ZM", "ZW" ],
"disc_number" : 1,
"duration_ms" : 416840,
"explicit" : false,
"external_urls" : {
"spotify" : "https://open.spotify.com/track/4FMI4Ln1LhyxGldkNXan5e"
},
"href" : "https://api.spotify.com/v1/tracks/4FMI4Ln1LhyxGldkNXan5e",
"id" : "4FMI4Ln1LhyxGldkNXan5e",
"is_local" : false,
"name" : "Köln, January 24, 1975, Pt. II C - Live",
"preview_url" : null,
"track_number" : 4,
"type" : "track",
"uri" : "spotify:track:4FMI4Ln1LhyxGldkNXan5e"
} ],
"limit" : 50,
"next" : null,
"offset" : 0,
"previous" : null,
"total" : 4
},
"type" : "album",
"uri" : "spotify:album:0I8vpSE1bSmysN2PhmHoQg"
}